From 71c8d3fd4100ba59f6d0eb7f0377eb83bc26dd4a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Feb 2012 15:13:08 -0800 Subject: [PATCH 001/688] reorganize directory structure From 76cacbd9918397c85c7fce56571a8658dd9c827c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 14 Feb 2012 15:51:32 -0800 Subject: [PATCH 002/688] Added coherence tile function defs, with traits and constants --- uncore/coherence.scala | 128 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 uncore/coherence.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala new file mode 100644 index 00000000..460214e1 --- /dev/null +++ b/uncore/coherence.scala @@ -0,0 +1,128 @@ +package Top { + +import Chisel._ +import Constants._ + +class TransactionInit extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionAbort extends Bundle { + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends Bundle { + val ptype = Bits(width = 2) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) +} + +class ProbeReply extends Bundle { + val ptype = Bits(width = 2) + val hasData = Bool() + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionReply extends Bundle { + val ttype = Bits(width = 2) + val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class TransactionFinish extends Bundle { + val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = new TransactionInit().asOutput + val xact_abort = new TransactionAbort().asInput + val probe_req = new ProbeRequest().asInput + val probe_rep = new ProbeReply().asOutput + val xact_rep = new TransactionReply().asInput + val xact_finish = new TransactionFinish().asOutput +} + +trait ThreeStateIncoherence { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + + def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { + val store = (cmd === M_XWR) + val load = (cmd === M_XRD) + val amo = cmd(3).toBool + val read = load || amo || (cmd === M_PFR) + val write = store || amo || (cmd === M_PFW) + (read, write) + } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + ( state === tileClean || state === tileDirty) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsWriteback (state: UFix): Bool = { + state === tileDirty + } + + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newState(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + } + def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, state) + } + +} + +trait FourStateCoherence { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + + def isHit ( cmd: Bits, state: UFix): Bool = { + val is_hit = Bool(false) + switch(cmd) { + is(M_XRD) { + is_hit := state === tileShared || + state === tileExclusiveClean || + state === tileExclusiveDirty + } + is(M_XWR) { + is_hit := state === tileExclusiveClean || + state === tileExclusiveDirty + } + } + is_hit + } + + def needsWriteback (state: UFix): Bool = { + state === tileExclusiveDirty + } + + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool + + def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { + val state = UFix(0) + switch(incoming.ptype) { + is(probeInvalidate) { state := tileInvalid } + is(probeDowngrade) { state := tileShared } + } + state.toBits + } +} + +} From ecbf47fe16327ae8a6c4453556a42b989df59dea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Feb 2012 13:54:36 -0800 Subject: [PATCH 003/688] Replace nbcache manipulation of meta state bits with abstracted functions --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 460214e1..7419555e 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -76,7 +76,7 @@ trait ThreeStateIncoherence { def newStateOnFlush() = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), tileInvalid)) + Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) From cfb818aa97e0c96a0b8bb033a5680d442b344433 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 16 Feb 2012 12:59:38 -0800 Subject: [PATCH 004/688] Abstract class for coherence policies --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7419555e..b55ac9a1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -47,9 +47,7 @@ class ioTileLink extends Bundle { val xact_finish = new TransactionFinish().asOutput } -trait ThreeStateIncoherence { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - +trait CoherencePolicy { def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) @@ -58,6 +56,10 @@ trait ThreeStateIncoherence { val write = store || amo || (cmd === M_PFW) (read, write) } +} + +trait ThreeStateIncoherence extends CoherencePolicy { + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -87,32 +89,35 @@ trait ThreeStateIncoherence { } -trait FourStateCoherence { +trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } def isHit ( cmd: Bits, state: UFix): Bool = { - val is_hit = Bool(false) - switch(cmd) { - is(M_XRD) { - is_hit := state === tileShared || - state === tileExclusiveClean || - state === tileExclusiveDirty - } - is(M_XWR) { - is_hit := state === tileExclusiveClean || - state === tileExclusiveDirty - } - } - is_hit + val (read, write) = cpuCmdToRW(cmd) + ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || + (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + } + + def isValid (state: UFix): Bool = { + state != tileInvalid } def needsWriteback (state: UFix): Bool = { state === tileExclusiveDirty } + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + + // TODO: New funcs as compared to incoherent protocol: + def newState(cmd: Bits, state: UFix): UFix + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnPrimaryMiss(cmd: Bits): UFix + def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix + def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { From b9c42a80c87dfae75110004a651d3bccc3217bc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 10:12:13 -0800 Subject: [PATCH 005/688] Added coherence message type enums --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b55ac9a1..9fcf32eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) val data = Bits(width = MEM_DATA_BITS) @@ -15,20 +15,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = 2) + val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) } class TransactionReply extends Bundle { - val ttype = Bits(width = 2) + val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) val data = Bits(width = MEM_DATA_BITS) From 33a26424bd1cdc52298a97dcc30952b8fe30d05a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 12:14:57 -0800 Subject: [PATCH 006/688] Refining tilelink interface --- uncore/coherence.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fcf32eb..cfde62ba 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -39,12 +39,12 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = new TransactionInit().asOutput - val xact_abort = new TransactionAbort().asInput - val probe_req = new ProbeRequest().asInput - val probe_rep = new ProbeReply().asOutput - val xact_rep = new TransactionReply().asInput - val xact_finish = new TransactionFinish().asOutput + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,4 +130,6 @@ trait FourStateCoherence extends CoherencePolicy { } } + + } From c3646e2e6411c3003a9683b4674a023b780c3848 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 22 Feb 2012 18:24:52 -0800 Subject: [PATCH 007/688] Improved TileIO organization, beginnings of hub implementation --- uncore/coherence.scala | 92 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index cfde62ba..776a64e0 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -24,6 +24,9 @@ class ProbeReply extends Bundle { val ptype = Bits(width = PTYPE_BITS) val hasData = Bool() val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -31,6 +34,9 @@ class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class TransactionReplyData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -39,12 +45,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_abort = (new ioDecoupled) { new TransactionAbort() } + val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } trait CoherencePolicy { @@ -130,6 +138,78 @@ trait FourStateCoherence extends CoherencePolicy { } } +class XactTracker(id: Int) extends Component { + val io = new Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit() } + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip + val mem_req = (new ioDecoupled) { new MemReq() }.flip + val xact_finish = Bool(INPUT) + val tile_id_in = Bits(TILE_ID_BITS, INPUT) + val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) + val ongoing_addr = Bits(PADDR_BITS, OUTPUT) + val busy = Bool(OUTPUT) + } + + val valid = Reg(resetVal = Bool(false)) + val addr = Reg{ Bits() } + val ttype = Reg{ Bits() } + val tile_id = Reg{ Bits() } + val tile_xact_id = Reg{ Bits() } + val probe_done = Reg{ Bits() } + +} + +abstract class CoherenceHub extends Component + +class CoherenceHubNoDir extends CoherenceHub { + val io = new Bundle { + val tiles = Vec(NTILES) { new ioTileLink() } + val mem = new ioDCache().flip + } + + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + + // In parallel, every cycle: nack conflicting transactions, free finished ones + for( j <- 0 until NTILES ) { + val init = io.tiles(j).xact_init + val abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + val busys = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busys(i) := t.busy + conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + } + abort.valid := conflicts.orR || busys.andR + abort.bits.tileTransactionID := init.bits.tileTransactionID + //if abort.rdy, init.pop() + + } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + val freed = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + } + t.xact_finish := freed.orR + //finish.pop() + } + + // Forward memory responses from mem to tile + //for( j <- until NTILES ) { + // tiles(j).xact_rep.ttype = + // tiles(j).xact_rep.tileTransactionID = + // tiles(j).xact_rep.globalTransactionID = + // val data = Bits + // + // Pick a single request of these types to process + //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } + //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } } + +} From ffb88467db080f48cb2d62f7a308613b50440cde Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 17:49:28 -0800 Subject: [PATCH 008/688] finished xact_rep transactor in coherence hub --- uncore/coherence.scala | 96 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 776a64e0..3744fdb6 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,6 +2,7 @@ package Top { import Chisel._ import Constants._ +import hwacha.GenArray class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -146,10 +147,12 @@ class XactTracker(id: Int) extends Component { val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip val mem_req = (new ioDecoupled) { new MemReq() }.flip val xact_finish = Bool(INPUT) - val tile_id_in = Bits(TILE_ID_BITS, INPUT) - val tile_id_out = Bits(TILE_ID_BITS, OUTPUT) - val ongoing_addr = Bits(PADDR_BITS, OUTPUT) val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val ttype = Bits(TTYPE_BITS, OUTPUT) } val valid = Reg(resetVal = Bool(false)) @@ -164,47 +167,104 @@ class XactTracker(id: Int) extends Component { abstract class CoherenceHub extends Component class CoherenceHubNoDir extends CoherenceHub { + + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { + addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) + } + def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + val ret = Wire() { Bits(width = TTYPE_BITS) } + switch (ttype) { + is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } + is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } + is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } + is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } + } + ret + } + val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + for( i <- 0 until NGLOBAL_XACTS) { + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) + ttype_arr.write( UFix(i), trackerList(i).io.ttype) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + } // In parallel, every cycle: nack conflicting transactions, free finished ones for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) - val busys = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - busys(i) := t.busy - conflicts(i) := t.busy && init.valid && (t.ongoing_addr === init.bits.address) + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := conflicts.orR || busys.andR + abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) abort.bits.tileTransactionID := init.bits.tileTransactionID - //if abort.rdy, init.pop() - + // TODO: + // Reg(aborted) := (abort.ready && abort.valid) + // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) + // init.rdy = aborted || allocated } + +/* +// Todo: which implementation is clearer? for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io val freed = Bits(width = NTILES) for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - freed(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) + finish.ready := Bool(true) // finsh.pop() } t.xact_finish := freed.orR - //finish.pop() + } +*/ + + free_arr := Bits(0, width=NGLOBAL_XACTS) + for( j <- 0 until NTILES ) { + val finish = io.tiles(j).xact_finish + when(finish.valid) { + free_arr.write(finish.bits.globalTransactionID, Bool(true)) + } + finish.ready := Bool(true) } // Forward memory responses from mem to tile - //for( j <- until NTILES ) { - // tiles(j).xact_rep.ttype = - // tiles(j).xact_rep.tileTransactionID = - // tiles(j).xact_rep.globalTransactionID = - // val data = Bits - // + val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val xrep_cnt_next = xrep_cnt + UFix(1) + when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } + val idx = io.mem.resp_tag + val readys = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data + readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) + val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val + io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) + io.tiles(j).xact_rep_data.valid := this_rep_valid + } + // If there were a ready signal due to e.g. intervening network: + //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } From 1dcf25586f9997f512cf88f025174262b12f9c52 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Feb 2012 18:12:50 -0800 Subject: [PATCH 009/688] finished xact_finish and xact_abort transactors in coherence hub --- uncore/coherence.scala | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3744fdb6..534c4bb9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -205,7 +205,9 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.xact_finish := free_arr.read(UFix(i)) } - // In parallel, every cycle: nack conflicting transactions, free finished ones + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + val initiating = Wire() { Bits(width = NTILES) } for( j <- 0 until NTILES ) { val init = io.tiles(j).xact_init val abort = io.tiles(j).xact_abort @@ -214,34 +216,16 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - abort.valid := init.valid && (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + abort.valid := init.valid && aborting abort.bits.tileTransactionID := init.bits.tileTransactionID - // TODO: - // Reg(aborted) := (abort.ready && abort.valid) - // Reg(allocated) : = had_priority(j) & !(abort.ready && abort.valid) - // init.rdy = aborted || allocated + init.ready := aborting(j) || initiating(j) } - -/* -// Todo: which implementation is clearer? - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - val freed = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - val finish = io.tiles(j).xact_finish - free(j) := finish.valid && (UFix(i) === finish.bits.globalTransactionID) - finish.ready := Bool(true) // finsh.pop() - } - t.xact_finish := freed.orR - } -*/ - free_arr := Bits(0, width=NGLOBAL_XACTS) + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - when(finish.valid) { - free_arr.write(finish.bits.globalTransactionID, Bool(true)) - } + free_arr.write(finish.bits.globalTransactionID, finish.valid) finish.ready := Bool(true) } From b3cf8f3f354b064c139ae040038d70c438542d19 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 12:56:09 -0800 Subject: [PATCH 010/688] Better abstraction of data bundles --- uncore/coherence.scala | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 534c4bb9..579af518 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,13 +4,27 @@ import Chisel._ import Constants._ import hwacha.GenArray +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + class TransactionInit extends Bundle { val ttype = Bits(width = TTYPE_BITS) val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) - val data = Bits(width = MEM_DATA_BITS) } +class TransactionInitData extends MemData + class TransactionAbort extends Bundle { val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) } @@ -27,9 +41,7 @@ class ProbeReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class ProbeReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class ProbeReplyData extends MemData class TransactionReply extends Bundle { val ttype = Bits(width = TTYPE_BITS) @@ -37,9 +49,7 @@ class TransactionReply extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} +class TransactionReplyData extends MemData class TransactionFinish extends Bundle { val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) @@ -47,6 +57,7 @@ class TransactionFinish extends Bundle { class ioTileLink extends Bundle { val xact_init = (new ioDecoupled) { new TransactionInit() }.flip + val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val xact_abort = (new ioDecoupled) { new TransactionAbort() } val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip From dfdcb7c696df600ddc9a234bf189fbd2600b3202 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:09 -0800 Subject: [PATCH 011/688] Better foldR --- uncore/coherence.scala | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 579af518..bb2f7d65 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -18,41 +18,43 @@ class MemData extends Bundle { } class TransactionInit extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class TransactionInitData extends MemData class TransactionAbort extends Bundle { - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } class ProbeRequest extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val ptype = Bits(width = PTYPE_BITS) - val hasData = Bool() - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val p_type = Bits(width = PTYPE_BITS) + val has_data = Bool() + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends Bundle { - val ttype = Bits(width = TTYPE_BITS) - val tileTransactionID = Bits(width = TILE_XACT_ID_BITS) - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class TransactionReplyData extends MemData class TransactionFinish extends Bundle { - val globalTransactionID = Bits(width = GLOBAL_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ioTileLink extends Bundle { @@ -142,7 +144,7 @@ trait FourStateCoherence extends CoherencePolicy { def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { val state = UFix(0) - switch(incoming.ptype) { + switch(incoming.p_type) { is(probeInvalidate) { state := tileInvalid } is(probeDowngrade) { state := tileShared } } From 8856e2b8bb7b5c5851d227704e8182c50ee1f01a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 25 Feb 2012 15:27:53 -0800 Subject: [PATCH 012/688] More stylish bundle param names, some hub progress --- uncore/coherence.scala | 105 ++++++++++++++++++++++++++++++----------- 1 file changed, 78 insertions(+), 27 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bb2f7d65..d7597cdf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,23 +154,27 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() }.flip - val mem_req = (new ioDecoupled) { new MemReq() }.flip + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val x_init_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val x_init_data_idx = Bits(log2up(NTILES), INPUT) + val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val ttype = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val pop_p_rep = Bool(OUTPUT) + val pop_p_rep_data = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } - val ttype = Reg{ Bits() } + val t_type = Reg{ Bits() } val tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } @@ -184,9 +188,9 @@ class CoherenceHubNoDir extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } - def getTransactionReplyType(ttype: UFix, count: UFix): Bits = { + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (ttype) { + switch (t_type) { is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } @@ -205,17 +209,27 @@ class CoherenceHubNoDir extends CoherenceHub { val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val ttype_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) - tile_xact_id_arr.write(UFix(i), trackerList(i).io.tile_xact_id) - ttype_arr.write( UFix(i), trackerList(i).io.ttype) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - trackerList(i).io.xact_finish := free_arr.read(UFix(i)) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + t_type_arr.write( UFix(i), trackerList(i).io.t_type) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) + trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) + trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } // Nack conflicting transaction init attempts @@ -231,14 +245,14 @@ class CoherenceHubNoDir extends CoherenceHub { } aborting(j) := (conflicts.orR || busy_arr.flatten().andR) abort.valid := init.valid && aborting - abort.bits.tileTransactionID := init.bits.tileTransactionID + abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - free_arr.write(finish.bits.globalTransactionID, finish.valid) + do_free_arr.write(finish.bits.global_xact_id, finish.valid) finish.ready := Bool(true) } @@ -249,18 +263,55 @@ class CoherenceHubNoDir extends CoherenceHub { val idx = io.mem.resp_tag val readys = Bits(width = NTILES) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.ttype := getTransactionReplyType(ttype_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tileTransactionID := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.globalTransactionID := idx + io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) + io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) + io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - val this_rep_valid = UFix(j) === tile_id_arr.read(idx) && io.mem.resp_val - io.tiles(j).xact_rep.valid := this_rep_valid && xrep_cnt === UFix(0) - io.tiles(j).xact_rep_data.valid := this_rep_valid + io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) } - // If there were a ready signal due to e.g. intervening network: + // If there were a ready signal due to e.g. intervening network use: //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + + val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + } + mem_req_arb.io.out.ready := io.mem.req_rdy + io.mem.req_val := mem_req_arb.io.out.valid + io.mem.req_rw := mem_req_arb.io.out.bits.rw + io.mem.req_tag := mem_req_arb.io.out.bits.tag + io.mem.req_addr := mem_req_arb.io.out.bits.addr + io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + Bits(0, width = MEM_DATA_BITS), + (0 until NTILES).map( j => + UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + io.tiles(j).probe_rep_data.bits.data, + io.tiles(j).xact_init_data.bits.data))) + + for( j <- 0 until NTILES ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_data_idx_arr.write(idx, UFix(j)) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES) { + val p_rep = io.tiles(j).probe_rep + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + rep_cnt_dec_arr.write(UFix(i), flags) + } + // Pick a single request of these types to process //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } From ca2e70454e7ad132e02478dac45cbfbb426c1f4d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 25 Feb 2012 17:09:26 -0800 Subject: [PATCH 013/688] change package name and sbt project name to rocket --- uncore/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7597cdf..d419dd52 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -1,4 +1,4 @@ -package Top { +package rocket import Chisel._ import Constants._ @@ -319,5 +319,3 @@ class CoherenceHubNoDir extends CoherenceHub { } - -} From e22106af3f3dfc2049d4214a93e60ee5fbd3616a Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:08 -0800 Subject: [PATCH 014/688] updating rocket code to lastest version of chisel, passes assembly tests in C++ and Verilog as long as you dont use the vector unit --- uncore/coherence.scala | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d419dd52..af0d27c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -2,7 +2,6 @@ package rocket import Chisel._ import Constants._ -import hwacha.GenArray class HubMemReq extends Bundle { val rw = Bool() @@ -205,18 +204,18 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) @@ -243,7 +242,7 @@ class CoherenceHubNoDir extends CoherenceHub { val t = trackerList(i).io conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) } - aborting(j) := (conflicts.orR || busy_arr.flatten().andR) + aborting(j) := (conflicts.orR || busy_arr.toBits().andR) abort.valid := init.valid && aborting abort.bits.tile_xact_id := init.bits.tile_xact_id init.ready := aborting(j) || initiating(j) From 0fd777f4807f1bf67d69fcbcac2a03aa14124da4 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Sun, 26 Feb 2012 17:24:23 -0800 Subject: [PATCH 015/688] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From b6e6d603cc77e6ab0cd46237922709b61b6ba562 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:34:40 -0800 Subject: [PATCH 016/688] xact init transactors in coherence hub --- uncore/coherence.scala | 125 +++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 43 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index af0d27c8..520f7c05 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -12,6 +12,12 @@ class HubMemReq extends Bundle { val is_probe_rep = Bool() } +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } @@ -154,6 +160,8 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) val x_init_has_data = Bool(INPUT) @@ -162,22 +170,27 @@ class XactTracker(id: Int) extends Component { val rep_cnt_dec = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) - val tile_id = Bits(TILE_ID_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) - val pop_p_rep = Bool(OUTPUT) - val pop_p_rep_data = Bool(OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) + val pop_x_init_data = Bool(OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } val valid = Reg(resetVal = Bool(false)) val addr = Reg{ Bits() } val t_type = Reg{ Bits() } - val tile_id = Reg{ Bits() } + val init_tile_id = Reg{ Bits() } val tile_xact_id = Reg{ Bits() } val probe_done = Reg{ Bits() } + //TODO: Decrement the probe count when final data piece is written + // Connent io.mem.ready sig to correct pop* outputs + // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component @@ -204,23 +217,23 @@ class CoherenceHubNoDir extends CoherenceHub { } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { busy_arr.write( UFix(i), trackerList(i).io.busy) addr_arr.write( UFix(i), trackerList(i).io.addr) - tile_id_arr.write( UFix(i), trackerList(i).io.tile_id) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) @@ -231,23 +244,6 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) } - // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } - val initiating = Wire() { Bits(width = NTILES) } - for( j <- 0 until NTILES ) { - val init = io.tiles(j).xact_init - val abort = io.tiles(j).xact_abort - val conflicts = Bits(width = NGLOBAL_XACTS) - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, init.bits.address) - } - aborting(j) := (conflicts.orR || busy_arr.toBits().andR) - abort.valid := init.valid && aborting - abort.bits.tile_xact_id := init.bits.tile_xact_id - init.ready := aborting(j) || initiating(j) - } - // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -255,6 +251,7 @@ class CoherenceHubNoDir extends CoherenceHub { finish.ready := Bool(true) } + // Reply to initial requestor // Forward memory responses from mem to tile val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val xrep_cnt_next = xrep_cnt + UFix(1) @@ -267,11 +264,11 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).xact_rep.bits.global_xact_id := idx io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === tile_id_arr.read(idx)) + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) + io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -293,28 +290,70 @@ class CoherenceHubNoDir extends CoherenceHub { io.tiles(j).probe_rep_data.bits.data, io.tiles(j).xact_init_data.bits.data))) + // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data) + p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) p_rep_data_idx_arr.write(idx, UFix(j)) - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data))(_ || _) + p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } rep_cnt_dec_arr.write(UFix(i), flags) } + // Nack conflicting transaction init attempts + val aborting = Wire() { Bits(width = NTILES) } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_abort = io.tiles(j).xact_abort + val conflicts = Bits(width = NGLOBAL_XACTS) + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + // Don't abort writebacks stalled on mem. + // TODO: This assumes overlapped writeback init reqs to + // the same addr will never be issued; is this ok? + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + val want_to_abort = conflicts.orR || busy_arr.flatten().andR + x_abort.valid := want_to_abort && x_init.valid + aborting(j) := want_to_abort && x_abort.ready + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + } - // Pick a single request of these types to process - //val xact_init_arb = (new Arbiter(NTILES)) { new TransactionInit() } - //val probe_reply_arb = (new Arbiter(NTILES)) { new ProbeReply() } + for( j <- 0 until NTILES ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.init_tile_id := UFix(j) + init_arb.io.in(j).bits.data_valid := x_init_data.valid + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) } From 70cdf869ac94af7c50405218cb0cf1a87cc76f0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 26 Feb 2012 00:53:07 -0800 Subject: [PATCH 017/688] probe req transactors in coherence hub --- uncore/coherence.scala | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 520f7c05..8cc616de 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -161,6 +161,7 @@ class XactTracker(id: Int) extends Component { val io = new Bundle { val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) @@ -174,6 +175,7 @@ class XactTracker(id: Int) extends Component { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) @@ -356,4 +358,17 @@ class CoherenceHubNoDir extends CoherenceHub { !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + // Handle probe request generation + // Must arbitrate for each request port + for( j <- 0 until NTILES ) { + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.probe_req.bits + p_req_arb.io.in(i).ready := t.probe_req.ready + p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + } + p_req_arb.io.out <> io.tiles(j).probe_req + } + } From 3393dc2d31aaa857daf3d604c60e8d95e5493450 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 11:26:18 -0800 Subject: [PATCH 018/688] Added probe_req ready sigs, GenArray to Vec --- uncore/coherence.scala | 68 ++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8cc616de..8aff1592 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -168,7 +168,8 @@ class XactTracker(id: Int) extends Component { val x_init_has_data = Bool(INPUT) val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val rep_cnt_dec = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -218,32 +219,35 @@ class CoherenceHubNoDir extends CoherenceHub { val mem = new ioDCache().flip } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val busy_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } - val init_tile_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } - val sh_count_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val do_free_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } - val rep_cnt_dec_arr = GenArray(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } + val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) + busy_arr.write( UFix(i), trackerList(i).io.busy) + addr_arr.write( UFix(i), trackerList(i).io.addr) + init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) + tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write( UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) + sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) + send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) + trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) - trackerList(i).io.rep_cnt_dec := rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) + trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } // Free finished transactions @@ -308,7 +312,7 @@ class CoherenceHubNoDir extends CoherenceHub { val p_rep = io.tiles(j).probe_rep flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) } - rep_cnt_dec_arr.write(UFix(i), flags) + p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -326,7 +330,7 @@ class CoherenceHubNoDir extends CoherenceHub { // the same addr will never be issued; is this ok? } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.flatten().andR + val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid aborting(j) := want_to_abort && x_abort.ready } @@ -354,21 +358,27 @@ class CoherenceHubNoDir extends CoherenceHub { x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.flatten().andR && + alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) // Handle probe request generation // Must arbitrate for each request port + val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) for( j <- 0 until NTILES ) { - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() } for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.probe_req.bits - p_req_arb.io.in(i).ready := t.probe_req.ready - p_req_arb.io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) } - p_req_arb.io.out <> io.tiles(j).probe_req + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + for( i <- 0 until NGLOBAL_XACTS ) { + val flags = Bits(width = NTILES) + for( j <- 0 until NTILES ) { + flags(j) := p_req_arb_arr(j).io.in(i).ready + } + p_rep_cnt_dec_arr.write(UFix(i), flags) } } From 7d331858e25c840e19fb745b01b0465137230901 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 27 Feb 2012 18:36:09 -0800 Subject: [PATCH 019/688] replace ioDCache with ioMem --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 8aff1592..d5a5e414 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -216,7 +216,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioDCache().flip + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) From ffc7652e0166b38b4db2612e93358dd267e22206 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Feb 2012 19:10:15 -0800 Subject: [PATCH 020/688] Null coherence hub. Begin work on internal tracker logic --- uncore/coherence.scala | 163 +++++++++++++++++++++++++++++++++-------- 1 file changed, 133 insertions(+), 30 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d5a5e414..20c95e78 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -157,46 +157,149 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component { +class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) - val x_init_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val x_init_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + val p_req_cnt_inc = Bits(NTILES, INPUT) + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val t_type = Bits(TTYPE_BITS, OUTPUT) + val push_p_req = Bits(NTILES, OUTPUT) + val pop_p_rep = Bits(NTILES, OUTPUT) + val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) } - val valid = Reg(resetVal = Bool(false)) - val addr = Reg{ Bits() } - val t_type = Reg{ Bits() } - val init_tile_id = Reg{ Bits() } - val tile_xact_id = Reg{ Bits() } - val probe_done = Reg{ Bits() } + val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ Bits() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val probe_done = Reg{ Bits() } + val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.t_type := t_type_ + +/* +class HubMemReq extends Bundle { + val rw = Bool() + val addr = UFix(width = PADDR_BITS-OFFSET_BITS) + val tag = Bits(width = GLOBAL_XACT_ID_BITS) + // Figure out which data-in port to pull from + val data_idx = Bits(width = TILE_ID_BITS) + val is_probe_rep = Bool() +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val t_type = Bits(width = TTYPE_BITS) + val has_data = Bool() + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val address = Bits(width = PADDR_BITS) + val init_tile_id = Bits(width = TILE_ID_BITS) + val data_valid = Bool() +*/ +/* + when( alloc_req.valid && can_alloc ) { + valid := Bool(true) + addr := alloc_req.bits.xact_init.address + t_type := alloc_req.bits.xact_init.t_type + init_tile_id := alloc_req.bits.init_tile_id + tile_xact_id := alloc_req.bits.xact_init.tile_xact_id + [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 + } + when ( alloc_req.bits.data_valid ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + } + when( p_rep_has_data ) { + io.mem_req.valid := + io.mem_req.bits.rw := + io.mem_req.bits.addr := + io.mem_req.bits.tag := + io.mem_req.bits.data_idx := + io.mem_req.bits.is_probe_rep := + := io.mem.ready + + } + + val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + push_p_req = Bits(0, width = NTILES) + pop_p_rep = Bits(0, width = NTILES) + pop_p_rep_data = Bits(0, width = NTILES) + pop_x_init = Bool(false) + pop_x_init_data = Bool(false) + send_x_rep_ack = Bool(false) + + + } +*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component +abstract class CoherenceHub extends Component with CoherencePolicy + +class CoherenceHubNull extends Component { + val io = new Bundle { + val tile = new ioTileLink() + val mem = new ioMem() + } + + val x_init = io.tile.xact_init + val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + x_init.ready := io.mem.req_rdy + io.mem.req_val := x_init.valid + io.mem.req_rw := is_write + io.mem.req_tag := x_init.bits.tile_xact_id + io.mem.req_addr := x_init.bits.address + + val x_rep = io.tile.xact_rep + x_rep.bits.t_type := Bits(width = TTYPE_BITS) + x_rep.bits.has_data := !is_write + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.valid := io.mem.resp_val + + //TODO: + val x_init_data = io.tile.xact_init_data + val x_rep_data = io.tile.xact_rep_data + x_init_data.ready := io.mem.req_rdy + io.mem.req_wdata := x_init_data.bits.data + x_rep_data.bits.data := io.mem.resp_data + x_rep_data.valid := io.mem.resp_val + // Should be: + //io.mem.req_data <> x_init_data + //x_rep_data <> io.mem.resp_data + +} + class CoherenceHubNoDir extends CoherenceHub { From e15284d22ce5dcef9a2a1af46e4de9c81d0a3566 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 28 Feb 2012 17:33:06 -0800 Subject: [PATCH 021/688] Added temporary ioMemHub and made coherence hub implementations depend on it rather than ioMem --- uncore/coherence.scala | 281 +++++++++++++++++++++++------------------ 1 file changed, 157 insertions(+), 124 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 20c95e78..11c764ec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,13 +3,39 @@ package rocket import Chisel._ import Constants._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(PADDR_BITS - OFFSET_BITS) + val tag = Bits(MEM_TAG_BITS) +} + +class MemResp () extends Bundle +{ + val tag = Bits(MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) + val valid = Bool() +} + +class ioMemHub() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val req_data = (new ioDecoupled) { new MemData() }.flip + val resp = new MemResp() +} + class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } +} + +class HubProbeRep extends Bundle { + val reply = (new ioDecoupled) { new ProbeReply } + val data_idx = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -18,9 +44,6 @@ class TrackerAllocReq extends Bundle { val data_valid = Bool() } -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) @@ -49,15 +72,13 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData -class TransactionReply extends Bundle { +class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class TransactionReplyData extends MemData - class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -70,7 +91,6 @@ class ioTileLink extends Bundle { val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip val xact_rep = (new ioDecoupled) { new TransactionReply() } - val xact_rep_data = (new ioDecoupled) { new TransactionReplyData() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -160,18 +180,21 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val probe_rep = (new ioDecoupled) { new HubProbeRep() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val x_init_data = (new ioDecoupled) { new TransactionInitData() } val mem_req = (new ioDecoupled) { new HubMemReq() }.flip val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(log2up(NTILES), INPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) @@ -183,8 +206,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val send_x_rep_ack = Bool(OUTPUT) } - - val s_idle :: s_mem_r :: s_mem_w :: mem_wr :: s_probe :: Nil = Enum(5){ UFix() } + def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, + (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + } + + val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -192,73 +221,99 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val tile_xact_id_ = Reg{ Bits() } val probe_done = Reg{ Bits() } val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_rep_data_idx_ = Reg{ Bits() } + val x_init_data_needs_wb = Reg{ Bool() } + val p_rep_data_needs_wb = Reg{ Bool() } + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ -/* -class HubMemReq extends Bundle { - val rw = Bool() - val addr = UFix(width = PADDR_BITS-OFFSET_BITS) - val tag = Bits(width = GLOBAL_XACT_ID_BITS) - // Figure out which data-in port to pull from - val data_idx = Bits(width = TILE_ID_BITS) - val is_probe_rep = Bool() -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() -*/ -/* - when( alloc_req.valid && can_alloc ) { - valid := Bool(true) - addr := alloc_req.bits.xact_init.address - t_type := alloc_req.bits.xact_init.t_type - init_tile_id := alloc_req.bits.init_tile_id - tile_xact_id := alloc_req.bits.xact_init.tile_xact_id - [counter] := REFILL_CYCLES-1 if alloc_req.bits.xact_init.has_data else 0 - } - when ( alloc_req.bits.data_valid ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready - } - when( p_rep_has_data ) { - io.mem_req.valid := - io.mem_req.bits.rw := - io.mem_req.bits.addr := - io.mem_req.bits.tag := - io.mem_req.bits.data_idx := - io.mem_req.bits.is_probe_rep := - := io.mem.ready + io.mem_req.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.bits.addr := addr_ + io.mem_req.bits.req_cmd.bits.tag := UFix(id) + // := io.mem.ready //sent mem req + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr + io.push_p_req := Bits(0, width = NTILES) + io.pop_p_rep := Bits(0, width = NTILES) + io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_x_init := Bool(false) + io.pop_x_init_data := Bool(false) + io.send_x_rep_ack := Bool(false) + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.init_tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + p_rep_count := UFix(NTILES) + p_req_flags := ~Bits(0, width = NTILES) + state := s_probe + io.pop_x_init := Bool(true) + } + } + is(s_mem_r) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_w) { + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_busy } + } + is(s_mem_wr) { + when(io.probe_rep.bits.reply.bits.has_data) { + //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy + //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO + } . otherwise { + //io.pop_x_init := io.mem_req_rdy + //io.pop_x_init_data := io.mem_req_rdy + } + io.mem_req.valid := Bool(true) + when(io.mem_req.ready) { state := s_mem_r } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + val p_rep_has_data = Bool(INPUT) + val p_rep_data_idx = Bits(log2up(NTILES), INPUT) + val p_rep_cnt_dec = Bits(NTILES, INPUT) + when(io.p_rep_cnt_dec.orR) { + val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + p_rep_count := p_rep_count_next + when(p_rep_count_next === UFix(0)) { + state := s_busy //TODO: XXXXXXXXXX + } + } + when(p_rep_has_data) { + p_rep_data_needs_wb := Bool(true) + p_rep_data_idx_ := p_rep_data_idx + } + } + is(s_busy) { + when (io.xact_finish) { + state := s_idle + } + } } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - push_p_req = Bits(0, width = NTILES) - pop_p_rep = Bits(0, width = NTILES) - pop_p_rep_data = Bits(0, width = NTILES) - pop_x_init = Bool(false) - pop_x_init_data = Bool(false) - send_x_rep_ack = Bool(false) - - - } -*/ //TODO: Decrement the probe count when final data piece is written // Connent io.mem.ready sig to correct pop* outputs // P_rep and x_init must be popped on same cycle of receipt @@ -269,35 +324,24 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { val tile = new ioTileLink() - val mem = new ioMem() + val mem = new ioMemHub() } val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_rdy - io.mem.req_val := x_init.valid - io.mem.req_rw := is_write - io.mem.req_tag := x_init.bits.tile_xact_id - io.mem.req_addr := x_init.bits.address + x_init.ready := io.mem.req_cmd.ready + io.mem.req_cmd.valid := x_init.valid + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Bits(width = TTYPE_BITS) - x_rep.bits.has_data := !is_write - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp_tag) + x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.valid := io.mem.resp_val - - //TODO: - val x_init_data = io.tile.xact_init_data - val x_rep_data = io.tile.xact_rep_data - x_init_data.ready := io.mem.req_rdy - io.mem.req_wdata := x_init_data.bits.data - x_rep_data.bits.data := io.mem.resp_data - x_rep_data.valid := io.mem.resp_val - // Should be: - //io.mem.req_data <> x_init_data - //x_rep_data <> io.mem.resp_data - + x_rep.bits.data := io.mem.resp.data + x_rep.valid := io.mem.resp.valid } @@ -319,7 +363,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem + val mem = new ioMemHub } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -333,8 +377,6 @@ class CoherenceHubNoDir extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_has_data_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_data_idx_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=log2up(NTILES))} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } @@ -347,8 +389,6 @@ class CoherenceHubNoDir extends CoherenceHub { sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_has_data := p_rep_has_data_arr.read(UFix(i)) - trackerList(i).io.p_rep_data_idx := p_rep_data_idx_arr.read(UFix(i)) trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) } @@ -362,22 +402,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val xrep_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val xrep_cnt_next = xrep_cnt + UFix(1) - when (io.mem.resp_val) { xrep_cnt := xrep_cnt_next } - val idx = io.mem.resp_tag - val readys = Bits(width = NTILES) + val idx = io.mem.resp.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep_data.bits.data := io.mem.resp_data - readys := Mux(xrep_cnt === UFix(0), io.tiles(j).xact_rep.ready && io.tiles(j).xact_rep_data.ready, io.tiles(j).xact_rep_data.ready) - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && ((io.mem.resp_val && xrep_cnt === UFix(0)) || send_x_rep_ack_arr.read(idx)) - io.tiles(j).xact_rep_data.valid := (UFix(j) === init_tile_id_arr.read(idx)) + io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp_rdy := readys(init_tile_id_arr.read(idx)).xact_rep.ready + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -387,33 +421,30 @@ class CoherenceHubNoDir extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { mem_req_arb.io.in(i) <> trackerList(i).io.mem_req } - mem_req_arb.io.out.ready := io.mem.req_rdy - io.mem.req_val := mem_req_arb.io.out.valid - io.mem.req_rw := mem_req_arb.io.out.bits.rw - io.mem.req_tag := mem_req_arb.io.out.bits.tag - io.mem.req_addr := mem_req_arb.io.out.bits.addr - io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - Bits(0, width = MEM_DATA_BITS), - (0 until NTILES).map( j => - UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - io.tiles(j).probe_rep_data.bits.data, - io.tiles(j).xact_init_data.bits.data))) + //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready + io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd + io.mem.req_data <> mem_req_arb.io.out.bits.req_data + //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, + // Bits(0, width = MEM_DATA_BITS), + // (0 until NTILES).map( j => + // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, + // io.tiles(j).probe_rep_data.bits.data, + // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep_has_data_arr.write(idx, p_rep.valid && p_rep.bits.has_data && p_rep_data.valid) - p_rep_data_idx_arr.write(idx, UFix(j)) p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) val flags = Bits(width = NTILES) for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && !p_rep.bits.has_data && (p_rep.bits.global_xact_id === UFix(i)) + flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } p_rep_cnt_dec_arr.write(UFix(i), flags) } @@ -448,6 +479,8 @@ class CoherenceHubNoDir extends CoherenceHub { trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) } for( j <- 0 until NTILES ) { From eec369a1c74d019443181b96afb21acfb4eb8e36 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 28 Feb 2012 18:59:15 -0800 Subject: [PATCH 022/688] separate memory request command and data also, merge some VLSI/C++ test harness functionality --- uncore/coherence.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 11c764ec..75c430a3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -424,12 +424,6 @@ class CoherenceHubNoDir extends CoherenceHub { //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd io.mem.req_data <> mem_req_arb.io.out.bits.req_data - //io.mem.req_wdata := MuxLookup(mem_req_arb.io.out.bits.data_idx, - // Bits(0, width = MEM_DATA_BITS), - // (0 until NTILES).map( j => - // UFix(j) -> Mux(mem_req_arb.io.out.bits.is_probe_rep, - // io.tiles(j).probe_rep_data.bits.data, - // io.tiles(j).xact_init_data.bits.data))) // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 6304aa992f92a131c5880d86c0cb659c2aa9a57f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 00:44:03 -0800 Subject: [PATCH 023/688] Fixed race between read resps/reps and write req/reps in null hub --- uncore/coherence.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75c430a3..d7443d71 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -329,19 +329,19 @@ class CoherenceHubNull extends Component { val x_init = io.tile.xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready - io.mem.req_cmd.valid := x_init.valid + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := X_READ_EXCLUSIVE + x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid + x_rep.valid := io.mem.resp.valid || is_write } From ef94f13087ab0758cae3958965d746c496e705ea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 02:59:27 -0800 Subject: [PATCH 024/688] Broadcast hub nears completion. Still does not handle generation/arbitration for decoupled mem reqs. --- uncore/coherence.scala | 107 +++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 41 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index d7443d71..75796a9d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -33,9 +33,9 @@ class HubMemReq extends Bundle { val req_data = (new ioDecoupled) { new MemData() } } -class HubProbeRep extends Bundle { - val reply = (new ioDecoupled) { new ProbeReply } - val data_idx = Bits(width = log2up(NTILES)) +class TrackerProbeData extends Bundle { + val valid = Bool() + val data_tile_id = Bits(width = log2up(NTILES)) } class TrackerAllocReq extends Bundle { @@ -180,10 +180,9 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } - val probe_rep = (new ioDecoupled) { new HubProbeRep() } + val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_has_data = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } @@ -207,13 +206,21 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, + MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, + (t_type === X_READ_UNCACHED) -> P_COPY, (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) } - val s_idle :: s_mem_r :: s_mem_w :: s_mem_wr :: s_probe :: s_busy :: Nil = Enum(6){ UFix() } + def needsMemRead(t_type: UFix, global_state: UFix): Bool = { + (t_type != X_WRITE_UNCACHED) + } + + def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_WRITE_UNCACHED) + } + + val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ Bits() } val t_type_ = Reg{ Bits() } @@ -223,9 +230,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) - val p_rep_data_idx_ = Reg{ Bits() } - val x_init_data_needs_wb = Reg{ Bool() } - val p_rep_data_needs_wb = Reg{ Bool() } + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg{ Bool() } + val x_init_data_needs_write = Reg{ Bool() } + val p_rep_data_needs_write = Reg{ Bool() } io.busy := state != s_idle io.addr := addr_ @@ -235,15 +243,17 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.t_type := t_type_ io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := state === s_mem_w || state === s_mem_wr + io.mem_req.bits.req_cmd.valid := Bool(false) + io.mem_req.bits.req_cmd.bits.rw := Bool(false) io.mem_req.bits.req_cmd.bits.addr := addr_ io.mem_req.bits.req_cmd.bits.tag := UFix(id) + io.mem_req.bits.req_data.valid := Bool(false) + io.mem_req.bits.req_data.bits.data := UFix(0) // := io.mem.ready //sent mem req io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - // := io.probe_req.ready //got through arbiter ---- p_rep_dec_arr io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) @@ -258,32 +268,14 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_wb := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES) p_req_flags := ~Bits(0, width = NTILES) state := s_probe io.pop_x_init := Bool(true) } } - is(s_mem_r) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_w) { - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_busy } - } - is(s_mem_wr) { - when(io.probe_rep.bits.reply.bits.has_data) { - //io.pop_p_rep(p_rep_data_idx) := io.mem_req_rdy - //io.pop_p_rep_data(p_rep_data_idx) := io.mem_req_rdy //TODO - } . otherwise { - //io.pop_x_init := io.mem_req_rdy - //io.pop_x_init_data := io.mem_req_rdy - } - io.mem_req.valid := Bool(true) - when(io.mem_req.ready) { state := s_mem_r } - } is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags @@ -292,22 +284,55 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } - val p_rep_has_data = Bool(INPUT) - val p_rep_data_idx = Bits(log2up(NTILES), INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { - state := s_busy //TODO: XXXXXXXXXX + state := s_mem } } - when(p_rep_has_data) { - p_rep_data_needs_wb := Bool(true) - p_rep_data_idx_ := p_rep_data_idx + when(io.probe_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_rep_tile_id } } - is(s_busy) { + is(s_mem) { + when(x_init_data_needs_write) { + //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid + //io.mem_req.bits.req_cmd.valid := // TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.x_init_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_x_init_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (p_rep_data_needs_write) { + //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid + //io.mem_req.bits.req_cmd.valid := //TODO ??? + io.mem_req.bits.req_cmd.bits.rw := Bool(true) + io.mem_req.bits.req_data <> io.p_rep_data + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + //TODO + } + when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { + io.pop_p_rep_data := Bool(true) + //TODO: count with mem_count somehow + } + } . elsewhen (x_needs_read) { + io.mem_req.valid := Bool(true) + io.mem_req.bits.req_cmd.valid := Bool(true) + when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { + x_needs_read := Bool(false) + } + } . otherwise { + io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) + state := s_busy + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } From 8b519e7ea8d578f66597ccd148aa5ec1fa7ee331 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 29 Feb 2012 03:08:04 -0800 Subject: [PATCH 025/688] replace tile memory interface with ioTileLink work in progress towards coherent HTIF. for now, requests are incoherently passed through a null coherence hub. --- uncore/coherence.scala | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 75796a9d..9fbd69e9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -10,22 +10,21 @@ class MemData extends Bundle { class MemReqCmd() extends Bundle { val rw = Bool() - val addr = UFix(PADDR_BITS - OFFSET_BITS) - val tag = Bits(MEM_TAG_BITS) + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) } class MemResp () extends Bundle { - val tag = Bits(MEM_TAG_BITS) + val tag = Bits(width = MEM_TAG_BITS) val data = Bits(width = MEM_DATA_BITS) - val valid = Bool() } -class ioMemHub() extends Bundle +class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = new MemResp() + val resp = (new ioValid) { new MemResp() } } class HubMemReq extends Bundle { @@ -49,7 +48,7 @@ class TransactionInit extends Bundle { val t_type = Bits(width = TTYPE_BITS) val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS) } class TransactionInitData extends MemData @@ -348,8 +347,8 @@ abstract class CoherenceHub extends Component with CoherencePolicy class CoherenceHubNull extends Component { val io = new Bundle { - val tile = new ioTileLink() - val mem = new ioMemHub() + val tile = new ioTileLink().flip + val mem = new ioMem } val x_init = io.tile.xact_init @@ -362,11 +361,11 @@ class CoherenceHubNull extends Component { io.mem.req_data <> io.tile.xact_init_data val x_rep = io.tile.xact_rep - x_rep.bits.t_type := Mux(is_write, X_WRITE_UNCACHED, X_READ_EXCLUSIVE) - x_rep.bits.tile_xact_id := Mux(is_write, x_init.bits.tile_xact_id, io.mem.resp.tag) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.data - x_rep.valid := io.mem.resp.valid || is_write + x_rep.bits.data := io.mem.resp.bits.data + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write } @@ -388,7 +387,7 @@ class CoherenceHubNoDir extends CoherenceHub { val io = new Bundle { val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMemHub + val mem = new ioMem } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) @@ -427,12 +426,12 @@ class CoherenceHubNoDir extends CoherenceHub { // Reply to initial requestor // Forward memory responses from mem to tile - val idx = io.mem.resp.tag + val idx = io.mem.resp.bits.tag for( j <- 0 until NTILES ) { io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) io.tiles(j).xact_rep.bits.global_xact_id := idx - io.tiles(j).xact_rep.bits.data := io.mem.resp.data + io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } // If there were a ready signal due to e.g. intervening network use: From 7e6d990bad5060fbed0dead78e902ba9ce3be04f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 026/688] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 4f00bcc760ebdd1a41451857f85dc527dbaddf97 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 29 Feb 2012 17:12:02 -0800 Subject: [PATCH 027/688] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 940027a888eaf9e9409186a2939ffb3ec5fadafd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Feb 2012 17:58:15 -0800 Subject: [PATCH 028/688] Finished broadcast hub with split mem req types. Untested. --- uncore/coherence.scala | 119 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9fbd69e9..f63e633d 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -28,8 +28,7 @@ class ioMem() extends Bundle } class HubMemReq extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } + val lock = Bool() } class TrackerProbeData extends Bundle { @@ -187,7 +186,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } - val mem_req = (new ioDecoupled) { new HubMemReq() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) @@ -225,14 +226,40 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val probe_done = Reg{ Bits() } - val mem_count = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg{ Bool() } - val x_init_data_needs_write = Reg{ Bool() } - val p_rep_data_needs_write = Reg{ Bool() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { + req_cmd.valid := mem_cmd_sent + req_cmd.bits.rw := Bool(true) + req_data <> data + lock := Bool(true) + when(req_cmd.ready && req_cmd.valid) { + mem_cmd_sent := Bool(false) + } + when(req_data.ready && req_data.valid) { + pop := Bool(true) + mem_cnt := mem_cnt_next + } + when(mem_cnt === ~UFix(0)) { + trigger := Bool(false) + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready ) { + trigger := Bool(false) + } + } io.busy := state != s_idle io.addr := addr_ @@ -241,14 +268,13 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req.valid := Bool(false) - io.mem_req.bits.req_cmd.valid := Bool(false) - io.mem_req.bits.req_cmd.bits.rw := Bool(false) - io.mem_req.bits.req_cmd.bits.addr := addr_ - io.mem_req.bits.req_cmd.bits.tag := UFix(id) - io.mem_req.bits.req_data.valid := Bool(false) - io.mem_req.bits.req_data.bits.data := UFix(0) - // := io.mem.ready //sent mem req + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) @@ -269,9 +295,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES) - p_req_flags := ~Bits(0, width = NTILES) - state := s_probe + p_rep_count := UFix(NTILES-1) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) + state := Mux(p_req_flags.orR, s_probe, s_mem) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) } } @@ -285,8 +313,11 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next when(p_rep_count_next === UFix(0)) { + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) state := s_mem } } @@ -296,36 +327,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } } is(s_mem) { - when(x_init_data_needs_write) { - //io.mem_req.valid := //?TODO ??? || io.x_init_data.valid - //io.mem_req.bits.req_cmd.valid := // TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.x_init_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_x_init_data := Bool(true) - //TODO: count with mem_count somehow - } - } . elsewhen (p_rep_data_needs_write) { - //io.mem_req.valid := //TODO ??? || io.p_rep_data.valid - //io.mem_req.bits.req_cmd.valid := //TODO ??? - io.mem_req.bits.req_cmd.bits.rw := Bool(true) - io.mem_req.bits.req_data <> io.p_rep_data - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - //TODO - } - when(io.mem_req.ready && io.mem_req.bits.req_data.ready) { - io.pop_p_rep_data := Bool(true) - //TODO: count with mem_count somehow - } + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) } . elsewhen (x_needs_read) { - io.mem_req.valid := Bool(true) - io.mem_req.bits.req_cmd.valid := Bool(true) - when(io.mem_req.ready && io.mem_req.bits.req_cmd.ready) { - x_needs_read := Bool(false) - } + doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) state := s_busy @@ -369,7 +376,7 @@ class CoherenceHubNull extends Component { } -class CoherenceHubNoDir extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub { def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) @@ -440,14 +447,16 @@ class CoherenceHubNoDir extends CoherenceHub { // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - - val mem_req_arb = (new Arbiter(NGLOBAL_XACTS)) { new HubMemReq() } + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_arb.io.in(i) <> trackerList(i).io.mem_req + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - //mem_req_arb.io.out.ready := io.mem.req_cmd.ready || io.mem.req_data.ready - io.mem.req_cmd <> mem_req_arb.io.out.bits.req_cmd - io.mem.req_data <> mem_req_arb.io.out.bits.req_data + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data for( j <- 0 until NTILES ) { From 4e33b68beaa90dcc3646c3c305123286427e369b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 01:19:09 -0800 Subject: [PATCH 029/688] Unified hub ios. Fixed some hub elaboration errors. --- uncore/coherence.scala | 52 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f63e633d..0ded6f22 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -222,12 +222,12 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ Bits() } + val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = UFix(0, width = NTILES)) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -350,24 +350,25 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { // P_rep and x_init must be popped on same cycle of receipt } -abstract class CoherenceHub extends Component with CoherencePolicy - -class CoherenceHubNull extends Component { +abstract class CoherenceHub extends Component with CoherencePolicy { val io = new Bundle { - val tile = new ioTileLink().flip + val tiles = Vec(NTILES) { new ioTileLink() }.flip val mem = new ioMem } +} - val x_init = io.tile.xact_init +class CoherenceHubNull extends CoherenceHub { + + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tile.xact_init_data + io.mem.req_data <> io.tiles(0).xact_init_data - val x_rep = io.tile.xact_rep + val x_rep = io.tiles(0).xact_rep x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care @@ -392,13 +393,9 @@ class CoherenceHubBroadcast extends CoherenceHub { ret } - val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() } - val mem = new ioMem - } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) +/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -467,13 +464,12 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data := MuxLookup(trackerList(i).io.p_rep_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).probe_rep_data }) - val flags = Bits(width = NTILES) + trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - flags(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) } - p_rep_cnt_dec_arr.write(UFix(i), flags) } // Nack conflicting transaction init attempts @@ -484,8 +480,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id)) + conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? @@ -493,7 +489,7 @@ class CoherenceHubBroadcast extends CoherenceHub { x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id val want_to_abort = conflicts.orR || busy_arr.toBits.andR x_abort.valid := want_to_abort && x_init.valid - aborting(j) := want_to_abort && x_abort.ready + aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) } // Handle transaction initiation requests @@ -504,10 +500,10 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data := MuxLookup(trackerList(i).io.init_tile_id, Bits(0), (0 until NTILES).map { j => UFix(j) -> io.tiles(j).xact_init_data }) + trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data } for( j <- 0 until NTILES ) { @@ -533,15 +529,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } - for( i <- 0 until NGLOBAL_XACTS ) { - val flags = Bits(width = NTILES) - for( j <- 0 until NTILES ) { - flags(j) := p_req_arb_arr(j).io.in(i).ready - } - p_rep_cnt_dec_arr.write(UFix(i), flags) - } - +*/ } From 2152b0283d0a3438b644210166d2122ba64c2580 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 17:03:56 -0800 Subject: [PATCH 030/688] Made xact_rep an ioValid, removed has_data member --- uncore/coherence.scala | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ded6f22..12c1e4a9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -14,10 +14,9 @@ class MemReqCmd() extends Bundle val tag = Bits(width = MEM_TAG_BITS) } -class MemResp () extends Bundle +class MemResp () extends MemData { val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) } class ioMem() extends Bundle @@ -72,7 +71,6 @@ class ProbeReplyData extends MemData class TransactionReply extends MemData { val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -88,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() } val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioDecoupled) { new TransactionReply() } + val xact_rep = (new ioValid) { new TransactionReply() } val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } @@ -173,6 +171,10 @@ trait FourStateCoherence extends CoherencePolicy { } state.toBits } + + def replyTypeHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_WRITE_UNCACHED) + } } class XactTracker(id: Int) extends Component with CoherencePolicy { From de73eef4091d41ed41ed824fe65ab19927dfd623 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 Mar 2012 18:23:46 -0800 Subject: [PATCH 031/688] Fixed elaboration errors in LockingArbiter and BoradcastHub. Fixed ioDecoupled direction error in XactTracker --- uncore/coherence.scala | 130 ++++++++++++++++++++++++----------------- 1 file changed, 76 insertions(+), 54 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 12c1e4a9..5f70d81f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -163,13 +163,12 @@ trait FourStateCoherence extends CoherencePolicy { def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - def getMetaUpdateOnProbe (incoming: ProbeRequest): Bits = { - val state = UFix(0) - switch(incoming.p_type) { - is(probeInvalidate) { state := tileInvalid } - is(probeDowngrade) { state := tileShared } - } - state.toBits + def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state + )) } def replyTypeHasData (reply: TransactionReply): Bool = { @@ -187,9 +186,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_req_cnt_inc = Bits(NTILES, INPUT) val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip + val mem_req_data = (new ioDecoupled) { new MemData() }.flip val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val busy = Bool(OUTPUT) @@ -222,7 +222,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { (t_type === X_WRITE_UNCACHED) } - val s_idle :: s_mem :: s_probe :: s_busy :: Nil = Enum(4){ UFix() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } val t_type_ = Reg{ Bits() } @@ -241,7 +241,10 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { req_cmd.valid := mem_cmd_sent req_cmd.bits.rw := Bool(true) - req_data <> data + //TODO: why does req_data <> data segfault? + req_data.valid := data.valid + req_data.bits.data := data.bits.data + data.ready := req_data.ready lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { mem_cmd_sent := Bool(false) @@ -270,7 +273,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -279,7 +282,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) @@ -287,6 +290,8 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) io.send_x_rep_ack := Bool(false) + io.x_init_data.ready := Bool(false) // don't care + io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -336,20 +341,19 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - io.send_x_rep_ack := needsAckRep(t_type_, UFix(0)) - state := s_busy + state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) } } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.xact_finish) { state := s_idle } } } - - //TODO: Decrement the probe count when final data piece is written - // Connent io.mem.ready sig to correct pop* outputs - // P_rep and x_init must be popped on same cycle of receipt } abstract class CoherenceHub extends Component with CoherencePolicy { @@ -385,19 +389,16 @@ class CoherenceHubBroadcast extends CoherenceHub { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - val ret = Wire() { Bits(width = TTYPE_BITS) } - switch (t_type) { - is(X_READ_SHARED) { ret := Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE) } - is(X_READ_EXCLUSIVE) { ret := X_READ_EXCLUSIVE } - is(X_READ_UNCACHED) { ret := X_READ_UNCACHED } - is(X_WRITE_UNCACHED) { ret := X_WRITE_UNCACHED } - } - ret + MuxLookup(t_type, X_READ_UNCACHED, Array( + X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), + X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, + X_READ_UNCACHED -> X_READ_UNCACHED, + X_WRITE_UNCACHED -> X_WRITE_UNCACHED + )) } val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) -/* val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } @@ -409,40 +410,61 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { - busy_arr.write( UFix(i), trackerList(i).io.busy) - addr_arr.write( UFix(i), trackerList(i).io.addr) - init_tile_id_arr.write( UFix(i), trackerList(i).io.init_tile_id) - tile_xact_id_arr.write( UFix(i), trackerList(i).io.tile_xact_id) - t_type_arr.write( UFix(i), trackerList(i).io.t_type) - sh_count_arr.write( UFix(i), trackerList(i).io.sharer_count) - send_x_rep_ack_arr.write(UFix(i), trackerList(i).io.send_x_rep_ack) - trackerList(i).io.xact_finish := do_free_arr.read(UFix(i)) - trackerList(i).io.p_rep_cnt_dec := p_rep_cnt_dec_arr.read(UFix(i)) - trackerList(i).io.p_req_cnt_inc := p_req_cnt_inc_arr.read(UFix(i)) + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + do_free_arr(i) := Bool(false) + p_rep_cnt_dec_arr(i) := Bits(0) + p_req_cnt_inc_arr(i) := Bits(0) + sent_x_rep_ack_arr(i) := Bool(false) + t.xact_finish := do_free_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) + t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) } // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish - do_free_arr.write(finish.bits.global_xact_id, finish.valid) + do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) } // Reply to initial requestor - // Forward memory responses from mem to tile - val idx = io.mem.resp.bits.tag + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) for( j <- 0 until NTILES ) { - io.tiles(j).xact_rep.bits.t_type := getTransactionReplyType(t_type_arr.read(idx), sh_count_arr.read(idx)) - io.tiles(j).xact_rep.bits.tile_xact_id := tile_xact_id_arr.read(idx) - io.tiles(j).xact_rep.bits.global_xact_id := idx + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.valid := Bool(false) + when(io.mem.resp.valid) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + } io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data - io.tiles(j).xact_rep.valid := (UFix(j) === init_tile_id_arr.read(idx)) && (io.mem.resp.valid || send_x_rep_ack_arr.read(idx)) } + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(idx)).xact_rep.ready - + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data @@ -466,7 +488,8 @@ class CoherenceHubBroadcast extends CoherenceHub { p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data <> io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -475,14 +498,14 @@ class CoherenceHubBroadcast extends CoherenceHub { } // Nack conflicting transaction init attempts - val aborting = Wire() { Bits(width = NTILES) } + val aborting = Bits(0, width = NTILES) for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy(i) && coherenceConflict(t.addr, x_init.bits.address) && + conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to @@ -505,9 +528,9 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data <> io.tiles(trackerList(i).io.init_tile_id).xact_init_data + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid } - for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data @@ -522,7 +545,6 @@ class CoherenceHubBroadcast extends CoherenceHub { alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) - // Handle probe request generation // Must arbitrate for each request port val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) @@ -535,5 +557,5 @@ class CoherenceHubBroadcast extends CoherenceHub { } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } -*/ + } From 4c3f7a36cedb2f49d657ea1bb151694f66f6b339 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 1 Mar 2012 18:49:00 -0800 Subject: [PATCH 032/688] clean up D$ store data unit --- uncore/coherence.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5f70d81f..2c5c6edf 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -90,8 +90,8 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip } -trait CoherencePolicy { - def cpuCmdToRW( cmd: Bits): (Bool, Bool) = { +object cpuCmdToRW { + def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool @@ -101,6 +101,9 @@ trait CoherencePolicy { } } +trait CoherencePolicy { +} + trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } From bef59527d6b155b8e888e1d1d57d9dd171903917 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 1 Mar 2012 20:48:46 -0800 Subject: [PATCH 033/688] clean up ioDecoupled/ioPipe interface --- uncore/coherence.scala | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2c5c6edf..97d098bd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val req_data = (new ioDecoupled) { new MemData() }.flip - val resp = (new ioValid) { new MemResp() } + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() } } class HubMemReq extends Bundle { @@ -80,14 +80,14 @@ class TransactionFinish extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() }.flip - val xact_init_data = (new ioDecoupled) { new TransactionInitData() }.flip - val xact_abort = (new ioDecoupled) { new TransactionAbort() } - val probe_req = (new ioDecoupled) { new ProbeRequest() } - val probe_rep = (new ioDecoupled) { new ProbeReply() }.flip - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val xact_rep = (new ioValid) { new TransactionReply() } - val xact_finish = (new ioDecoupled) { new TransactionFinish() }.flip + val xact_init = (new ioDecoupled) { new TransactionInit() } + val xact_init_data = (new ioDecoupled) { new TransactionInitData() } + val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply() } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } + val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_finish = (new ioDecoupled) { new TransactionFinish() } } object cpuCmdToRW { @@ -181,20 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip val probe_data = (new TrackerProbeData).asInput val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val x_init_data = (new ioDecoupled) { new TransactionInitData() } + val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() }.flip - val mem_req_data = (new ioDecoupled) { new MemData() }.flip + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } + val mem_req_data = (new ioDecoupled) { new MemData() } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest() } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 9ad4fd5814cd1578a7aa6de372e77ec5a9e08416 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 12:19:21 -0800 Subject: [PATCH 034/688] BroadcastHub can be elaborated by C and vlsi backends --- uncore/coherence.scala | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 97d098bd..536a4775 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -198,9 +198,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(log2up(NTILES), INPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(TTYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) @@ -411,8 +411,8 @@ class CoherenceHubBroadcast extends CoherenceHub { val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } - val p_req_cnt_inc_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=NTILES)} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } for( i <- 0 until NGLOBAL_XACTS) { @@ -424,14 +424,16 @@ class CoherenceHubBroadcast extends CoherenceHub { t_type_arr(i) := t.t_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack - do_free_arr(i) := Bool(false) - p_rep_cnt_dec_arr(i) := Bits(0) - p_req_cnt_inc_arr(i) := Bits(0) - sent_x_rep_ack_arr(i) := Bool(false) t.xact_finish := do_free_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i) - t.p_req_cnt_inc := p_req_cnt_inc_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + for( j <- 0 until NTILES) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } } // Free finished transactions @@ -495,8 +497,7 @@ class CoherenceHubBroadcast extends CoherenceHub { trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep - val dec = p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - p_rep_cnt_dec_arr(UFix(i)) := p_rep_cnt_dec_arr(UFix(i)).bitSet(UFix(j), dec) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } } @@ -556,7 +557,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_rep_cnt_dec_arr(i) = p_rep_cnt_dec_arr(i).bitSet(UFix(j), p_req_arb_arr(j).io.in(i).ready) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } p_req_arb_arr(j).io.out <> io.tiles(j).probe_req } From 6ad7aa5cdc586af0ec6a9744f0d6d7cc4cafa1f0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 2 Mar 2012 16:18:32 -0800 Subject: [PATCH 035/688] flip direction of ioPipe to match ioDecoupled --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 536a4775..195234a4 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -23,7 +23,7 @@ class ioMem() extends Bundle { val req_cmd = (new ioDecoupled) { new MemReqCmd() } val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() } + val resp = (new ioPipe) { new MemResp() }.flip } class HubMemReq extends Bundle { @@ -86,7 +86,7 @@ class ioTileLink extends Bundle { val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip val probe_rep = (new ioDecoupled) { new ProbeReply() } val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() } + val xact_rep = (new ioPipe) { new TransactionReply() }.flip val xact_finish = (new ioDecoupled) { new TransactionFinish() } } From 993341d27efcc486234716a1e1c45fedc3cf10fb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 13:41:01 -0800 Subject: [PATCH 036/688] Correction to probe reply w/ data handling --- uncore/coherence.scala | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 195234a4..638cc0d7 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -31,8 +31,7 @@ class HubMemReq extends Bundle { } class TrackerProbeData extends Bundle { - val valid = Bool() - val data_tile_id = Bits(width = log2up(NTILES)) + val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { @@ -182,7 +181,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val probe_data = (new TrackerProbeData).asInput + val p_data = (new ioPipe) { new TrackerProbeData() } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) @@ -331,9 +330,9 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { state := s_mem } } - when(io.probe_data.valid) { + when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_rep_tile_id + p_rep_tile_id_ := io.p_data.bits.tile_id } } is(s_mem) { @@ -413,7 +412,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -425,14 +426,18 @@ class CoherenceHubBroadcast extends CoherenceHub { sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) for( j <- 0 until NTILES) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) } } @@ -490,7 +495,9 @@ class CoherenceHubBroadcast extends CoherenceHub { val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid From d76f6640070bd34819dc62ddbe8c131ac933ebd0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 21:58:50 -0800 Subject: [PATCH 037/688] Filled out 4 state coherence functions for cache --- uncore/coherence.scala | 44 ++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 638cc0d7..4532794b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -156,16 +156,26 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.t_type, tileInvalid, Array( + X_READ_SHARED -> tileShared, + X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_READ_UNCACHED -> tileInvalid, + X_WRITE_UNCACHED -> tileInvalid + )) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + } - // TODO: New funcs as compared to incoherent protocol: - def newState(cmd: Bits, state: UFix): UFix - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnPrimaryMiss(cmd: Bits): UFix - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix - - def needsSecondaryXact (cmd: Bits, outstanding: TransactionInit): Bool - - def newStateOnProbe (incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -174,26 +184,26 @@ trait FourStateCoherence extends CoherencePolicy { } def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED) + (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) } } class XactTracker(id: Int) extends Component with CoherencePolicy { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq() }.flip - val p_data = (new ioPipe) { new TrackerProbeData() } + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData() }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData() }.flip + val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip + val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd() } - val mem_req_data = (new ioDecoupled) { new MemData() } + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest() } + val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) From 7846d5e01d8a1cf86895e56589b98946572cccf9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Mar 2012 23:51:53 -0800 Subject: [PATCH 038/688] Removed has_data fields from all coherence messages, increased message type names to compensate --- uncore/coherence.scala | 78 +++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 4532794b..0ce1f1c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,8 +42,7 @@ class TrackerAllocReq extends Bundle { class TransactionInit extends Bundle { - val t_type = Bits(width = TTYPE_BITS) - val has_data = Bool() + val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS) } @@ -55,21 +54,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = PTYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = PTYPE_BITS) - val has_data = Bool() + val p_type = Bits(width = P_REP_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = TTYPE_BITS) + val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } @@ -162,17 +160,17 @@ trait FourStateCoherence extends CoherencePolicy { } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( - X_READ_SHARED -> tileShared, - X_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_READ_UNCACHED -> tileInvalid, - X_WRITE_UNCACHED -> tileInvalid + X_REP_READ_SHARED -> tileShared, + X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), + X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, + X_REP_READ_UNCACHED -> tileInvalid, + X_REP_WRITE_UNCACHED -> tileInvalid )) } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_READ_UNCACHED || outstanding.t_type === X_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_READ_EXCLUSIVE)) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { @@ -183,12 +181,20 @@ trait FourStateCoherence extends CoherencePolicy { )) } - def replyTypeHasData (reply: TransactionReply): Bool = { - (reply.t_type != X_WRITE_UNCACHED && reply.t_type != X_READ_EXCLUSIVE_ACK) + def probeReplyHasData (reply: ProbeReply): Bool = { + (reply.p_type === P_REP_INVALIDATE_DATA || + reply.p_type === P_REP_DOWNGRADE_DATA || + reply.p_type === P_REP_COPY_DATA) + } + def transactionInitHasData (init: TransactionInit): Bool = { + (init.t_type != X_INIT_WRITE_UNCACHED) + } + def transactionReplyHasData (reply: TransactionReply): Bool = { + (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } } -class XactTracker(id: Int) extends Component with CoherencePolicy { +class XactTracker(id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } @@ -210,7 +216,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(TTYPE_BITS, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) @@ -220,18 +226,20 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { } def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { - MuxCase(P_COPY, Array((t_type === X_READ_SHARED) -> P_DOWNGRADE, - (t_type === X_READ_EXCLUSIVE) -> P_INVALIDATE, - (t_type === X_READ_UNCACHED) -> P_COPY, - (t_type === X_WRITE_UNCACHED) -> P_INVALIDATE)) + MuxLookup(t_type, P_REQ_COPY, Array( + X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, + X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, + X_INIT_READ_UNCACHED -> P_REQ_COPY, + X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + )) } def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_WRITE_UNCACHED) + (t_type != X_INIT_WRITE_UNCACHED) } def needsAckRep(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_WRITE_UNCACHED) + (t_type === X_INIT_WRITE_UNCACHED) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } @@ -312,7 +320,7 @@ class XactTracker(id: Int) extends Component with CoherencePolicy { t_type_ := io.alloc_req.bits.xact_init.t_type init_tile_id_ := io.alloc_req.bits.init_tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := io.alloc_req.bits.xact_init.has_data + x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) @@ -378,7 +386,7 @@ abstract class CoherenceHub extends Component with CoherencePolicy { class CoherenceHubNull extends CoherenceHub { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_WRITE_UNCACHED + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -387,7 +395,7 @@ class CoherenceHubNull extends CoherenceHub { io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_READ_EXCLUSIVE, X_WRITE_UNCACHED) + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -395,17 +403,17 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub { +class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) } def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_READ_UNCACHED, Array( - X_READ_SHARED -> Mux(count > UFix(0), X_READ_SHARED, X_READ_EXCLUSIVE), - X_READ_EXCLUSIVE -> X_READ_EXCLUSIVE, - X_READ_UNCACHED -> X_READ_UNCACHED, - X_WRITE_UNCACHED -> X_WRITE_UNCACHED + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED )) } @@ -415,7 +423,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TTYPE_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -506,7 +514,7 @@ class CoherenceHubBroadcast extends CoherenceHub { val idx = p_rep.bits.global_xact_id p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && p_rep.bits.has_data + p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) } for( i <- 0 until NGLOBAL_XACTS ) { @@ -527,7 +535,7 @@ class CoherenceHubBroadcast extends CoherenceHub { for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(x_init.bits.has_data && (UFix(j) === t.init_tile_id))) + !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) // Don't abort writebacks stalled on mem. // TODO: This assumes overlapped writeback init reqs to // the same addr will never be issued; is this ok? From 802f857cb69bb00086e841ddaddfca135669a8bb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 3 Mar 2012 15:07:22 -0800 Subject: [PATCH 039/688] fix store prefetch bug, it no longer occupies an entry in the sdq --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 0ce1f1c9..005f17c9 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -92,8 +92,8 @@ object cpuCmdToRW { val store = (cmd === M_XWR) val load = (cmd === M_XRD) val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) - val write = store || amo || (cmd === M_PFW) + val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) + val write = store || amo (read, write) } } From a49625c114b5f108451a7c8e64c68cbb36569c34 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 16:34:27 -0800 Subject: [PATCH 040/688] Broadcast hub control logic bugfixes and code cleanup --- uncore/coherence.scala | 81 +++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 44 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 005f17c9..c6e702cd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -26,18 +26,13 @@ class ioMem() extends Bundle val resp = (new ioPipe) { new MemResp() }.flip } -class HubMemReq extends Bundle { - val lock = Bool() -} - class TrackerProbeData extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } class TrackerAllocReq extends Bundle { val xact_init = new TransactionInit() - val init_tile_id = Bits(width = TILE_ID_BITS) - val data_valid = Bool() + val tile_id = Bits(width = TILE_ID_BITS) } @@ -98,8 +93,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { -} +trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } @@ -242,36 +236,18 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop: Bool) { - req_cmd.valid := mem_cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { + req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) - //TODO: why does req_data <> data segfault? - req_data.valid := data.valid - req_data.bits.data := data.bits.data data.ready := req_data.ready + req_data <> data lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { - mem_cmd_sent := Bool(false) + cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop := Bool(true) - mem_cnt := mem_cnt_next + pop_data := Bool(true) + mem_cnt := mem_cnt_next } when(mem_cnt === ~UFix(0)) { trigger := Bool(false) @@ -286,6 +262,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ @@ -318,16 +313,16 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.init_tile_id + init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.init_tile_id ) - state := Mux(p_req_flags.orR, s_probe, s_mem) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) + state := Mux(p_req_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -342,9 +337,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec p_rep_count := p_rep_count_next - when(p_rep_count_next === UFix(0)) { - mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + when(p_rep_count === UFix(0)) { + io.pop_p_rep := Bool(true) state := s_mem } } @@ -355,9 +349,9 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data) + doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -565,10 +559,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data init_arb.io.in(j).valid := x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.init_tile_id := UFix(j) - init_arb.io.in(j).bits.data_valid := x_init_data.valid - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.init_tile_id === UFix(j)))(_||_) + init_arb.io.in(j).bits.tile_id := UFix(j) + x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && From 82822dfeecd8a5ed3fe9090c3fad56d52ea92dc4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Mar 2012 17:44:30 -0800 Subject: [PATCH 041/688] Added aborted data dequeueing state machine for BroadcastHub --- uncore/coherence.scala | 58 +++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index c6e702cd..fc4f3570 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -240,7 +240,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { req_cmd.valid := !cmd_sent req_cmd.bits.rw := Bool(true) data.ready := req_data.ready - req_data <> data + req_data.bits := data.bits + req_data.valid := data.valid lock := Bool(true) when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) @@ -521,23 +522,51 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } // Nack conflicting transaction init attempts - val aborting = Bits(0, width = NTILES) + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && coherenceConflict(t.addr, x_init.bits.address) && - !(transactionInitHasData(x_init.bits) && (UFix(j) === t.init_tile_id))) - // Don't abort writebacks stalled on mem. - // TODO: This assumes overlapped writeback init reqs to - // the same addr will never be issued; is this ok? + conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val want_to_abort = conflicts.orR || busy_arr.toBits.andR - x_abort.valid := want_to_abort && x_init.valid - aborting.bitSet(UFix(j), want_to_abort && x_abort.ready) + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(transactionInitHasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + } + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } } // Handle transaction initiation requests @@ -557,15 +586,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data - init_arb.io.in(j).valid := x_init.valid + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data.ready := aborting(j) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) } - alloc_arb.io.out.ready := init_arb.io.out.valid && !busy_arr.toBits.andR && - !foldR(trackerList.map(t => t.io.busy && coherenceConflict(t.io.addr, init_arb.io.out.bits.xact_init.address)))(_||_) + alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port From 1024a460f238ff6df738c0d7e119f1ba9d0234be Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 00:31:44 -0800 Subject: [PATCH 042/688] support memory transaction aborts --- uncore/coherence.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc4f3570..7bcd19c3 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -395,6 +395,8 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + + io.tiles(0).xact_abort.valid := Bool(false) } From 012b991ffead7aa724adf6565f6ced61e7c9f034 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 6 Mar 2012 15:47:19 -0800 Subject: [PATCH 043/688] implement transaction finish messages --- uncore/coherence.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7bcd19c3..817e6d73 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -397,6 +397,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) } From 3874da4a4ffa304688a0b708322160f48e8b8fd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 13:49:20 -0800 Subject: [PATCH 044/688] Added store dependency queues to BroadcastHub. Minor improvements to utils. --- uncore/coherence.scala | 80 ++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 817e6d73..51505882 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -35,6 +35,9 @@ class TrackerAllocReq extends Bundle { val tile_id = Bits(width = TILE_ID_BITS) } +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) @@ -196,9 +199,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(NTILES, INPUT) val p_req_cnt_inc = Bits(NTILES, INPUT) - val p_rep_data = (new ioDecoupled) { new ProbeReplyData }.flip - val x_init_data = (new ioDecoupled) { new TransactionInitData }.flip + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } val mem_req_data = (new ioDecoupled) { new MemData } @@ -214,8 +219,10 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val push_p_req = Bits(NTILES, OUTPUT) val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) + val pop_p_rep_dep = Bits(NTILES, OUTPUT) val pop_x_init = Bool(OUTPUT) val pop_x_init_data = Bool(OUTPUT) + val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -236,13 +243,12 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioDecoupled[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool) { - req_cmd.valid := !cmd_sent + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - data.ready := req_data.ready + req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - req_data.valid := data.valid - lock := Bool(true) + lock := at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -250,7 +256,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { pop_data := Bool(true) mem_cnt := mem_cnt_next } - when(mem_cnt === ~UFix(0)) { + when(mem_cnt_next === UFix(0)) { + pop_dep := Bool(true) trigger := Bool(false) } } @@ -277,7 +284,6 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) @@ -285,6 +291,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(NTILES) // TODO: Broadcast only io.t_type := t_type_ @@ -303,11 +310,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.push_p_req := Bits(0, width = NTILES) io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) + io.pop_p_rep_dep := Bits(0, width = NTILES) io.pop_x_init := Bool(false) io.pop_x_init_data := Bool(false) + io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) - io.x_init_data.ready := Bool(false) // don't care - io.p_rep_data.ready := Bool(false) // don't care switch (state) { is(s_idle) { @@ -321,7 +328,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) mem_cnt := UFix(0) - mem_cmd_sent := Bool(false) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) io.pop_x_init := Bool(true) state := Mux(p_req_flags.orR, s_probe, s_mem) } @@ -350,9 +358,25 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } is(s_mem) { when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, io.pop_p_rep_data, p_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + io.pop_p_rep_data, + p_w_mem_cmd_sent, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, io.pop_x_init_data, x_w_mem_cmd_sent) + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + io.pop_x_init_data, + x_w_mem_cmd_sent, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -457,6 +481,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ } } + val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + // Free finished transactions for( j <- 0 until NTILES ) { val finish = io.tiles(j).xact_finish @@ -510,14 +537,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id - p_rep.ready := foldR(trackerList.map(_.io.pop_p_rep(j)))(_ || _) + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := do_pop + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) p_data_tile_id_arr(idx) := UFix(j) + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + for( j <- 0 until NTILES) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) @@ -532,14 +568,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) val conflicts = Bits(width = NGLOBAL_XACTS) for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR + want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { @@ -585,6 +621,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) + trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init @@ -592,8 +631,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || foldR(trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) + x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 762f2551a70ba40ea8f527522d48551ecc8994a8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 15:54:19 -0800 Subject: [PATCH 045/688] newTransactionOnMiss() --- uncore/coherence.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 51505882..5a52ac44 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -155,6 +155,10 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, From 1e02618cc800bea9c2de5231db628f38ab5ddb62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:01:47 -0800 Subject: [PATCH 046/688] hub code cleanup --- uncore/coherence.scala | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 5a52ac44..a04b7096 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -141,6 +141,9 @@ trait FourStateCoherence extends CoherencePolicy { (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) } + //TODO: do we need isPresent() for determining that a line needs to be + //upgraded but that no replacement is needed? + def isValid (state: UFix): Bool = { state != tileInvalid } @@ -224,8 +227,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val pop_p_rep = Bits(NTILES, OUTPUT) val pop_p_rep_data = Bits(NTILES, OUTPUT) val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bool(OUTPUT) - val pop_x_init_data = Bool(OUTPUT) + val pop_x_init = Bits(NTILES, OUTPUT) + val pop_x_init_data = Bits(NTILES, OUTPUT) val pop_x_init_dep = Bits(NTILES, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -247,7 +250,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { (t_type === X_INIT_WRITE_UNCACHED) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, pop_data: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -257,11 +260,11 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { cmd_sent := Bool(true) } when(req_data.ready && req_data.valid) { - pop_data := Bool(true) + pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next } when(mem_cnt_next === UFix(0)) { - pop_dep := Bool(true) + pop_dep := UFix(1) << tile_id trigger := Bool(false) } } @@ -315,8 +318,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.pop_p_rep := Bits(0, width = NTILES) io.pop_p_rep_data := Bits(0, width = NTILES) io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bool(false) - io.pop_x_init_data := Bool(false) + io.pop_x_init := Bits(0, width = NTILES) + io.pop_x_init_data := Bits(0, width = NTILES) io.pop_x_init_dep := Bits(0, width = NTILES) io.send_x_rep_ack := Bool(false) @@ -334,7 +337,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := Bool(true) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id state := Mux(p_req_flags.orR, s_probe, s_mem) } } @@ -367,20 +370,22 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.mem_req_lock, io.p_rep_data, p_rep_data_needs_write, - io.pop_p_rep_data, p_w_mem_cmd_sent, + io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, io.x_init_data, x_init_data_needs_write, - io.pop_x_init_data, x_w_mem_cmd_sent, + io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -625,23 +630,23 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - //TODO trackerList(i).io.x_init_data_dep <> x_init_data_dep_arr(trackerList(i).io.init_tile_id) - trackerList(i).io.x_init_data_dep.bits <> MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until NTILES ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init && init_arb.io.out.bits.tile_id === UFix(j)) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data && init_arb.io.out.bits.tile_id === UFix(j)))(_||_) - x_init_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid From 35ffb80911ec380695fe1629ff695672f00cb33c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Mar 2012 17:33:11 -0800 Subject: [PATCH 047/688] unified coherence trait functions --- uncore/coherence.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a04b7096..bda67671 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -126,7 +126,15 @@ trait ThreeStateIncoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, state) } - + def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileDirty, tileClean) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -162,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), From 792960087434d227f164cb63a65579ecfb875a80 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 01:26:35 -0800 Subject: [PATCH 048/688] change D$ to use FourStateCoherence protocol instead of ThreeStateIncoherence. --- uncore/coherence.scala | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index bda67671..2b34b8ae 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -121,17 +121,18 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnPrimaryMiss(cmd: Bits): UFix = newState(cmd, tileInvalid) - def newStateOnSecondaryMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, state) + Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = X_INIT_READ_EXCLUSIVE - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, tileClean) - } + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) @@ -166,11 +167,20 @@ trait FourStateCoherence extends CoherencePolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileExclusiveDirty, state) } - def newTransactionOnMiss(cmd: Bits, state: UFix): UFix = { + def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newStateOnTransactionRep(cmd: Bits, incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || + (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + } + def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -179,11 +189,6 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) - } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( From 1a90e2a1624acbf58ed127c042ffe4847f83792e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 Mar 2012 11:40:49 -0800 Subject: [PATCH 049/688] Broadcast hub bug fixes for load uncached mem req and store uncached xact rep --- uncore/coherence.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 2b34b8ae..843ff2be 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,7 +135,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type != X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } trait FourStateCoherence extends CoherencePolicy { @@ -204,7 +204,7 @@ trait FourStateCoherence extends CoherencePolicy { reply.p_type === P_REP_COPY_DATA) } def transactionInitHasData (init: TransactionInit): Bool = { - (init.t_type != X_INIT_WRITE_UNCACHED) + (init.t_type === X_INIT_WRITE_UNCACHED) } def transactionReplyHasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) p_rep_count := UFix(NTILES-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) + p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) @@ -516,12 +516,14 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) + //val ack_idx_ = Reg(ack_idx) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -534,7 +536,6 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.global_xact_id := ack_idx rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) } - io.tiles(j).xact_rep.bits.data := io.mem.resp.bits.data } sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) // If there were a ready signal due to e.g. intervening network use: From df6ac34821800d07c9b953b19e5af09a7bc3cd68 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 7 Mar 2012 21:03:44 -0800 Subject: [PATCH 050/688] coherence hub fixes --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 843ff2be..042ee666 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -516,8 +516,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = UFix(0)//PriorityEncoder(send_x_rep_ack_arr.toBits, NGLOBAL_XACTS) - //val ack_idx_ = Reg(ack_idx) + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until NTILES ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) @@ -534,10 +533,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr(ack_idx) + rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid && send_x_rep_ack_arr(ack_idx) + sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready @@ -592,13 +591,13 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Bits(width = NGLOBAL_XACTS) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(UFix(i), t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address)) + conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 3fc94d627d8f6426433418dccb636d245e1dc204 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 11:36:10 -0800 Subject: [PATCH 051/688] Fixed dependency queue bug in Broadcast Hub --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 042ee666..14504062 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -638,7 +638,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits <> init_arb.io.out.bits + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits @@ -655,7 +655,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop + x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From ca5caf898c6a26b5074685a4abfdde65d125c8b5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 16:39:05 -0800 Subject: [PATCH 052/688] Hub addr comparison bug fix --- uncore/coherence.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 14504062..e3827c72 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -42,7 +42,7 @@ class TrackerDependency extends Bundle { class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS) + val address = UFix(width = PADDR_BITS - OFFSET_BITS) } class TransactionInitData extends MemData @@ -54,7 +54,7 @@ class TransactionAbort extends Bundle { class ProbeRequest extends Bundle { val p_type = Bits(width = P_REQ_TYPE_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS) + val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -230,7 +230,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val mem_req_lock = Bool(OUTPUT) val probe_req = (new ioDecoupled) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS, OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) @@ -449,9 +449,8 @@ class CoherenceHubNull extends CoherenceHub { class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = { - addr1(PADDR_BITS-1, OFFSET_BITS) === addr2(PADDR_BITS-1, OFFSET_BITS) - } + def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { MuxLookup(t_type, X_REP_READ_UNCACHED, Array( X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), @@ -464,7 +463,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS)} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } From 2df5c19f13adf1ae67a9955122c3d6de1f4b51d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 8 Mar 2012 18:07:44 -0800 Subject: [PATCH 053/688] Added require_ack field to TransactionReply bundle --- uncore/coherence.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e3827c72..a2c6ba5f 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -68,6 +68,7 @@ class TransactionReply extends MemData { val t_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() } class TransactionFinish extends Bundle { @@ -440,6 +441,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write io.tiles(0).xact_abort.valid := Bool(false) @@ -522,6 +524,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) From 2607153b6785875ead46253bc513f86578c0fa53 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 9 Mar 2012 02:08:55 -0800 Subject: [PATCH 054/688] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 2014db41bd0e826ee5469b8149c6284896a737d2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 9 Mar 2012 11:04:58 -0800 Subject: [PATCH 055/688] Special cased NTILES == 1 due to log2up revision --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index a2c6ba5f..275e427b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -297,7 +297,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = Reg(resetVal = UFix(0, width = log2up(NTILES))) + val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -346,7 +346,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - p_rep_count := UFix(NTILES-1) + if(NTILES > 1) p_rep_count := UFix(NTILES-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -366,7 +366,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - p_rep_count := p_rep_count_next + if(NTILES > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem From d777f1cb24cb951ece5962f79677552a7e899758 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 10 Mar 2012 15:18:57 -0800 Subject: [PATCH 056/688] fix null hub store ack bug --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 275e427b..41959915 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -124,7 +124,7 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -170,7 +170,7 @@ trait FourStateCoherence extends CoherencePolicy { } def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -442,7 +442,7 @@ class CoherenceHubNull extends CoherenceHub { x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) From cb5ce3fe7310c6ae7686665f03bc40a8a8958932 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 14:17:27 -0700 Subject: [PATCH 057/688] More broadcast hub bugfixes --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 41959915..f128036a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -276,17 +276,17 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(req_data.ready && req_data.valid) { pop_data := UFix(1) << tile_id mem_cnt := mem_cnt_next - } - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) - when(req_cmd.ready ) { + when(req_cmd.ready) { trigger := Bool(false) } } @@ -535,10 +535,11 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - rep.valid := (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR + rep.valid := do_send_ack + sent_x_rep_ack_arr(ack_idx) := do_send_ack } } - sent_x_rep_ack_arr(ack_idx) := !io.mem.resp.valid // If there were a ready signal due to e.g. intervening network use: //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready From 95f880da704542ea7d346cbb275610ac8cc30ff7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 11 Mar 2012 17:13:01 -0700 Subject: [PATCH 058/688] fixed abort bug --- uncore/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index f128036a..3db3ac46 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -600,7 +600,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits)) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { From 23c822a82e854a70ebc7bd1303e89bd532604600 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Mar 2012 10:38:37 -0700 Subject: [PATCH 059/688] fix hit logic for amos --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 3db3ac46..653ab6dc 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -147,8 +147,8 @@ trait FourStateCoherence extends CoherencePolicy { def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) - ((read && ( state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) || - (write && (state === tileExclusiveClean || state === tileExclusiveDirty))) + Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } //TODO: do we need isPresent() for determining that a line needs to be From 1b733e7cf07c9e8abd95358198bf06d7ade92f48 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 13 Mar 2012 12:34:39 -0700 Subject: [PATCH 060/688] Merge branch 'master' of github.com:ucb-bar/riscv-rocket From 1258f31825c7a3e0787784de4fbfe0c86704dfdf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 16:43:35 -0700 Subject: [PATCH 061/688] add probe unit --- uncore/coherence.scala | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 653ab6dc..de158417 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -135,6 +135,12 @@ trait ThreeStateIncoherence extends CoherencePolicy { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := P_REP_INVALIDATE_ACK + reply.global_xact_id := UFix(0) + reply + } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) } @@ -199,6 +205,22 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_DATA, + probeDowngrade -> P_REP_DOWNGRADE_DATA, + probeCopy -> P_REP_COPY_DATA + )) + val without_data = MuxLookup(incoming.p_type, state, Array( + probeInvalidate -> P_REP_INVALIDATE_ACK, + probeDowngrade -> P_REP_DOWNGRADE_ACK, + probeCopy -> P_REP_COPY_ACK + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } def probeReplyHasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || @@ -446,6 +468,9 @@ class CoherenceHubNull extends CoherenceHub { io.tiles(0).xact_abort.valid := Bool(false) io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) } From 53d69d3006b5e52b1b881fcd135eaa8240e734af Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 17:12:01 -0700 Subject: [PATCH 062/688] parameterize broadcast hub by # of tiles --- uncore/coherence.scala | 94 +++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index de158417..ae9321b1 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -234,14 +234,14 @@ trait FourStateCoherence extends CoherencePolicy { } } -class XactTracker(id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData } val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(NTILES, INPUT) - val p_req_cnt_inc = Bits(NTILES, INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip val x_init_data = (new ioPipe) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -259,13 +259,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(NTILES, OUTPUT) - val pop_p_rep = Bits(NTILES, OUTPUT) - val pop_p_rep_data = Bits(NTILES, OUTPUT) - val pop_p_rep_dep = Bits(NTILES, OUTPUT) - val pop_x_init = Bits(NTILES, OUTPUT) - val pop_x_init_data = Bits(NTILES, OUTPUT) - val pop_x_init_dep = Bits(NTILES, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) val send_x_rep_ack = Bool(OUTPUT) } @@ -319,8 +319,8 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { val t_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (NTILES == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(NTILES))) - val p_req_flags = Reg(resetVal = Bits(0, width = NTILES)) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -336,7 +336,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(NTILES) // TODO: Broadcast only + io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.t_type := t_type_ io.mem_req_cmd.valid := Bool(false) @@ -350,13 +350,13 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = NTILES) - io.pop_p_rep := Bits(0, width = NTILES) - io.pop_p_rep_data := Bits(0, width = NTILES) - io.pop_p_rep_dep := Bits(0, width = NTILES) - io.pop_x_init := Bits(0, width = NTILES) - io.pop_x_init_data := Bits(0, width = NTILES) - io.pop_x_init_dep := Bits(0, width = NTILES) + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -368,7 +368,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(NTILES > 1) p_rep_count := UFix(NTILES-1) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -388,7 +388,7 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { when(io.p_rep_cnt_dec.orR) { val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(NTILES > 1) p_rep_count := p_rep_count_next + if(ntiles > 1) p_rep_count := p_rep_count_next when(p_rep_count === UFix(0)) { io.pop_p_rep := Bool(true) state := s_mem @@ -440,14 +440,14 @@ class XactTracker(id: Int) extends Component with FourStateCoherence { } } -abstract class CoherenceHub extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { val io = new Bundle { - val tiles = Vec(NTILES) { new ioTileLink() }.flip + val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub { +class CoherenceHubNull extends CoherenceHub(1) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED @@ -474,7 +474,7 @@ class CoherenceHubNull extends CoherenceHub { } -class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -487,7 +487,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ )) } - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -498,8 +498,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(NTILES){ Wire(){Bool()} } } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } @@ -523,17 +523,17 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ sent_x_rep_ack_arr(i) := Bool(false) p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(NTILES)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish do_free_arr(finish.bits.global_xact_id) := finish.valid finish.ready := Bool(true) @@ -543,7 +543,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.t_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -583,7 +583,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ io.mem.req_data <> mem_req_data_arb.io.out // Handle probe replies, which may or may not have data - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -601,10 +601,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until NTILES) { + for( j <- 0 until ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -612,9 +612,9 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(NTILES) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(NTILES) { Wire() { Bool()} } - for( j <- 0 until NTILES ) { + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -662,7 +662,7 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(NTILES)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -671,10 +671,10 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until NTILES).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until NTILES ) { + for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -694,8 +694,8 @@ class CoherenceHubBroadcast extends CoherenceHub with FourStateCoherence{ // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(NTILES)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until NTILES ) { + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 53cd543d3f6ad91c44262225431b1558836256a0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 13 Mar 2012 19:10:54 -0700 Subject: [PATCH 063/688] fix minor coherence bugs --- uncore/coherence.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ae9321b1..fc32b6eb 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -207,12 +207,12 @@ trait FourStateCoherence extends CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, state, Array( + val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( probeInvalidate -> P_REP_INVALIDATE_DATA, probeDowngrade -> P_REP_DOWNGRADE_DATA, probeCopy -> P_REP_COPY_DATA )) - val without_data = MuxLookup(incoming.p_type, state, Array( + val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( probeInvalidate -> P_REP_INVALIDATE_ACK, probeDowngrade -> P_REP_DOWNGRADE_ACK, probeCopy -> P_REP_COPY_ACK @@ -369,12 +369,13 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) - p_req_flags := ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_flags.orR, s_probe, s_mem) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } } is(s_probe) { @@ -389,7 +390,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(0)) { + when(p_rep_count === UFix(1)) { io.pop_p_rep := Bool(true) state := s_mem } @@ -529,8 +530,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS, true)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { @@ -589,7 +590,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val idx = p_rep.bits.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := do_pop + p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) From 77c405ffa17d1121439a59d32d6fa38f876d9856 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 14 Mar 2012 06:13:16 -0700 Subject: [PATCH 064/688] use broadcast hub and coherent HTIF --- uncore/coherence.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index fc32b6eb..7cf36e8a 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -237,7 +237,7 @@ trait FourStateCoherence extends CoherencePolicy { class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData } + val p_data = (new ioPipe) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) @@ -387,10 +387,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs } when(io.p_rep_cnt_dec.orR) { - val p_rep_count_next = p_rep_count - PopCount(io.p_rep_cnt_dec) + val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count_next - when(p_rep_count === UFix(1)) { + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { io.pop_p_rep := Bool(true) state := s_mem } @@ -536,7 +536,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS // Free finished transactions for( j <- 0 until ntiles ) { val finish = io.tiles(j).xact_finish - do_free_arr(finish.bits.global_xact_id) := finish.valid + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } finish.ready := Bool(true) } @@ -552,18 +554,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid) { + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx - rep.valid := (UFix(j) === init_tile_id_arr(mem_idx)) + rep.valid := Bool(true) } . otherwise { rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx - val do_send_ack = (UFix(j) === init_tile_id_arr(ack_idx)) && send_x_rep_ack_arr.toBits.orR - rep.valid := do_send_ack - sent_x_rep_ack_arr(ack_idx) := do_send_ack + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } } } // If there were a ready signal due to e.g. intervening network use: @@ -594,8 +597,10 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - p_data_valid_arr(idx) := p_rep.valid && probeReplyHasData(p_rep.bits) - p_data_tile_id_arr(idx) := UFix(j) + when (p_rep.valid) { + p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + p_data_tile_id_arr(idx) := UFix(j) + } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { From 3129040bda033ae34deffc14f7c4f8f3e0b58f8d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 15 Mar 2012 18:36:07 -0700 Subject: [PATCH 065/688] use divided clk for htif. UDPATE YOUR FESVR by default, we now load programs via a backdoor, because otherwise it takes too long to simulate. --- uncore/slowio.scala | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 uncore/slowio.scala diff --git a/uncore/slowio.scala b/uncore/slowio.scala new file mode 100644 index 00000000..c1535044 --- /dev/null +++ b/uncore/slowio.scala @@ -0,0 +1,49 @@ +package rocket + +import Chisel._ +import Constants._ + +class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +{ + val io = new Bundle { + val out_fast = new ioDecoupled()(data).flip + val out_slow = new ioDecoupled()(data) + + val in_fast = new ioDecoupled()(data) + val in_slow = new ioDecoupled()(data).flip + + val clk_slow = Bool(OUTPUT) + } + + require((divisor & (divisor-1)) == 0) + require(hold_cycles < divisor/2 && hold_cycles >= 2) + + val cnt = Reg() { UFix(width = log2up(divisor)) } + cnt := cnt + UFix(1) + val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time + val in_en = cnt === UFix(divisor/2-1) // rising edge + + val in_slow_rdy = Reg(resetVal = Bool(false)) + val out_slow_val = Reg(resetVal = Bool(false)) + val out_slow_bits = Reg() { data } + + val fromhost_q = new queue(1)(data) + fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.bits := io.in_slow.bits + fromhost_q.io.deq <> io.in_fast + + val tohost_q = new queue(1)(data) + tohost_q.io.enq <> io.out_fast + tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + + when (out_en) { + in_slow_rdy := fromhost_q.io.enq.ready + out_slow_val := tohost_q.io.deq.valid + out_slow_bits := tohost_q.io.deq.bits + } + + io.in_slow.ready := in_slow_rdy + io.out_slow.valid := out_slow_val + io.out_slow.bits := out_slow_bits + io.clk_slow := cnt(log2up(divisor)-1).toBool +} From e38114e4b0f87777a98d3e8e85f8870d3e6bf269 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Mar 2012 01:24:07 -0700 Subject: [PATCH 066/688] fix coherence bug popping wrong store dependence queue --- uncore/coherence.scala | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7cf36e8a..9160a84b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -295,12 +295,14 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && req_data.valid) { - pop_data := UFix(1) << tile_id - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } } } } @@ -391,7 +393,6 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.pop_p_rep := io.p_rep_cnt_dec if(ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { - io.pop_p_rep := Bool(true) state := s_mem } } From 5f69c5a764689ee6c85712563100d6b03ffaf5b0 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 20 Mar 2012 00:18:32 -0700 Subject: [PATCH 067/688] fix bug in coherence hub, specifically in abort handling logic --- uncore/coherence.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9160a84b..9cd8fc7c 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -648,9 +648,9 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - } - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } } } is(s_abort_send) { // nothing is dequeued for now From a7ebea13fcab7fc79b56ad402655531ed8c4650a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 17:03:58 -0700 Subject: [PATCH 068/688] add mem serdes unit --- uncore/memserdes.scala | 89 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 uncore/memserdes.scala diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala new file mode 100644 index 00000000..2305ed69 --- /dev/null +++ b/uncore/memserdes.scala @@ -0,0 +1,89 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ +import scala.math._ + +class ioMemSerialized(w: Int) extends Bundle +{ + val req = (new ioDecoupled) { Bits(width = w) } + val resp = (new ioPipe) { Bits(width = w) }.flip +} + +class MemSerdes(w: Int) extends Component +{ + val io = new Bundle { + val wide = new ioMem().flip + val narrow = new ioMemSerialized(w) + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + val out_buf = Reg() { Bits() } + val in_buf = Reg() { Bits() } + + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_idle) + val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + + when (state === s_idle) { + when (io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) + } + } + when (state === s_read_addr && adone) { + state := s_idle + send_cnt := UFix(0) + } + when (state === s_write_addr && adone) { + state := s_write_idle + send_cnt := UFix(0) + } + when (state === s_write_idle && io.wide.req_data.valid) { + state := s_write_data + } + when (state === s_write_data && ddone) { + data_send_cnt := data_send_cnt + UFix(1) + state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + } + + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + val recv_cnt = Reg() { UFix(width = log2up(rbits)) } + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val resp_val = Reg(resetVal = Bool(false)) + + resp_val := Bool(false) + when (io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + when (recv_cnt === UFix((rbits-1)/w)) { + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + resp_val := Bool(true) + } + in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + } + + io.wide.resp.valid := resp_val + io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) + io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) +} From 5a00143035d1053bd30db2daac4460360c470fa1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 21:45:10 -0700 Subject: [PATCH 069/688] loop host.in to host.out during reset --- uncore/slowio.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index c1535044..395bdc8d 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,7 +28,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val out_slow_bits = Reg() { data } val fromhost_q = new queue(1)(data) - fromhost_q.io.enq.valid := in_en && io.in_slow.valid && in_slow_rdy + fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -39,7 +39,7 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp when (out_en) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := tohost_q.io.deq.bits + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 4e6302fedc326e2c95549c10839561e0c5d2759f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 25 Mar 2012 23:03:20 -0700 Subject: [PATCH 070/688] add dessert --- uncore/memserdes.scala | 117 ++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 24 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 2305ed69..f68d36e8 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -27,14 +27,28 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) - when (state === s_idle) { - when (io.wide.req_cmd.valid) { - state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) - } + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := send_cnt + UFix(1) + out_buf := out_buf >> UFix(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + when (state === s_idle && io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) } when (state === s_read_addr && adone) { state := s_idle @@ -50,26 +64,11 @@ class MemSerdes(w: Int) extends Component when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UFix(1) state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UFix(0) } - when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := Mux(adone, UFix(0), send_cnt + UFix(1)) - out_buf := out_buf >> UFix(w) - } - when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { - out_buf := io.wide.req_cmd.bits.toBits - } - when (io.wide.req_data.valid && io.wide.req_data.ready) { - out_buf := io.wide.req_data.bits.toBits - } - - io.wide.req_cmd.ready := state === s_idle - io.wide.req_data.ready := state === s_write_idle - io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data - io.narrow.req.bits := out_buf - - val recv_cnt = Reg() { UFix(width = log2up(rbits)) } - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(MEM_DATA_BITS))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -80,10 +79,80 @@ class MemSerdes(w: Int) extends Component data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf(rbits-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } + +class MemDessert(w: Int) extends Component // test rig side +{ + val io = new Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + require(dbits >= abits && rbits >= dbits) + val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } + val state = Reg(resetVal = s_cmd_recv) + + val in_buf = Reg() { Bits() } + when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { + recv_cnt := recv_cnt + UFix(1) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + } + io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv + + when (state === s_cmd_recv && adone) { + state := s_cmd + recv_cnt := UFix(0) + } + when (state === s_cmd && io.wide.req_cmd.ready) { + state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) + } + when (state === s_data_recv && ddone) { + state := s_data + recv_cnt := UFix(0) + } + when (state === s_data && io.wide.req_data.ready) { + state := s_data_recv + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + data_recv_cnt := data_recv_cnt + UFix(1) + } + when (rdone) { // state === s_reply + when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + state := s_cmd_recv + } + recv_cnt := UFix(0) + data_recv_cnt := data_recv_cnt + UFix(1) + } + + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + io.wide.req_cmd.valid := state === s_cmd + io.wide.req_cmd.bits.tag := req_cmd + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + + io.wide.req_data.valid := state === s_data + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + + val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + dataq.io.enq <> io.wide.resp + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + + io.narrow.resp.valid := dataq.io.deq.valid + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) +} From 25fe46dc18554e2a3bf78d977a943b64258535c9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 14:18:57 -0700 Subject: [PATCH 071/688] remove bug from dessert --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index f68d36e8..c1d1fccb 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,7 +26,7 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up(max(abits, dbits)))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) @@ -98,7 +98,7 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up(rbits))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) From 257747a3a149f3f90d9645ab3a2cd3b5ca10b09e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Mar 2012 23:50:09 -0700 Subject: [PATCH 072/688] no dessert tonight :( --- uncore/memserdes.scala | 46 +++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index c1d1fccb..27bbaa95 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -5,17 +5,17 @@ import Node._ import Constants._ import scala.math._ -class ioMemSerialized(w: Int) extends Bundle +class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = w) } - val resp = (new ioPipe) { Bits(width = w) }.flip + val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip } -class MemSerdes(w: Int) extends Component +class MemSerdes extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val narrow = new ioMemSerialized } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -26,14 +26,14 @@ class MemSerdes(w: Int) extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+w-1)/w))) + val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -67,19 +67,19 @@ class MemSerdes(w: Int) extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { + when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.wide.resp.valid := resp_val @@ -87,10 +87,10 @@ class MemSerdes(w: Int) extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert(w: Int) extends Component // test rig side +class MemDessert extends Component // test rig side { val io = new Bundle { - val narrow = new ioMemSerialized(w).flip + val narrow = new ioMemSerialized().flip val wide = new ioMem } val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -98,11 +98,11 @@ class MemDessert(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+w-1)/w))) + val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -110,7 +110,7 @@ class MemDessert(w: Int) extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -140,19 +140,19 @@ class MemDessert(w: Int) extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) val dataq = (new queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) } From 73d8d42515680d903b18e2a05ecd593dc55d73e9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 9 Apr 2012 21:40:35 -0700 Subject: [PATCH 073/688] fix coherence bug with multiple probe replies --- uncore/coherence.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 9cd8fc7c..678f83ab 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -598,8 +598,8 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS p_rep_data_dep_list(j).io.enq.valid := do_pop p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid) { - p_data_valid_arr(idx) := probeReplyHasData(p_rep.bits) + when (p_rep.valid && probeReplyHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) p_data_tile_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) From 27e3346c14778bac96b87ec9ecea7f1118dd781f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 074/688] Refactored coherence better from uncore hub, better coherence function names --- uncore/coherence.scala | 632 ++++++----------------------------------- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+), 544 deletions(-) create mode 100644 uncore/uncore.scala diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 678f83ab..ee51e469 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -3,42 +3,6 @@ package rocket import Chisel._ import Constants._ -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends Bundle -{ - val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - class TransactionInit extends Bundle { val t_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) @@ -75,17 +39,6 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit() } - val xact_init_data = (new ioDecoupled) { new TransactionInitData() } - val xact_abort = (new ioDecoupled) { new TransactionAbort() }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest() }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply() } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData() } - val xact_rep = (new ioPipe) { new TransactionReply() }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish() } -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) @@ -101,6 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, 2) + val X_INIT_READ_EXCLUSIVE = UFix(1, 2) + val X_INIT_WRITE_UNCACHED = UFix(3, 2) + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -116,25 +76,25 @@ trait ThreeStateIncoherence extends CoherencePolicy { } def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid + def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) } - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = state + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } reply.p_type := P_REP_INVALIDATE_ACK @@ -151,43 +111,66 @@ trait FourStateCoherence extends CoherencePolicy { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) + val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) + val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) + val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + + val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) + val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) + val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) + val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) + + val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) + val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) + val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) + + val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) + val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) + val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) + val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) + + def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - - //TODO: do we need isPresent() for determining that a line needs to be - //upgraded but that no replacement is needed? - def isValid (state: UFix): Bool = { state != tileInvalid } - def needsWriteback (state: UFix): Bool = { - state === tileExclusiveDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) - } - def newTransactionOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) - } - def newTransactionOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) - } - def needsSecondaryXact(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) } - def newStateOnTransactionRep(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { MuxLookup(incoming.t_type, tileInvalid, Array( X_REP_READ_SHARED -> tileShared, X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), @@ -196,8 +179,7 @@ trait FourStateCoherence extends CoherencePolicy { X_REP_WRITE_UNCACHED -> tileInvalid )) } - - def newStateOnProbeReq(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -205,6 +187,17 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( @@ -221,55 +214,31 @@ trait FourStateCoherence extends CoherencePolicy { reply.global_xact_id := incoming.global_xact_id reply } - def probeReplyHasData (reply: ProbeReply): Bool = { + + def hasData (reply: ProbeReply): Bool = { (reply.p_type === P_REP_INVALIDATE_DATA || reply.p_type === P_REP_DOWNGRADE_DATA || reply.p_type === P_REP_COPY_DATA) } - def transactionInitHasData (init: TransactionInit): Bool = { + def hasData (init: TransactionInit): Bool = { (init.t_type === X_INIT_WRITE_UNCACHED) } - def transactionReplyHasData (reply: TransactionReply): Bool = { + def hasData (reply: TransactionReply): Bool = { (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) } -} -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { - val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) - val send_x_rep_ack = Bool(OUTPUT) + def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { + MuxLookup(t_type, X_REP_READ_UNCACHED, Array( + X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), + X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, + X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, + X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + )) } - def sendProbeReqType(t_type: UFix, global_state: UFix): UFix = { + def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { MuxLookup(t_type, P_REQ_COPY, Array( X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, @@ -281,435 +250,10 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc def needsMemRead(t_type: UFix, global_state: UFix): Bool = { (t_type != X_INIT_WRITE_UNCACHED) } - - def needsAckRep(t_type: UFix, global_state: UFix): Bool = { + def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { (t_type === X_INIT_WRITE_UNCACHED) } - - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue - req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue - req_data.bits := data.bits - lock := at_front_of_dep_queue - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := sendProbeReqType(t_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := transactionInitHasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(needsAckRep(t_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } + def needsAckReply(t_type: UFix, global_state: UFix): Bool = { + (t_type === X_INIT_WRITE_UNCACHED) } } - -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull extends CoherenceHub(1) { - - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence{ - - def coherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED - )) - } - - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) - } - } - } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && probeReplyHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && coherenceConflict(t.addr, x_init.bits.address) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && transactionInitHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(transactionInitHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete - } - } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && transactionInitHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From d301336c3366bc18263ed486d4aa055b85a598ba Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 12:03:05 -0700 Subject: [PATCH 075/688] Refactored coherence better from uncore hub, better coherence function names --- uncore/uncore.scala | 503 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 503 insertions(+) create mode 100644 uncore/uncore.scala diff --git a/uncore/uncore.scala b/uncore/uncore.scala new file mode 100644 index 00000000..fa6e2b1e --- /dev/null +++ b/uncore/uncore.scala @@ -0,0 +1,503 @@ +package rocket + +import Chisel._ +import Constants._ + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd() extends Bundle +{ + val rw = Bool() + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp () extends MemData +{ + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem() extends Bundle +{ + val req_cmd = (new ioDecoupled) { new MemReqCmd() } + val req_data = (new ioDecoupled) { new MemData() } + val resp = (new ioPipe) { new MemResp() }.flip +} + +class TrackerProbeData extends Bundle { + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerAllocReq extends Bundle { + val xact_init = new TransactionInit() + val tile_id = Bits(width = TILE_ID_BITS) +} + +class TrackerDependency extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new ioDecoupled) { new TransactionInit } + val xact_init_data = (new ioDecoupled) { new TransactionInitData } + val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip + val probe_req = (new ioDecoupled) { new ProbeRequest }.flip + val probe_rep = (new ioDecoupled) { new ProbeReply } + val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } + val xact_rep = (new ioPipe) { new TransactionReply }.flip + val xact_finish = (new ioDecoupled) { new TransactionFinish } +} + +class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { + val io = new Bundle { + val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip + val p_data = (new ioPipe) { new TrackerProbeData }.flip + val can_alloc = Bool(INPUT) + val xact_finish = Bool(INPUT) + val p_rep_cnt_dec = Bits(ntiles, INPUT) + val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip + val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val sent_x_rep_ack = Bool(INPUT) + val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip + val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + + val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } + val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val probe_req = (new ioDecoupled) { new ProbeRequest } + val busy = Bool(OUTPUT) + val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) + val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) + val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) + val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) + val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val push_p_req = Bits(ntiles, OUTPUT) + val pop_p_rep = Bits(ntiles, OUTPUT) + val pop_p_rep_data = Bits(ntiles, OUTPUT) + val pop_p_rep_dep = Bits(ntiles, OUTPUT) + val pop_x_init = Bits(ntiles, OUTPUT) + val pop_x_init_data = Bits(ntiles, OUTPUT) + val pop_x_init_dep = Bits(ntiles, OUTPUT) + val send_x_rep_ack = Bool(OUTPUT) + } + + def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.bits.rw := Bool(true) + req_data.valid := data.valid && at_front_of_dep_queue + req_data.bits := data.bits + lock := at_front_of_dep_queue + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when(req_data.ready && at_front_of_dep_queue) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt_next === UFix(0)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } + } + } + + def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val addr_ = Reg{ UFix() } + val t_type_ = Reg{ Bits() } + val init_tile_id_ = Reg{ Bits() } + val tile_xact_id_ = Reg{ Bits() } + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_tile_id_ = Reg{ Bits() } + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + + io.busy := state != s_idle + io.addr := addr_ + io.init_tile_id := init_tile_id_ + io.p_rep_tile_id := p_rep_tile_id_ + io.tile_xact_id := tile_xact_id_ + io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.t_type := t_type_ + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.probe_req.valid := Bool(false) + io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.global_xact_id := UFix(id) + io.probe_req.bits.address := addr_ + io.push_p_req := Bits(0, width = ntiles) + io.pop_p_rep := Bits(0, width = ntiles) + io.pop_p_rep_data := Bits(0, width = ntiles) + io.pop_p_rep_dep := Bits(0, width = ntiles) + io.pop_x_init := Bits(0, width = ntiles) + io.pop_x_init_data := Bits(0, width = ntiles) + io.pop_x_init_dep := Bits(0, width = ntiles) + io.send_x_rep_ack := Bool(false) + + switch (state) { + is(s_idle) { + when( io.alloc_req.valid && io.can_alloc ) { + addr_ := io.alloc_req.bits.xact_init.address + t_type_ := io.alloc_req.bits.xact_init.t_type + init_tile_id_ := io.alloc_req.bits.tile_id + tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id + x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + if(ntiles > 1) p_rep_count := UFix(ntiles-1) + val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } + } + is(s_probe) { + when(p_req_flags.orR) { + io.push_p_req := p_req_flags + io.probe_req.valid := Bool(true) + } + when(io.p_req_cnt_inc.orR) { + p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + } + when(io.p_rep_cnt_dec.orR) { + val dec = PopCount(io.p_rep_cnt_dec) + io.pop_p_rep := io.p_rep_cnt_dec + if(ntiles > 1) p_rep_count := p_rep_count - dec + when(p_rep_count === dec) { + state := s_mem + } + } + when(io.p_data.valid) { + p_rep_data_needs_write := Bool(true) + p_rep_tile_id_ := io.p_data.bits.tile_id + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.pop_p_rep_data, + io.pop_p_rep_dep, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + p_rep_tile_id_) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.pop_x_init_data, + io.pop_x_init_dep, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + init_tile_id_) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + } + } + is(s_ack) { + io.send_x_rep_ack := Bool(true) + when(io.sent_x_rep_ack) { state := s_busy } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.xact_finish) { + state := s_idle + } + } + } +} + +abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { + val io = new Bundle { + val tiles = Vec(ntiles) { new ioTileLink() }.flip + val mem = new ioMem + } +} + +class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +{ + val x_init = io.tiles(0).xact_init + val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + io.mem.req_cmd.bits.rw := is_write + io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_data <> io.tiles(0).xact_init_data + + val x_rep = io.tiles(0).xact_rep + x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) + x_rep.bits.global_xact_id := UFix(0) // don't care + x_rep.bits.data := io.mem.resp.bits.data + x_rep.bits.require_ack := Bool(true) + x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + + io.tiles(0).xact_abort.valid := Bool(false) + io.tiles(0).xact_finish.ready := Bool(true) + io.tiles(0).probe_req.valid := Bool(false) + io.tiles(0).probe_rep.ready := Bool(true) + io.tiles(0).probe_rep_data.ready := Bool(true) +} + + +class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +{ + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + + val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } + val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + + val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + busy_arr(i) := t.busy + addr_arr(i) := t.addr + init_tile_id_arr(i) := t.init_tile_id + tile_xact_id_arr(i) := t.tile_xact_id + t_type_arr(i) := t.t_type + sh_count_arr(i) := t.sharer_count + send_x_rep_ack_arr(i) := t.send_x_rep_ack + t.xact_finish := do_free_arr(i) + t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.valid := p_data_valid_arr(i) + t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits + t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + do_free_arr(i) := Bool(false) + sent_x_rep_ack_arr(i) := Bool(false) + p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_valid_arr(i) := Bool(false) + for( j <- 0 until ntiles) { + p_rep_cnt_dec_arr(i)(j) := Bool(false) + p_req_cnt_inc_arr(i)(j) := Bool(false) + } + } + + val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + + // Free finished transactions + for( j <- 0 until ntiles ) { + val finish = io.tiles(j).xact_finish + when (finish.valid) { + do_free_arr(finish.bits.global_xact_id) := Bool(true) + } + finish.ready := Bool(true) + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val mem_idx = io.mem.resp.bits.tag + val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + for( j <- 0 until ntiles ) { + val rep = io.tiles(j).xact_rep + rep.bits.t_type := UFix(0) + rep.bits.tile_xact_id := UFix(0) + rep.bits.global_xact_id := UFix(0) + rep.bits.data := io.mem.resp.bits.data + rep.bits.require_ack := Bool(true) + rep.valid := Bool(false) + when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { + rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.global_xact_id := mem_idx + rep.valid := Bool(true) + } . otherwise { + rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.global_xact_id := ack_idx + when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.valid := send_x_rep_ack_arr.toBits.orR + sent_x_rep_ack_arr(ack_idx) := Bool(true) + } + } + } + // If there were a ready signal due to e.g. intervening network use: + //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> mem_req_cmd_arb.io.out + io.mem.req_data <> mem_req_data_arb.io.out + + // Handle probe replies, which may or may not have data + for( j <- 0 until ntiles ) { + val p_rep = io.tiles(j).probe_rep + val p_rep_data = io.tiles(j).probe_rep_data + val idx = p_rep.bits.global_xact_id + val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) + val do_pop = foldR(pop_p_reps)(_ || _) + p_rep.ready := Bool(true) + p_rep_data_dep_list(j).io.enq.valid := do_pop + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) + when (p_rep.valid && co.messageHasData(p_rep.bits)) { + p_data_valid_arr(idx) := Bool(true) + p_data_tile_id_arr(idx) := UFix(j) + } + p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + } + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + + for( j <- 0 until ntiles) { + val p_rep = io.tiles(j).probe_rep + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + } + } + + // Nack conflicting transaction init attempts + val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_abort = io.tiles(j).xact_abort + val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + for( i <- 0 until NGLOBAL_XACTS) { + val t = trackerList(i).io + conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + } + x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + + x_abort.valid := Bool(false) + switch(abort_state_arr(j)) { + is(s_idle) { + when(want_to_abort_arr(j)) { + when(hasData(x_init.bits)) { + abort_state_arr(j) := s_abort_drain + } . otherwise { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + abort_state_arr(j) := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { + abort_state_arr(j) := s_abort_complete + } + } + is(s_abort_complete) { // raises x_init.ready below + abort_state_arr(j) := s_idle + } + } + } + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready + trackerList(i).io.alloc_req.bits := init_arb.io.out.bits + trackerList(i).io.alloc_req.valid := init_arb.io.out.valid + + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + } + for( j <- 0 until ntiles ) { + val x_init = io.tiles(j).xact_init + val x_init_data = io.tiles(j).xact_init_data + val x_init_data_dep = x_init_data_dep_list(j).io.deq + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid + init_arb.io.in(j).bits.xact_init := x_init.bits + init_arb.io.in(j).bits.tile_id := UFix(j) + val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) + val do_pop = foldR(pop_x_inits)(_||_) + x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) + x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + } + + alloc_arb.io.out.ready := init_arb.io.out.valid + + // Handle probe request generation + // Must arbitrate for each request port + val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until ntiles ) { + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits + p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + } + p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + } + +} From 6bc47a55b42e28b34cf4845a82387f33cc3c91e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 076/688] changed coherence message type names --- uncore/coherence.scala | 147 ++++++++++++++++++----------------------- uncore/uncore.scala | 28 ++++---- 2 files changed, 78 insertions(+), 97 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index ee51e469..7af9c903 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val t_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -29,7 +29,7 @@ class ProbeReply extends Bundle { class ProbeReplyData extends MemData class TransactionReply extends MemData { - val t_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -54,13 +54,13 @@ trait CoherencePolicy { } trait ThreeStateIncoherence extends CoherencePolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, 2) - val X_INIT_READ_EXCLUSIVE = UFix(1, 2) - val X_INIT_WRITE_UNCACHED = UFix(3, 2) - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) + val xactInitReadShared = UFix(0, 2) + val xactInitReadExclusive = UFix(1, 2) + val xactInitWriteUncached = UFix(3, 2) + val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -84,57 +84,38 @@ trait ThreeStateIncoherence extends CoherencePolicy { def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileDirty, tileClean) + Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { val reply = Wire() { new ProbeReply() } - reply.p_type := P_REP_INVALIDATE_ACK + reply.p_type := probeRepInvalidateAck reply.global_xact_id := UFix(0) reply } def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.t_type === X_INIT_WRITE_UNCACHED) + def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) } trait FourStateCoherence extends CoherencePolicy { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val X_INIT_READ_SHARED = UFix(0, X_INIT_TYPE_BITS) - val X_INIT_READ_EXCLUSIVE = UFix(1, X_INIT_TYPE_BITS) - val X_INIT_READ_UNCACHED = UFix(2, X_INIT_TYPE_BITS) - val X_INIT_WRITE_UNCACHED = UFix(3, X_INIT_TYPE_BITS) + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val X_REP_READ_SHARED = UFix(0, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE = UFix(1, X_REP_TYPE_BITS) - val X_REP_READ_UNCACHED = UFix(2, X_REP_TYPE_BITS) - val X_REP_WRITE_UNCACHED = UFix(3, X_REP_TYPE_BITS) - val X_REP_READ_EXCLUSIVE_ACK = UFix(4, X_REP_TYPE_BITS) - - val P_REQ_INVALIDATE = UFix(0, P_REQ_TYPE_BITS) - val P_REQ_DOWNGRADE = UFix(1, P_REQ_TYPE_BITS) - val P_REQ_COPY = UFix(2, P_REQ_TYPE_BITS) - - val P_REP_INVALIDATE_DATA = UFix(0, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_DATA = UFix(1, P_REP_TYPE_BITS) - val P_REP_COPY_DATA = UFix(2, P_REP_TYPE_BITS) - val P_REP_INVALIDATE_ACK = UFix(3, P_REP_TYPE_BITS) - val P_REP_DOWNGRADE_ACK = UFix(4, P_REP_TYPE_BITS) - val P_REP_COPY_ACK = UFix(5, P_REP_TYPE_BITS) - - - def isHit ( cmd: Bits, state: UFix): Bool = { + def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) @@ -145,8 +126,8 @@ trait FourStateCoherence extends CoherencePolicy { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.t_type === X_INIT_READ_UNCACHED || outstanding.t_type === X_INIT_WRITE_UNCACHED)) || - (write && (outstanding.t_type != X_INIT_READ_EXCLUSIVE)) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -171,44 +152,44 @@ trait FourStateCoherence extends CoherencePolicy { def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.t_type, tileInvalid, Array( - X_REP_READ_SHARED -> tileShared, - X_REP_READ_EXCLUSIVE -> Mux(outstanding.t_type === X_INIT_READ_EXCLUSIVE, tileExclusiveDirty, tileExclusiveClean), - X_REP_READ_EXCLUSIVE_ACK -> tileExclusiveDirty, - X_REP_READ_UNCACHED -> tileInvalid, - X_REP_WRITE_UNCACHED -> tileInvalid + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state )) } def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, X_INIT_READ_EXCLUSIVE, X_INIT_READ_SHARED) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, X_INIT_READ_EXCLUSIVE, outstanding.t_type) + Mux(write, xactInitReadExclusive, outstanding.x_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = X_INIT_WRITE_UNCACHED + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { val reply = Wire() { new ProbeReply() } - val with_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_DATA, Array( - probeInvalidate -> P_REP_INVALIDATE_DATA, - probeDowngrade -> P_REP_DOWNGRADE_DATA, - probeCopy -> P_REP_COPY_DATA + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData )) - val without_data = MuxLookup(incoming.p_type, P_REP_INVALIDATE_ACK, Array( - probeInvalidate -> P_REP_INVALIDATE_ACK, - probeDowngrade -> P_REP_DOWNGRADE_ACK, - probeCopy -> P_REP_COPY_ACK + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck )) reply.p_type := Mux(needsWriteback(state), with_data, without_data) reply.global_xact_id := incoming.global_xact_id @@ -216,44 +197,44 @@ trait FourStateCoherence extends CoherencePolicy { } def hasData (reply: ProbeReply): Bool = { - (reply.p_type === P_REP_INVALIDATE_DATA || - reply.p_type === P_REP_DOWNGRADE_DATA || - reply.p_type === P_REP_COPY_DATA) + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) } def hasData (init: TransactionInit): Bool = { - (init.t_type === X_INIT_WRITE_UNCACHED) + (init.x_type === xactInitWriteUncached) } def hasData (reply: TransactionReply): Bool = { - (reply.t_type != X_REP_WRITE_UNCACHED && reply.t_type != X_REP_READ_EXCLUSIVE_ACK) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(t_type: UFix, count: UFix): Bits = { - MuxLookup(t_type, X_REP_READ_UNCACHED, Array( - X_INIT_READ_SHARED -> Mux(count > UFix(0), X_REP_READ_SHARED, X_REP_READ_EXCLUSIVE), - X_INIT_READ_EXCLUSIVE -> X_REP_READ_EXCLUSIVE, - X_INIT_READ_UNCACHED -> X_REP_READ_UNCACHED, - X_INIT_WRITE_UNCACHED -> X_REP_WRITE_UNCACHED + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached )) } - def getProbeRequestType(t_type: UFix, global_state: UFix): UFix = { - MuxLookup(t_type, P_REQ_COPY, Array( - X_INIT_READ_SHARED -> P_REQ_DOWNGRADE, - X_INIT_READ_EXCLUSIVE -> P_REQ_INVALIDATE, - X_INIT_READ_UNCACHED -> P_REQ_COPY, - X_INIT_WRITE_UNCACHED -> P_REQ_INVALIDATE + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate )) } - def needsMemRead(t_type: UFix, global_state: UFix): Bool = { - (t_type != X_INIT_WRITE_UNCACHED) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) } - def needsMemWrite(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } - def needsAckReply(t_type: UFix, global_state: UFix): Bool = { - (t_type === X_INIT_WRITE_UNCACHED) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) } } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From f7307ee41164fe47fee1a1730c23f8c2929f0c0f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Apr 2012 18:06:02 -0700 Subject: [PATCH 077/688] changed coherence message type names --- uncore/uncore.scala | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index fa6e2b1e..8dc88d44 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val t_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -117,7 +117,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val addr_ = Reg{ UFix() } - val t_type_ = Reg{ Bits() } + val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) @@ -138,7 +138,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.t_type := t_type_ + io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(t_type_, UFix(0)) + io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -164,11 +164,11 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { addr_ := io.alloc_req.bits.xact_init.address - t_type_ := io.alloc_req.bits.xact_init.t_type + x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.t_type, UFix(0)) + x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(t_type_, UFix(0)), s_ack, s_busy) + state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.t_type === X_INIT_WRITE_UNCACHED + val is_write = x_init.bits.x_type === xactInitWriteUncached x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.t_type := Mux(io.mem.resp.valid, X_REP_READ_EXCLUSIVE, X_REP_WRITE_UNCACHED) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val t_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } @@ -300,7 +300,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS addr_arr(i) := t.addr init_tile_id_arr(i) := t.init_tile_id tile_xact_id_arr(i) := t.tile_xact_id - t_type_arr(i) := t.t_type + x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) @@ -337,19 +337,19 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until ntiles ) { val rep = io.tiles(j).xact_rep - rep.bits.t_type := UFix(0) + rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) rep.bits.global_xact_id := UFix(0) rep.bits.data := io.mem.resp.bits.data rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.t_type := getTransactionReplyType(t_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.t_type := getTransactionReplyType(t_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { From 17a5d26c1ed11bd1f8adc09a3bf011f0dbf1785c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 078/688] changed coherence type width names to represent max sizes for all protocols --- uncore/coherence.scala | 16 ++++++++-------- uncore/uncore.scala | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 7af9c903..be6638aa 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ class TransactionInit extends Bundle { - val x_type = Bits(width = X_INIT_TYPE_BITS) + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val address = UFix(width = PADDR_BITS - OFFSET_BITS) } @@ -16,20 +16,20 @@ class TransactionAbort extends Bundle { } class ProbeRequest extends Bundle { - val p_type = Bits(width = P_REQ_TYPE_BITS) + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_BITS) + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } class ProbeReplyData extends MemData class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_BITS) + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) val require_ack = Bool() @@ -57,10 +57,10 @@ trait ThreeStateIncoherence extends CoherencePolicy { val xactInitReadShared = UFix(0, 2) val xactInitReadExclusive = UFix(1, 2) val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_BITS) + val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) + val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) + val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) + val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) def isHit ( cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From a68f5e016d5a9567095750909582e5b5e2a98153 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 13:57:08 -0700 Subject: [PATCH 079/688] changed coherence type width names to represent max sizes for all protocols --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 8dc88d44..b907c51d 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -74,7 +74,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_BITS, OUTPUT) + val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) val push_p_req = Bits(ntiles, OUTPUT) val pop_p_rep = Bits(ntiles, OUTPUT) val pop_p_rep_data = Bits(ntiles, OUTPUT) @@ -283,7 +283,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_BITS)} } + val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } From e71e3ce38f16b1b47e386807d0bf8f99b82fa372 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 080/688] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/coherence.scala | 114 ++++++++++++++++++++++++++++------------- uncore/uncore.scala | 4 +- 2 files changed, 79 insertions(+), 39 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index be6638aa..1bd19bbd 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,59 +50,99 @@ object cpuCmdToRW { } } -trait CoherencePolicy { } +trait CoherencePolicy { + def isHit (cmd: Bits, state: UFix): Bool + def isValid (state: UFix): Bool -trait ThreeStateIncoherence extends CoherencePolicy { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool + def needsWriteback (state: UFix): Bool + + def newStateOnHit(cmd: Bits, state: UFix): UFix + def newStateOnCacheControl(cmd: Bits): UFix + def newStateOnWriteback(): UFix + def newStateOnFlush(): UFix + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits + def getTransactionInitTypeOnWriteback(): Bits + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + + def hasData (reply: ProbeReply): Bool + def hasData (init: TransactionInit): Bool + def hasData (reply: TransactionReply): Bool + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def getTransactionReplyType(x_type: UFix, count: UFix): Bits + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix + def needsMemRead(x_type: UFix, global_state: UFix): Bool + def needsMemWrite(x_type: UFix, global_state: UFix): Bool + def needsAckReply(x_type: UFix, global_state: UFix): Bool +} + +trait IncoherentPolicy extends CoherencePolicy { + // UNIMPLEMENTED + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + reply.p_type := UFix(0) + reply.global_xact_id := UFix(0) + reply + } + def hasData (reply: ProbeReply) = Bool(false) + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) +} + +trait ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadShared = UFix(0, 2) - val xactInitReadExclusive = UFix(1, 2) - val xactInitWriteUncached = UFix(3, 2) - val xactReplyReadShared = UFix(0, X_REP_TYPE_MAX_BITS) - val xactReplyReadExclusive = UFix(1, X_REP_TYPE_MAX_BITS) - val xactReplyWriteUncached = UFix(3, X_REP_TYPE_MAX_BITS) - val probeRepInvalidateAck = UFix(3, P_REP_TYPE_MAX_BITS) + val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } + val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } + val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } - def isHit ( cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - ( state === tileClean || state === tileDirty) - } + def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = { - state != tileInvalid - } + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty + def needsWriteback (state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = { - state === tileDirty - } - - def newStateOnWriteback() = tileInvalid - def newStateOnCacheControl(cmd: Bits) = tileInvalid def newState(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) + def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnWriteback() = tileInvalid + def newStateOnFlush() = tileInvalid + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), + xactReplyAck -> tileInvalid + )) + } + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) } def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, xactInitReadDirty, outstanding.x_type) } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = Bool(false) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - Mux(outstanding.x_type === xactInitReadExclusive, tileDirty, tileClean) - } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, has_data: Bool): ProbeReply = { - val reply = Wire() { new ProbeReply() } - reply.p_type := probeRepInvalidateAck - reply.global_xact_id := UFix(0) - reply - } - def probeReplyHasData (reply: ProbeReply): Bool = Bool(false) - def transactionInitHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteUncached) + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO + def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + + def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) } trait FourStateCoherence extends CoherencePolicy { diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From 5acf1d982014bab2c571faf20317ea92ca8f96c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 4 Apr 2012 15:51:33 -0700 Subject: [PATCH 081/688] defined abstract coherence traits in base trait, added Incoherent trait, cleaned up incoherent policy --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index b907c51d..14164f7f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -251,7 +251,7 @@ abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteUncached + val is_write = x_init.bits.x_type === xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyReadExclusive, xactReplyWriteUncached) + x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data From b22d7f81929b8a01e67306e9e5664b9bdd79d46f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 082/688] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/coherence.scala | 284 +++++++++++++++++++++++++++++++++++++++-- uncore/uncore.scala | 36 +++--- 2 files changed, 289 insertions(+), 31 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index 1bd19bbd..e0f5e5c8 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -50,7 +50,7 @@ object cpuCmdToRW { } } -trait CoherencePolicy { +abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -72,9 +72,10 @@ trait CoherencePolicy { def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply - def hasData (reply: ProbeReply): Bool - def hasData (init: TransactionInit): Bool - def hasData (reply: TransactionReply): Bool + def messageHasData (reply: ProbeReply): Bool + def messageHasData (init: TransactionInit): Bool + def messageHasData (reply: TransactionReply): Bool + def messageUpdatesDataArray (reply: TransactionReply): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -84,7 +85,14 @@ trait CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool } -trait IncoherentPolicy extends CoherencePolicy { +trait UncachedTransactions { + def getTransactionInitTypeOnUncachedRead(): UFix + def getTransactionInitTypeOnUncachedWrite(): UFix +} + +abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions + +abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { @@ -93,7 +101,7 @@ trait IncoherentPolicy extends CoherencePolicy { reply.global_xact_id := UFix(0) reply } - def hasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: ProbeReply) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) @@ -102,7 +110,7 @@ trait IncoherentPolicy extends CoherencePolicy { def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) } -trait ThreeStateIncoherence extends IncoherentPolicy { +class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } @@ -141,11 +149,256 @@ trait ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def hasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) - def hasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -trait FourStateCoherence extends CoherencePolicy { +class TwoStateCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } + val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileValid), Array( + M_INV -> (state === tileValid), + M_CLN -> (state === tileValid) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = state + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileValid + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> tileValid, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI + + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid + def isValid (state: UFix): Bool = state != tileInvalid + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileExclusiveClean + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileExclusiveClean, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadExclusiveClean -> xactReplyReadExclusive, + xactInitReadExclusiveDirty -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadExclusiveClean -> probeReqInvalidate, + xactInitReadExclusiveDirty -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class FourStateCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -208,6 +461,8 @@ trait FourStateCoherence extends CoherencePolicy { )) } + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -236,17 +491,20 @@ trait FourStateCoherence extends CoherencePolicy { reply } - def hasData (reply: ProbeReply): Bool = { + def messageHasData (reply: ProbeReply): Bool = { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def hasData (init: TransactionInit): Bool = { + def messageHasData (init: TransactionInit): Bool = { (init.x_type === xactInitWriteUncached) } - def hasData (reply: TransactionReply): Bool = { + def messageHasData (reply: TransactionReply): Bool = { (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 1920c97066f6d1a22f6389621a559243fdfc2ab9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Apr 2012 00:09:58 -0700 Subject: [PATCH 083/688] Refactored coherence as member rather than trait. MI and MEI protocols. --- uncore/uncore.scala | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 14164f7f..a8fc9432 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -50,7 +50,7 @@ class ioTileLink extends Bundle { val xact_finish = (new ioDecoupled) { new TransactionFinish } } -class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherence { +class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip val p_data = (new ioPipe) { new TrackerProbeData }.flip @@ -140,7 +140,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.sharer_count := UFix(ntiles) // TODO: Broadcast only io.x_type := x_type_ - io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := addr_ io.mem_req_cmd.bits.tag := UFix(id) @@ -148,7 +148,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.address := addr_ io.push_p_req := Bits(0, width = ntiles) @@ -167,8 +167,8 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := hasData(io.alloc_req.bits.xact_init) - x_needs_read := needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) + x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only p_req_flags := p_req_initial_flags @@ -226,7 +226,7 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) } } is(s_ack) { @@ -241,17 +241,17 @@ class XactTracker(ntiles: Int, id: Int) extends Component with FourStateCoherenc } } -abstract class CoherenceHub(ntiles: Int) extends Component with CoherencePolicy { +abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { val io = new Bundle { val tiles = Vec(ntiles) { new ioTileLink() }.flip val mem = new ioMem } } -class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence +class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) { val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === xactInitWriteback + val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write @@ -260,7 +260,7 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, xactReplyData, xactReplyAck) + x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) x_rep.bits.global_xact_id := UFix(0) // don't care x_rep.bits.data := io.mem.resp.bits.data @@ -275,9 +275,9 @@ class CoherenceHubNull extends CoherenceHub(1) with ThreeStateIncoherence } -class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourStateCoherence +class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } @@ -344,12 +344,12 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS rep.bits.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) rep.bits.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { @@ -417,16 +417,16 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && hasData(x_init.bits))) + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(hasData(x_init.bits)) { + when(co.messageHasData(x_init.bits)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -478,7 +478,7 @@ class CoherenceHubBroadcast(ntiles: Int) extends CoherenceHub(ntiles) with FourS init_arb.io.in(j).bits.tile_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && hasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) From 2a7d2888a78c4bac3978b5fb79cd6e6513df7e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 084/688] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 98a5d682a59c3a183145ce85628f58e11e8febcf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Apr 2012 02:22:45 -0700 Subject: [PATCH 085/688] coherence mostly works now --- uncore/uncore.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a8fc9432..756c6e7b 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -86,11 +86,11 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && at_front_of_dep_queue + req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := at_front_of_dep_queue + lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } @@ -383,8 +383,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := do_pop - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_p_reps) + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits)) { p_data_valid_arr(idx) := Bool(true) From 4a6c7dbc26d9fd2197fb7e21367c0cabb920c28a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Apr 2012 17:56:59 -0700 Subject: [PATCH 086/688] Policy determined by constants. MSI policy added. --- uncore/coherence.scala | 147 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 143 insertions(+), 4 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index e0f5e5c8..b4b23b4b 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -154,7 +154,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) } -class TwoStateCoherence extends CoherencePolicyWithUncached { +class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } @@ -266,7 +266,7 @@ class TwoStateCoherence extends CoherencePolicyWithUncached { } } -class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI +class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } @@ -398,7 +398,146 @@ class ThreeStateCoherence extends CoherencePolicyWithUncached { //MEI } } -class FourStateCoherence extends CoherencePolicyWithUncached { +class MSICoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, (state === tileExclusiveDirty), + (state === tileShared || state === tileExclusiveDirty)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (write && (outstanding.x_type != xactInitReadExclusive)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> (state === tileExclusiveDirty), + M_CLN -> (state === tileExclusiveDirty) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, tileExclusiveDirty, state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> tileExclusiveDirty, + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqDowngrade -> tileShared, + probeReqCopy -> state + )) + } + + def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached + def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, xactInitReadExclusive, outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + val reply = Wire() { new ProbeReply() } + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> probeRepInvalidateData, + probeReqDowngrade -> probeRepDowngradeData, + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> probeRepInvalidateAck, + probeReqDowngrade -> probeRepDowngradeAck, + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + (reply.p_type === probeRepInvalidateData || + reply.p_type === probeRepDowngradeData || + reply.p_type === probeRepCopyData) + } + def messageHasData (init: TransactionInit): Bool = { + (init.x_type === xactInitWriteUncached) + } + def messageHasData (reply: TransactionReply): Bool = { + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + } + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached) + } +} + +class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } @@ -439,7 +578,7 @@ class FourStateCoherence extends CoherencePolicyWithUncached { def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean + M_CLN -> tileShared )) } def newStateOnWriteback() = newStateOnCacheControl(M_INV) From 37eb1a4ae615a8e0d8f94fdff342c8bc38204ec1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 087/688] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From d61e6ee0806466edfc87bc786754290502d4d673 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Apr 2012 16:31:14 -0700 Subject: [PATCH 088/688] Fixed coherence bug: probe counting for single tile --- uncore/uncore.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 756c6e7b..29d0befd 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -169,14 +169,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - if(ntiles > 1) p_rep_count := UFix(ntiles-1) val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only - p_req_flags := p_req_initial_flags + p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + if(ntiles > 1) { + p_rep_count := UFix(ntiles-1) + state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + } else state := s_mem } } is(s_probe) { From 00155f4bc48247b661ffd46831d999d8a35f9918 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 089/688] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From 99bc99f2ad506a8ec5d2e42dbf4447fe19acd9f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Apr 2012 15:11:59 -0700 Subject: [PATCH 090/688] Fixed abort bug: removed uneeded state, added mshr guard on xact_abort.valid and xact_init.ready on same cycle --- uncore/uncore.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 29d0befd..0f482113 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -408,7 +408,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: s_abort_complete :: Nil = Enum(4){ UFix() } + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } for( j <- 0 until ntiles ) { @@ -445,13 +445,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { - abort_state_arr(j) := s_abort_complete + when(x_abort.ready) { // raises x_init.ready below + abort_state_arr(j) := s_idle } } - is(s_abort_complete) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } } } @@ -475,6 +472,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq + val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid init_arb.io.in(j).bits.xact_init := x_init.bits init_arb.io.in(j).bits.tile_id := UFix(j) @@ -482,7 +480,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (abort_state_arr(j) === s_abort_complete) || do_pop + x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) } From f804c57bb02243004a76c6b7d9b9e6dc38454bfb Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 3 May 2012 04:21:11 -0700 Subject: [PATCH 091/688] reduce HTIF clock divider for now --- uncore/slowio.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 395bdc8d..e54996ef 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -3,7 +3,7 @@ package rocket import Chisel._ import Constants._ -class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Component +class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { val out_fast = new ioDecoupled()(data).flip @@ -15,8 +15,9 @@ class slowIO[T <: Data](divisor: Int, hold_cycles: Int)(data: => T) extends Comp val clk_slow = Bool(OUTPUT) } + val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 2) + require(hold_cycles < divisor/2 && hold_cycles >= 1) val cnt = Reg() { UFix(width = log2up(divisor)) } cnt := cnt + UFix(1) From 0208e9f95e4283d4971fb8b24d1a8cc5f772254d Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 092/688] removing wires --- uncore/coherence.scala | 10 +++++----- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/uncore/coherence.scala b/uncore/coherence.scala index b4b23b4b..04843fec 100644 --- a/uncore/coherence.scala +++ b/uncore/coherence.scala @@ -96,7 +96,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() reply.p_type := UFix(0) reply.global_xact_id := UFix(0) reply @@ -209,7 +209,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqCopy -> probeRepCopyData @@ -336,7 +336,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -475,7 +475,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, @@ -614,7 +614,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = Wire() { new ProbeReply() } + val reply = new ProbeReply() val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( probeReqInvalidate -> probeRepInvalidateData, probeReqDowngrade -> probeRepDowngradeData, diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From 6f2f1ba21c573392a7b21f5afa9c863bb9114abe Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 24 May 2012 10:33:15 -0700 Subject: [PATCH 093/688] removing wires --- uncore/uncore.scala | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 0f482113..ab56d0f4 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -281,20 +281,20 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH { val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - val busy_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val addr_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=PADDR_BITS-OFFSET_BITS)} } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_XACT_ID_BITS)} } - val x_type_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=X_INIT_TYPE_MAX_BITS)} } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Wire(){Bits(width=TILE_ID_BITS)} } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } + val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } + val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } + val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val do_free_arr = Vec(NGLOBAL_XACTS){ Wire(){Bool()} } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Wire(){Bool()} } } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bits(width = TILE_ID_BITS)} } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Wire(){ Bool()} } + val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -410,13 +410,13 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Wire() { Bool()} } + val want_to_abort_arr = Vec(ntiles) { Bool() } for( j <- 0 until ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Wire() { Bool() } } + val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) From f2942f79f98ad17c3fbd423d0896b3a7631b3d21 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 094/688] moving util out into Chisel standard library --- uncore/uncore.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 9b3161920f9a2f0b0905f2df4ee9788a413a3d6e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 12:47:17 -0700 Subject: [PATCH 095/688] moving util out into Chisel standard library --- uncore/memserdes.scala | 12 ++++++------ uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 27bbaa95..8dc6a8ae 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -26,8 +26,8 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_send_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) @@ -67,8 +67,8 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) @@ -98,8 +98,8 @@ class MemDessert extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2up(REFILL_CYCLES))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index e54996ef..0c513849 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -19,7 +19,7 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) require((divisor & (divisor-1)) == 0) require(hold_cycles < divisor/2 && hold_cycles >= 1) - val cnt = Reg() { UFix(width = log2up(divisor)) } + val cnt = Reg() { UFix(width = log2Up(divisor)) } cnt := cnt + UFix(1) val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time val in_en = cnt === UFix(divisor/2-1) // rising edge @@ -46,5 +46,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2up(divisor)-1).toBool + io.clk_slow := cnt(log2Up(divisor)-1).toBool } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ab56d0f4..9791ea0f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -120,7 +120,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2up(ntiles))) + val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) @@ -128,9 +128,9 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_rep_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2up(REFILL_CYCLES)) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) io.busy := state != s_idle io.addr := addr_ @@ -415,7 +415,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2up(REFILL_CYCLES))) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io @@ -438,7 +438,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH is(s_abort_drain) { // raises x_init_data.ready below when(x_init_data.valid) { abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2up(REFILL_CYCLES))) { + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send } } From 0c6bade592ca5c6216413831d5464b5850e6ec58 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 096/688] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/uncore.scala | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 166b85705533ef7d6d37ea049617b211087f7b1e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 6 Jun 2012 18:22:56 -0700 Subject: [PATCH 097/688] ioDecoupled -> FIFOIO, ioPipe -> PipeIO --- uncore/memserdes.scala | 4 ++-- uncore/slowio.scala | 8 ++++---- uncore/uncore.scala | 44 +++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 8dc6a8ae..e20f1ec3 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -7,8 +7,8 @@ import scala.math._ class ioMemSerialized extends Bundle { - val req = (new ioDecoupled) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new ioPipe) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } + val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip } class MemSerdes extends Component diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 0c513849..6cf5a3d9 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -6,11 +6,11 @@ import Constants._ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component { val io = new Bundle { - val out_fast = new ioDecoupled()(data).flip - val out_slow = new ioDecoupled()(data) + val out_fast = new FIFOIO()(data).flip + val out_slow = new FIFOIO()(data) - val in_fast = new ioDecoupled()(data) - val in_slow = new ioDecoupled()(data).flip + val in_fast = new FIFOIO()(data) + val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 9791ea0f..6239981f 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -21,9 +21,9 @@ class MemResp () extends MemData class ioMem() extends Bundle { - val req_cmd = (new ioDecoupled) { new MemReqCmd() } - val req_data = (new ioDecoupled) { new MemData() } - val resp = (new ioPipe) { new MemResp() }.flip + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip } class TrackerProbeData extends Bundle { @@ -40,34 +40,34 @@ class TrackerDependency extends Bundle { } class ioTileLink extends Bundle { - val xact_init = (new ioDecoupled) { new TransactionInit } - val xact_init_data = (new ioDecoupled) { new TransactionInitData } - val xact_abort = (new ioDecoupled) { new TransactionAbort }.flip - val probe_req = (new ioDecoupled) { new ProbeRequest }.flip - val probe_rep = (new ioDecoupled) { new ProbeReply } - val probe_rep_data = (new ioDecoupled) { new ProbeReplyData } - val xact_rep = (new ioPipe) { new TransactionReply }.flip - val xact_finish = (new ioDecoupled) { new TransactionFinish } + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val io = new Bundle { - val alloc_req = (new ioDecoupled) { new TrackerAllocReq }.flip - val p_data = (new ioPipe) { new TrackerProbeData }.flip + val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip + val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(ntiles, INPUT) val p_req_cnt_inc = Bits(ntiles, INPUT) - val p_rep_data = (new ioPipe) { new ProbeReplyData }.flip - val x_init_data = (new ioPipe) { new TransactionInitData }.flip + val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip + val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new ioPipe) { new TrackerDependency }.flip - val x_init_data_dep = (new ioPipe) { new TrackerDependency }.flip + val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip + val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - val mem_req_cmd = (new ioDecoupled) { new MemReqCmd } - val mem_req_data = (new ioDecoupled) { new MemData } + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new ioDecoupled) { new ProbeRequest } + val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) @@ -85,7 +85,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: ioDecoupled[MemReqCmd], req_data: ioDecoupled[MemData], lock: Bool, data: ioPipe[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) req_data.valid := data.valid && at_front_of_dep_queue @@ -106,7 +106,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } - def doMemReqRead(req_cmd: ioDecoupled[MemReqCmd], trigger: Bool) { + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { req_cmd.valid := Bool(true) req_cmd.bits.rw := Bool(false) when(req_cmd.ready) { From 66cf6902619c60f9b181c52653b14e1e70b4375a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 098/688] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 1ebfeeca8a0024e2dc2281dc926d1da6ba439e69 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 Jul 2012 05:23:29 -0700 Subject: [PATCH 099/688] add L2$ It still has performance bugs but no correctness bugs AFAIK. --- uncore/llc.scala | 392 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 uncore/llc.scala diff --git a/uncore/llc.scala b/uncore/llc.scala new file mode 100644 index 00000000..71dfa536 --- /dev/null +++ b/uncore/llc.scala @@ -0,0 +1,392 @@ +package rocket + +import Chisel._ +import Node._ +import Constants._ + +class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +{ + val io = new Bundle { + val addr = UFix(log2Up(n), INPUT) + val en = Bool(INPUT) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + val rdata = gen.asOutput + } + require(readLatency >= 0 && readLatency <= 2) + val data = gen + val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 + val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) + val rdataDeep = Vec(nDeep) { Bits() } + val rdataSel = Vec(nDeep) { Bool() } + val cond = Vec(nDeep) { Bool() } + val ren = Vec(nDeep) { Bool() } + val reg_ren = Vec(nDeep) { Reg() { Bool() } } + val reg2_ren = Vec(nDeep) { Reg() { Bool() } } + val reg_raddr = Vec(nDeep) { Reg() { UFix() } } + val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } + val renOut = Vec(nDeep) { Bool() } + val raddrOut = Vec(nDeep) { UFix() } + val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } + val wdata = io.wdata.toBits + val wmask = io.wmask.toBits + for (i <- 0 until nDeep) { + cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) + ren(i) := cond(i) && !io.rw + reg_ren(i) := ren(i) + reg2_ren(i) := reg_ren(i) + when (ren(i)) { reg_raddr(i) := io.addr } + when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } + renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) + raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val dout1 = if (readLatency > 0) Reg() { Bits() } else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + when (cond(i)) { + when (io.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (readLatency > 0) dout1 := mem(idx) } + } + + if (readLatency == 0) { + dout = mem(idx) + } else if (readLatency == 1) { + dout = dout1 + } else { + val dout2 = Reg() { Bits() } + when (reg_ren(i)) { dout2 := dout1 } + dout = dout2 + } + + rdata(i)(j) := dout + } + val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := renOut(i) + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} + +class LLCDataReq(ways: Int) extends MemReqCmd +{ + val way = UFix(width = log2Up(ways)) + val isWriteback = Bool() + + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +{ + val io = new Bundle { + val cpu = (new FIFOIO) { new MemReqCmd }.flip + val repl_way = UFix(log2Up(ways), INPUT) + val repl_dirty = Bool(INPUT) + val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val data = (new FIFOIO) { new LLCDataReq(ways) } + val tag = (new PipeIO) { new Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + } } + val mem = new ioMem + val mem_resp_set = UFix(log2Up(sets), OUTPUT) + val mem_resp_way = UFix(log2Up(ways), OUTPUT) + } + + class MSHR extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val way = UFix(width = log2Up(ways)) + val tag = io.cpu.bits.tag.clone + val refilled = Bool() + val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val requested = Bool() + val old_dirty = Bool() + val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + + override def clone = new MSHR().asInstanceOf[this.type] + } + + val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val validBits = valid.toBits + val freeId = PriorityEncoder(~validBits) + val mshr = Vec(outstanding) { Reg() { new MSHR } } + when (io.cpu.valid && io.cpu.ready) { + valid(freeId) := Bool(true) + mshr(freeId).addr := io.cpu.bits.addr + mshr(freeId).tag := io.cpu.bits.tag + mshr(freeId).way := io.repl_way + mshr(freeId).old_dirty := io.repl_dirty + mshr(freeId).old_tag := io.repl_tag + mshr(freeId).requested := Bool(false) + mshr(freeId).refillCount := UFix(0) + mshr(freeId).refilled := Bool(false) + } + + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val request = requests.orR + val requestId = PriorityEncoder(requests) + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } + + val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) + val refillCount = mshr(refillId).refillCount + when (io.mem.resp.valid) { + mshr(refillId).refillCount := refillCount + UFix(1) + when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + } + + val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) + val replay = replays.orR + val replayId = PriorityEncoder(replays) + when (replay && io.data.ready) { valid(replayId) := Bool(false) } + + val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) + val writeback = writebacks.orR + val writebackId = PriorityEncoder(writebacks) + when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + + val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) + io.cpu.ready := !conflicts.orR && !validBits.andR + + io.data.valid := replay || writeback + io.data.bits.rw := Bool(false) + io.data.bits.tag := mshr(replayId).tag + io.data.bits.isWriteback := Bool(true) + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.way := mshr(writebackId).way + when (replay) { + io.data.bits.isWriteback := Bool(false) + io.data.bits.addr := mshr(replayId).addr + io.data.bits.way := mshr(replayId).way + } + io.tag.valid := replay && io.data.ready + io.tag.bits.addr := io.data.bits.addr + io.tag.bits.way := io.data.bits.way + + io.mem.req_cmd.valid := request + io.mem.req_cmd.bits.rw := Bool(false) + io.mem.req_cmd.bits.addr := mshr(requestId).addr + io.mem.req_cmd.bits.tag := requestId + io.mem_resp_set := mshr(refillId).addr + io.mem_resp_way := mshr(refillId).way +} + +class LLCWriteback(requestors: Int) extends Component +{ + val io = new Bundle { + val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val mem = new ioMem + } + + val valid = Reg(resetVal = Bool(false)) + val who = Reg() { UFix() } + val addr = Reg() { UFix() } + val cmd_sent = Reg() { Bool() } + val data_sent = Reg() { Bool() } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + var anyReq = Bool(false) + for (i <- 0 until requestors) { + io.req(i).ready := !valid && !anyReq + io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + anyReq = anyReq || io.req(i).valid + } + + val nextWho = PriorityEncoder(io.req.map(_.valid)) + when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { + valid := Bool(true) + cmd_sent := Bool(false) + data_sent := Bool(false) + who := nextWho + addr := io.req(nextWho).bits + } + + when (io.mem.req_data.valid && io.mem.req_data.ready) { + count := count + UFix(1) + when (count === UFix(REFILL_CYCLES-1)) { + data_sent := Bool(true) + when (cmd_sent) { valid := Bool(false) } + } + } + when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } + when (valid && cmd_sent && data_sent) { valid := Bool(false) } + + io.mem.req_cmd.valid := valid && !cmd_sent + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := Bool(true) + + io.mem.req_data.valid := valid && !data_sent && io.data(who).valid + io.mem.req_data.bits := io.data(who).bits +} + +class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val req = (new FIFOIO) { new LLCDataReq(ways) }.flip + val req_data = (new FIFOIO) { new MemData }.flip + val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback_data = (new FIFOIO) { new MemData } + val resp = (new PipeIO) { new MemResp } + val mem_resp = (new PipeIO) { new MemResp }.flip + val mem_resp_set = UFix(log2Up(sets), INPUT) + val mem_resp_way = UFix(log2Up(ways), INPUT) + } + + val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + class QEntry extends MemResp { + val isWriteback = Bool() + override def clone = new QEntry().asInstanceOf[this.type] + } + val q = (new queue(4)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries - 3) + val valid = Reg(resetVal = Bool(false)) + val req = Reg() { io.req.bits.clone } + val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + + when (data.io.en && !io.mem_resp.valid) { + count := count + UFix(1) + when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + } + when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } + when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + + data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := io.req.bits.rw + data.io.wdata := io.req_data.bits.data + data.io.wmask := Fix(-1, io.req_data.bits.data.width) + when (valid) { + data.io.en := Mux(req.rw, io.req_data.valid, qReady) + data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.rw := req.rw + } + when (io.mem_resp.valid) { + data.io.en := Bool(true) + data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.rw := Bool(true) + data.io.wdata := io.mem_resp.bits.data + } + + q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) + q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + q.io.enq.bits.data := data.io.rdata + q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) + + io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) + io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) + + io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback + io.writeback.bits := io.req.bits.addr + + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback + io.resp.bits := q.io.deq.bits + io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback + io.writeback_data.bits := q.io.deq.bits +} + +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMem + } + + val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val metaWidth = tagWidth + 2 // valid + dirty + + val memCmdArb = (new Arbiter(2)) { new MemReqCmd } + val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } + val mshr = new LLCMSHRFile(sets, ways, outstanding) + val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(sets, ways, dataLeaf) + val writeback = new LLCWriteback(2) + + val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initialize = !initCount(log2Up(sets)) + when (initialize) { initCount := initCount + UFix(1) } + + val stall_s1 = Bool() + val replay_s1 = Reg(resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + replay_s1 := s1_valid && stall_s1 + val s1 = Reg() { new MemReqCmd } + when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) + tags.io.rw := initialize || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) + + val stall_s2 = Bool() + val s2_valid = Reg(resetVal = Bool(false)) + s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 + val s2 = Reg() { new MemReqCmd } + when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } + val s2_tags = Vec(ways) { Bits(width = metaWidth) } + for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit = s2_hits.reduceLeft(_||_) + stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) + val repl_tag = s2_tags(repl_way).toUFix + + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.bits := s2 + mshr.io.repl_way := repl_way + mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_tag := repl_tag + mshr.io.mem.resp := io.mem.resp + + data.io.req <> dataArb.io.out + data.io.mem_resp := io.mem.resp + data.io.mem_resp_set := mshr.io.mem_resp_set + data.io.mem_resp_way := mshr.io.mem_resp_way + data.io.req_data.bits := io.cpu.req_data.bits + + writeback.io.req(0) <> data.io.writeback + writeback.io.data(0) <> data.io.writeback_data + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).bits := s2.addr + writeback.io.data(1).valid := io.cpu.req_data.valid + writeback.io.data(1).bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + + memCmdArb.io.in(0) <> mshr.io.mem.req_cmd + memCmdArb.io.in(1) <> writeback.io.mem.req_cmd + + dataArb.io.in(0) <> mshr.io.data + dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).bits := s2 + dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.isWriteback := Bool(false) + + stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + + io.cpu.resp <> data.io.resp + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.mem.req_cmd <> memCmdArb.io.out + io.mem.req_data <> writeback.io.mem.req_data +} From 0aa33bf9095ddc6b8980a7f12046fbf42bfbf230 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 100/688] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 62a3ea411385f65e8984acc14198b143f0606be1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 11 Jul 2012 17:56:39 -0700 Subject: [PATCH 101/688] fix some LLC bugs --- uncore/llc.scala | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 71dfa536..e6f39c4a 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -103,7 +103,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val repl_dirty = Bool(INPUT) val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new PipeIO) { new Bundle { + val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } @@ -156,7 +156,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) val replay = replays.orR val replayId = PriorityEncoder(replays) - when (replay && io.data.ready) { valid(replayId) := Bool(false) } + when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR @@ -166,7 +166,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay || writeback + io.data.valid := replay && io.tag.ready || writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) @@ -318,7 +318,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 2, tagLeaf)(Bits(width = metaWidth*ways)) + val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) val data = new LLCData(sets, ways, dataLeaf) val writeback = new LLCWriteback(2) @@ -333,31 +333,38 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr))(log2Up(sets)-1,0)) - tags.io.rw := initialize || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(Bool(false), Bool(true), mshr.io.tag.bits.addr(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(mshr.io.tag.bits.way))) - val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 val s2 = Reg() { new MemReqCmd } - when (s1_valid && !stall_s1 && !replay_s1) { s2 := s1 } - val s2_tags = Vec(ways) { Bits(width = metaWidth) } - for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + when (s1_valid && !stall_s1 && !replay_s1) { + s2 := s1 + for (i <- 0 until ways) + s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) + } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hit_way = OHToUFix(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) - stall_s1 := initialize || mshr.io.tag.valid || s2_valid && !s2_hit || stall_s2 + val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix + val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty + stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + + tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth).toBool + mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp + mshr.io.tag.ready := !setDirty data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -379,7 +386,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da dataArb.io.in(0) <> mshr.io.data dataArb.io.in(1).valid := s2_valid && s2_hit dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := OHToUFix(s2_hits) + dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) From 18bc14058b3e21ef58b45e59039fd65733f2fd83 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 102/688] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From a79747a06221802ac05c65144d3b8b42d5b7adb2 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 12 Jul 2012 18:12:49 -0700 Subject: [PATCH 103/688] INPUT/OUTPUT orderring swapped --- uncore/llc.scala | 14 +++++++------- uncore/uncore.scala | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index e6f39c4a..5f5007d1 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -7,7 +7,7 @@ import Constants._ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { val io = new Bundle { - val addr = UFix(log2Up(n), INPUT) + val addr = UFix(INPUT, log2Up(n)) val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput @@ -99,17 +99,17 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component { val io = new Bundle { val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(log2Up(ways), INPUT) + val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(PADDR_BITS - OFFSET_BITS - log2Up(sets), INPUT) + val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMem - val mem_resp_set = UFix(log2Up(sets), OUTPUT) - val mem_resp_way = UFix(log2Up(ways), OUTPUT) + val mem_resp_set = UFix(OUTPUT, log2Up(sets)) + val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { @@ -247,8 +247,8 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val writeback_data = (new FIFOIO) { new MemData } val resp = (new PipeIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(log2Up(sets), INPUT) - val mem_resp_way = UFix(log2Up(ways), INPUT) + val mem_resp_set = UFix(INPUT, log2Up(sets)) + val mem_resp_way = UFix(INPUT, log2Up(ways)) } val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6239981f..7e34492c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -56,8 +56,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(ntiles, INPUT) - val p_req_cnt_inc = Bits(ntiles, INPUT) + val p_rep_cnt_dec = Bits(INPUT, ntiles) + val p_req_cnt_inc = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -69,19 +69,19 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_req_lock = Bool(OUTPUT) val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) - val addr = Bits(PADDR_BITS - OFFSET_BITS, OUTPUT) - val init_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val p_rep_tile_id = Bits(TILE_ID_BITS, OUTPUT) - val tile_xact_id = Bits(TILE_XACT_ID_BITS, OUTPUT) - val sharer_count = Bits(TILE_ID_BITS+1, OUTPUT) - val x_type = Bits(X_INIT_TYPE_MAX_BITS, OUTPUT) - val push_p_req = Bits(ntiles, OUTPUT) - val pop_p_rep = Bits(ntiles, OUTPUT) - val pop_p_rep_data = Bits(ntiles, OUTPUT) - val pop_p_rep_dep = Bits(ntiles, OUTPUT) - val pop_x_init = Bits(ntiles, OUTPUT) - val pop_x_init_data = Bits(ntiles, OUTPUT) - val pop_x_init_dep = Bits(ntiles, OUTPUT) + val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) + val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val push_p_req = Bits(OUTPUT, ntiles) + val pop_p_rep = Bits(OUTPUT, ntiles) + val pop_p_rep_data = Bits(OUTPUT, ntiles) + val pop_p_rep_dep = Bits(OUTPUT, ntiles) + val pop_x_init = Bits(OUTPUT, ntiles) + val pop_x_init_data = Bits(OUTPUT, ntiles) + val pop_x_init_dep = Bits(OUTPUT, ntiles) val send_x_rep_ack = Bool(OUTPUT) } From 0258dfb23f28e109dbcd7032e1f5f64aa03d2c5f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 104/688] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From d01e70c6720310386c7c3f8708d24db7798da72d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 17 Jul 2012 22:55:00 -0700 Subject: [PATCH 105/688] decouple all interfaces between tile and top also, add an "incoherent" bit to tilelink to indicate no probes needed --- uncore/llc.scala | 10 +++++----- uncore/uncore.scala | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 5f5007d1..1c9d49b8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -107,7 +107,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val way = UFix(width = log2Up(ways)) } } - val mem = new ioMem + val mem = new ioMemPipe val mem_resp_set = UFix(OUTPUT, log2Up(sets)) val mem_resp_way = UFix(OUTPUT, log2Up(ways)) } @@ -194,7 +194,7 @@ class LLCWriteback(requestors: Int) extends Component val io = new Bundle { val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMem + val mem = new ioMemPipe } val valid = Reg(resetVal = Bool(false)) @@ -245,7 +245,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val req_data = (new FIFOIO) { new MemData }.flip val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } val writeback_data = (new FIFOIO) { new MemData } - val resp = (new PipeIO) { new MemResp } + val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip val mem_resp_set = UFix(INPUT, log2Up(sets)) val mem_resp_way = UFix(INPUT, log2Up(ways)) @@ -298,7 +298,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback io.writeback.bits := io.req.bits.addr - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, Bool(true)) + q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback io.resp.bits := q.io.deq.bits io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback @@ -309,7 +309,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da { val io = new Bundle { val cpu = new ioMem().flip - val mem = new ioMem + val mem = new ioMemPipe } val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7e34492c..ead75252 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -20,6 +20,13 @@ class MemResp () extends MemData } class ioMem() extends Bundle +{ + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe() extends Bundle { val req_cmd = (new FIFOIO) { new MemReqCmd() } val req_data = (new FIFOIO) { new MemData() } @@ -46,8 +53,9 @@ class ioTileLink extends Bundle { val probe_req = (new FIFOIO) { new ProbeRequest }.flip val probe_rep = (new FIFOIO) { new ProbeReply } val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new PipeIO) { new TransactionReply }.flip + val xact_rep = (new FIFOIO) { new TransactionReply }.flip val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) } class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { @@ -58,6 +66,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val xact_finish = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, ntiles) val p_req_cnt_inc = Bits(INPUT, ntiles) + val tile_incoherent = Bits(INPUT, ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -169,7 +178,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~( UFix(1) << io.alloc_req.bits.tile_id ) //TODO: Broadcast only + val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only p_req_flags := p_req_initial_flags(ntiles-1,0) mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -310,6 +319,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) @@ -360,8 +370,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } } - // If there were a ready signal due to e.g. intervening network use: - //io.mem.resp.ready := io.tiles(init_tile_id_arr.read(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests From c6ac83658130d6c8c80f234aaad939eb80f80875 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 106/688] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From df8aff090672eb388bf165fc85c45e5ebd7fd0fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 22 Jul 2012 21:05:52 -0700 Subject: [PATCH 107/688] don't dequeue probe queue during reset --- uncore/uncore.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index ead75252..6ea359ce 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -140,6 +140,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = ntiles) + p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -178,15 +180,14 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - val p_req_initial_flags = ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id)) //TODO: Broadcast only - p_req_flags := p_req_initial_flags(ntiles-1,0) + p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id if(ntiles > 1) { - p_rep_count := UFix(ntiles-1) - state := Mux(p_req_initial_flags(ntiles-1,0).orR, s_probe, s_mem) + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } From d0e12c13f669ff1c1d3c3562dbcfe3bd98366b2d Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 108/688] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 77364057267769be338f82a13e0e2c9a223afbb5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 23 Jul 2012 20:56:55 -0700 Subject: [PATCH 109/688] fix bug in coherence hub, respect xact_rep.ready --- uncore/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 6ea359ce..2035453c 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -367,7 +367,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH rep.bits.global_xact_id := ack_idx when (UFix(j) === init_tile_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := Bool(true) + sent_x_rep_ack_arr(ack_idx) := rep.ready } } } From 1405718ca8e1a79ffc39c2727ef17a037fcef93f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 110/688] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 1ae3091261f64e1f8e6de7f9e9e2afc4b3c855ce Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 Jul 2012 00:03:55 -0700 Subject: [PATCH 111/688] memory system bug fixes --- uncore/llc.scala | 4 ++-- uncore/uncore.scala | 33 ++++++++++++++++++--------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 1c9d49b8..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -378,7 +378,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && !writeback.io.data(1).ready + data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd @@ -393,7 +393,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 2035453c..5c58f151 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -95,21 +95,25 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.valid := !cmd_sent && data.valid && at_front_of_dep_queue req_cmd.bits.rw := Bool(true) - req_data.valid := data.valid && at_front_of_dep_queue req_data.bits := data.bits - lock := data.valid && at_front_of_dep_queue when(req_cmd.ready && req_cmd.valid) { cmd_sent := Bool(true) } - when(req_data.ready && at_front_of_dep_queue) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt_next === UFix(0)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready + lock := Bool(true) + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + pop_data := UFix(1) << tile_id + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := UFix(1) << tile_id + trigger := Bool(false) + } + } } } } @@ -141,7 +145,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0))) //TODO: Broadcast only + p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ @@ -376,16 +380,15 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } for( i <- 0 until NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_cmd_arb.io.lock(i) <> trackerList(i).io.mem_req_lock mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock } - io.mem.req_cmd <> mem_req_cmd_arb.io.out - io.mem.req_data <> mem_req_data_arb.io.out + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data for( j <- 0 until ntiles ) { From 0a2d284d24cda2bd6559975d6ebc823bbea1079c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 112/688] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 465f2efca71727aa952bf157f22ca27ba72d457f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 27 Jul 2012 18:44:17 -0700 Subject: [PATCH 113/688] add reset pin to llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..739fd791 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) { val io = new Bundle { val cpu = new ioMem().flip From 914b6b622d98646a51437d972f823c4ccab1a27f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 114/688] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 4d4e28c1387b309e9247ba3da94a56b7e5da4dcf Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 28 Jul 2012 21:14:33 -0700 Subject: [PATCH 115/688] remove reset pin on llc --- uncore/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 739fd791..d95aeff7 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -305,7 +305,7 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits], resetSignal: Bool = null) extends Component(resetSignal) +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { val cpu = new ioMem().flip From 85dc34df803dae3d715531a2a85a8c5793acbc62 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 116/688] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 8db233c9b721abf66a47caf2d79c4e5c97c9a8fd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 30 Jul 2012 20:12:11 -0700 Subject: [PATCH 117/688] further pipeline the LLC --- uncore/llc.scala | 84 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 22 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index d95aeff7..89fc5e2f 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -14,7 +14,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val wmask = gen.asInput val rdata = gen.asOutput } - require(readLatency >= 0 && readLatency <= 2) val data = gen val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 @@ -28,9 +27,6 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex val cond = Vec(nDeep) { Bool() } val ren = Vec(nDeep) { Bool() } val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val reg2_ren = Vec(nDeep) { Reg() { Bool() } } - val reg_raddr = Vec(nDeep) { Reg() { UFix() } } - val reg2_raddr = Vec(nDeep) { Reg() { UFix() } } val renOut = Vec(nDeep) { Bool() } val raddrOut = Vec(nDeep) { UFix() } val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } @@ -40,11 +36,14 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) ren(i) := cond(i) && !io.rw reg_ren(i) := ren(i) - reg2_ren(i) := reg_ren(i) - when (ren(i)) { reg_raddr(i) := io.addr } - when (reg_ren(i)) { reg2_raddr(i) := reg_raddr(i) } - renOut(i) := (if (readLatency > 1) reg2_ren(i) else if (readLatency > 0) reg_ren(i) else ren(i)) - raddrOut(i) := (if (readLatency > 1) reg2_raddr(i) else if (readLatency > 0) reg_raddr(i) else io.addr) + + renOut(i) := ren(i) + raddrOut(i) := io.addr + if (readLatency > 0) { + val r = Pipe(ren(i), io.addr, readLatency) + renOut(i) := r.valid + raddrOut(i) := r.bits + } for (j <- 0 until nWide) { val mem = leaf.clone @@ -64,11 +63,8 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex dout = mem(idx) } else if (readLatency == 1) { dout = dout1 - } else { - val dout2 = Reg() { Bits() } - when (reg_ren(i)) { dout2 := dout1 } - dout = dout2 - } + } else + dout = Pipe(reg_ren(i), dout1, readLatency-1).bits rdata(i)(j) := dout } @@ -238,7 +234,7 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component { val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip @@ -251,13 +247,13 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 2, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(4)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries - 3) + val q = (new queue(latency+2)) { new QEntry } + val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) @@ -287,10 +283,11 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component data.io.wdata := io.mem_resp.bits.data } - q.io.enq.valid := Reg(Reg(data.io.en && !data.io.rw, resetVal = Bool(false)), resetVal = Bool(false)) - q.io.enq.bits.tag := Reg(Reg(Mux(valid, req.tag, io.req.bits.tag))) + val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + q.io.enq.valid := tagPipe.valid + q.io.enq.bits.tag := tagPipe.bits + q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid q.io.enq.bits.data := data.io.rdata - q.io.enq.bits.isWriteback := Reg(Reg(Mux(valid, req.isWriteback, io.req.bits.isWriteback))) io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) @@ -305,6 +302,49 @@ class LLCData(sets: Int, ways: Int, leaf: Mem[Bits]) extends Component io.writeback_data.bits := q.io.deq.bits } +class MemReqArb(n: Int) extends Component // UNTESTED +{ + val io = new Bundle { + val cpu = Vec(n) { new ioMem().flip } + val mem = new ioMem + } + + val lock = Reg(resetVal = Bool(false)) + val locker = Reg() { UFix() } + + val arb = new RRArbiter(n)(new MemReqCmd) + val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) + val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + for (i <- 0 until n) { + val me = UFix(i, log2Up(n)) + arb.io.in(i).valid := io.cpu(i).req_cmd.valid + arb.io.in(i).bits := io.cpu(i).req_cmd.bits + arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) + io.cpu(i).req_cmd.ready := arb.io.in(i).ready + io.cpu(i).req_data.ready := Bool(false) + + val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock + val haveLock = lock && locker === me + when (getLock) { + lock := Bool(true) + locker := UFix(i) + } + when (getLock || haveLock) { + io.cpu(i).req_data.ready := io.mem.req_data.ready + io.mem.req_data.valid := Bool(true) + io.mem.req_data.bits := io.cpu(i).req_data.bits + } + + io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me + io.cpu(i).resp.bits := io.mem.resp.bits + io.cpu(i).resp.bits.tag := respTag + } + io.mem.resp.ready := io.cpu(respWho).resp.ready + + val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + when (unlock) { lock := Bool(false) } +} + class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component { val io = new Bundle { @@ -319,7 +359,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(sets, ways, dataLeaf) + val data = new LLCData(3, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) From 7b9cfd0b90b7f76a5bda02393af873a8e9e9868b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 118/688] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 7a75334bb97d103f7c4ff1c3b936434c5683bebf Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 Jul 2012 17:44:53 -0700 Subject: [PATCH 119/688] pipeline LLC further --- uncore/llc.scala | 104 ++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 56 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 89fc5e2f..546da774 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -4,14 +4,17 @@ import Chisel._ import Node._ import Constants._ -class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { - val io = new Bundle { + class Inputs extends Bundle { val addr = UFix(INPUT, log2Up(n)) - val en = Bool(INPUT) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = new PipeIO()(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,63 +24,52 @@ class BigMem[T <: Data](n: Int, readLatency: Int, leaf: Mem[Bits])(gen: => T) ex if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val idx = io.addr(log2Up(n/nDeep/colMux)-1, 0) val rdataDeep = Vec(nDeep) { Bits() } val rdataSel = Vec(nDeep) { Bool() } - val cond = Vec(nDeep) { Bool() } - val ren = Vec(nDeep) { Bool() } - val reg_ren = Vec(nDeep) { Reg() { Bool() } } - val renOut = Vec(nDeep) { Bool() } - val raddrOut = Vec(nDeep) { UFix() } - val rdata = Vec(nDeep) { Vec(nWide) { Bits() } } - val wdata = io.wdata.toBits - val wmask = io.wmask.toBits for (i <- 0 until nDeep) { - cond(i) := (if (nDeep == 1) io.en else io.en && UFix(i) === io.addr(log2Up(n)-1, log2Up(n/nDeep))) - ren(i) := cond(i) && !io.rw - reg_ren(i) := ren(i) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(ren) + val rdata = Vec(nWide) { Bits() } - renOut(i) := ren(i) - raddrOut(i) := io.addr - if (readLatency > 0) { - val r = Pipe(ren(i), io.addr, readLatency) - renOut(i) := r.valid - raddrOut(i) := r.bits - } + val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (readLatency > 0) Reg() { Bits() } else null + val dout1 = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(io.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (cond(i)) { - when (io.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (readLatency > 0) dout1 := mem(idx) } + when (in.valid) { + when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + .otherwise { if (postLatency > 0) dout1 := mem(idx) } } - if (readLatency == 0) { + if (postLatency == 0) { dout = mem(idx) - } else if (readLatency == 1) { + } else if (postLatency == 1) { dout = dout1 } else - dout = Pipe(reg_ren(i), dout1, readLatency-1).bits + dout = Pipe(reg_ren, dout1, postLatency-1).bits - rdata(i)(j) := dout + rdata(j) := dout } - val rdataWide = rdata(i).reduceLeft((x, y) => Cat(y, x)) + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) var colMuxOut = rdataWide if (colMux > 1) { val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(raddrOut(i)(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } rdataDeep(i) := colMuxOut - rdataSel(i) := renOut(i) + rdataSel(i) := r.valid } io.rdata := Mux1H(rdataSel, rdataDeep) @@ -247,7 +239,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val mem_resp_way = UFix(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, latency, leaf)(Bits(width = MEM_DATA_BITS)) + val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -259,31 +251,31 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - when (data.io.en && !io.mem_resp.valid) { + when (data.io.in.valid && !io.mem_resp.valid) { count := count + UFix(1) when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - data.io.en := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := io.req.bits.rw - data.io.wdata := io.req_data.bits.data - data.io.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := io.req.bits.rw + data.io.in.bits.wdata := io.req_data.bits.data + data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) when (valid) { - data.io.en := Mux(req.rw, io.req_data.valid, qReady) - data.io.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.rw := req.rw + data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { - data.io.en := Bool(true) - data.io.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.rw := Bool(true) - data.io.wdata := io.mem_resp.bits.data + data.io.in.valid := Bool(true) + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.rw := Bool(true) + data.io.in.bits.wdata := io.mem_resp.bits.data } - val tagPipe = Pipe(data.io.en && !data.io.rw, Mux(valid, req.tag, io.req.bits.tag), latency) + val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) q.io.enq.valid := tagPipe.valid q.io.enq.bits.tag := tagPipe.bits q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid @@ -358,8 +350,8 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val memCmdArb = (new Arbiter(2)) { new MemReqCmd } val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(3, sets, ways, dataLeaf) + val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) + val data = new LLCData(4, sets, ways, dataLeaf) val writeback = new LLCWriteback(2) val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) @@ -392,11 +384,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 - tags.io.en := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) + tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 From 92b7504c9a02c4bd1c067bc71eb1223fd212c05e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 120/688] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From e346f21725e3de2ac2939db11e644f29e9e3ac3b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 18:59:37 -0700 Subject: [PATCH 121/688] fix control bug in LLC structural hazard on tag ram caused deadlock --- uncore/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 546da774..eece03ed 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -154,13 +154,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR - io.data.valid := replay && io.tag.ready || writeback + io.data.valid := writeback io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix io.data.bits.way := mshr(writebackId).way when (replay) { + io.data.valid := io.tag.ready io.data.bits.isWriteback := Bool(false) io.data.bits.addr := mshr(replayId).addr io.data.bits.way := mshr(replayId).way From 962423d2d150944c6e37e5ccf0b08aac2a05e55f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 122/688] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 875f3622af6520e2985312b07aa0d2ca65732403 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Aug 2012 19:00:03 -0700 Subject: [PATCH 123/688] fix deadlock in coherence hub --- uncore/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 5c58f151..a1726242 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -101,8 +101,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready - lock := Bool(true) + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { From 115c25c34b2f5fe0524b6108e8088c912d5cb428 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 124/688] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From 17dc2075dd2c73b9446eb669a41d45aefd718ff1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 6 Aug 2012 17:10:04 -0700 Subject: [PATCH 125/688] fix some LLC control bugs --- uncore/llc.scala | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index eece03ed..64d48ce8 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -383,15 +383,20 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || mshr.io.tag.valid || setDirty || s2_valid && !s2_hit || stall_s2 + stall_s1 := initialize || stall_s2 - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(setDirty, s2.addr, Mux(mshr.io.tag.valid, mshr.io.tag.bits.addr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)))(log2Up(sets)-1,0)) - tags.io.in.bits.rw := initialize || setDirty || mshr.io.tag.valid - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)))) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way)))) + val tag_we = setDirty || mshr.io.tag.valid + val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) + val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) + tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we + tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) + tags.io.in.bits.rw := initialize || tag_we + tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) + when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR @@ -404,29 +409,29 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da data.io.mem_resp_set := mshr.io.mem_resp_set data.io.mem_resp_way := mshr.io.mem_resp_way data.io.req_data.bits := io.cpu.req_data.bits + data.io.req_data.valid := io.cpu.req_data.valid writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid && writeback.io.req(1).ready memCmdArb.io.in(0) <> mshr.io.mem.req_cmd memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit + dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !Mux(s2_hit, dataArb.io.in(1).ready, Mux(s2.rw, writeback.io.req(1).ready, mshr.io.cpu.ready)) + stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready && writeback.io.req(1).ready + io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } From aa7fd1f40b8fb86b48f4d753e3dc5f72cfa7aaaa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 126/688] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/uncore.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From 6546dc84e2e53f5ec4e5b261217981e179f67ce6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 8 Aug 2012 22:11:32 -0700 Subject: [PATCH 127/688] rename queue to Queue fixes build with case-insensitive file system --- uncore/llc.scala | 2 +- uncore/memserdes.scala | 2 +- uncore/slowio.scala | 4 ++-- uncore/uncore.scala | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/llc.scala b/uncore/llc.scala index 64d48ce8..a2a5d361 100644 --- a/uncore/llc.scala +++ b/uncore/llc.scala @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new queue(latency+2)) { new QEntry } + val q = (new Queue(latency+2)) { new QEntry } val qReady = q.io.count <= UFix(q.entries-latency-1) val valid = Reg(resetVal = Bool(false)) val req = Reg() { io.req.bits.clone } diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index e20f1ec3..cdd109c2 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -149,7 +149,7 @@ class MemDessert extends Component // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) - val dataq = (new queue(REFILL_CYCLES)) { new MemResp } + val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) diff --git a/uncore/slowio.scala b/uncore/slowio.scala index 6cf5a3d9..068e90c5 100644 --- a/uncore/slowio.scala +++ b/uncore/slowio.scala @@ -28,12 +28,12 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } - val fromhost_q = new queue(1)(data) + val fromhost_q = new Queue(1)(data) fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new queue(1)(data) + val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val diff --git a/uncore/uncore.scala b/uncore/uncore.scala index a1726242..7ffef155 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -336,8 +336,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH } } - val p_rep_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until ntiles ) { From fa8075570ab7a5f1fc28ca3c8695dcad5997ef36 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 27 Sep 2012 12:59:45 -0700 Subject: [PATCH 128/688] move srcs into src dir, factoring out uncore consts into consts --- uncore/{ => src}/coherence.scala | 2 +- uncore/src/consts.scala | 28 ++++++++++++++++++++++++++++ uncore/{ => src}/llc.scala | 2 +- uncore/{ => src}/uncore.scala | 2 +- 4 files changed, 31 insertions(+), 3 deletions(-) rename uncore/{ => src}/coherence.scala (99%) create mode 100644 uncore/src/consts.scala rename uncore/{ => src}/llc.scala (99%) rename uncore/{ => src}/uncore.scala (99%) diff --git a/uncore/coherence.scala b/uncore/src/coherence.scala similarity index 99% rename from uncore/coherence.scala rename to uncore/src/coherence.scala index 04843fec..e4f85436 100644 --- a/uncore/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala new file mode 100644 index 00000000..8ebf5c49 --- /dev/null +++ b/uncore/src/consts.scala @@ -0,0 +1,28 @@ +package uncore + +import Chisel._ + +object Constants +{ + + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; + + + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 + val TILE_ID_BITS = log2Up(NTILES)+1 + val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + +} diff --git a/uncore/llc.scala b/uncore/src/llc.scala similarity index 99% rename from uncore/llc.scala rename to uncore/src/llc.scala index a2a5d361..2bb2b62a 100644 --- a/uncore/llc.scala +++ b/uncore/src/llc.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ diff --git a/uncore/uncore.scala b/uncore/src/uncore.scala similarity index 99% rename from uncore/uncore.scala rename to uncore/src/uncore.scala index 7ffef155..56acece0 100644 --- a/uncore/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From da6ec486f1e2cfbb0eb501c29262e05d3e52014d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 27 Sep 2012 16:46:36 -0700 Subject: [PATCH 129/688] uncore and rocket changes for new xact types --- uncore/uncore.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/uncore/uncore.scala b/uncore/uncore.scala index 7ffef155..84b74366 100644 --- a/uncore/uncore.scala +++ b/uncore/uncore.scala @@ -3,14 +3,17 @@ package rocket import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From 2413763f3dfb21acb1c05b1d37ebc34692d1a54e Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:05:37 -0700 Subject: [PATCH 130/688] henry's uncore and rocket changes for new xact types --- uncore/src/coherence.scala | 194 +++++++++++++++++++++++++++---------- uncore/src/consts.scala | 60 +++++++++++- uncore/src/uncore.scala | 15 +-- 3 files changed, 205 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e4f85436..7fc03ece 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,10 +3,39 @@ package uncore import Chisel._ import Constants._ -class TransactionInit extends Bundle { +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} +class TransactionInit extends PhysicalAddress { val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val address = UFix(width = PADDR_BITS - OFFSET_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) } class TransactionInitData extends MemData @@ -15,10 +44,9 @@ class TransactionAbort extends Bundle { val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) } -class ProbeRequest extends Bundle { +class ProbeRequest extends PhysicalAddress { val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val address = Bits(width = PADDR_BITS - OFFSET_BITS) } class ProbeReply extends Bundle { @@ -76,6 +104,7 @@ abstract class CoherencePolicy { def messageHasData (init: TransactionInit): Bool def messageHasData (reply: TransactionReply): Bool def messageUpdatesDataArray (reply: TransactionReply): Bool + def messageIsUncached(init: TransactionInit): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def getTransactionReplyType(x_type: UFix, count: UFix): Bits @@ -86,8 +115,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getTransactionInitTypeOnUncachedRead(): UFix - def getTransactionInitTypeOnUncachedWrite(): UFix + def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -115,6 +147,8 @@ class ThreeStateIncoherence extends IncoherentPolicy { val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val uncachedTypeList = List() + val hasDataTypeList = List(xactInitWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -149,9 +183,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback - def messageHasData (init: TransactionInit): Bool = (init.x_type === xactInitWriteback) + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -159,10 +194,12 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(3){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: Nil = Enum(3){ UFix() } + val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -191,7 +228,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(incoming.x_type, tileInvalid, Array( xactReplyReadExclusive -> tileValid, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -201,8 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached @@ -227,15 +271,14 @@ class MICoherence extends CoherencePolicyWithUncached { (reply.p_type === probeRepInvalidateData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -243,7 +286,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, xactReplyReadUncached, Array( xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -251,7 +297,10 @@ class MICoherence extends CoherencePolicyWithUncached { MuxLookup(x_type, probeReqCopy, Array( xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -271,17 +320,19 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(4){ UFix() } + val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -311,7 +362,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -322,8 +376,12 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) @@ -357,15 +415,14 @@ class MEICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -374,7 +431,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> xactReplyReadExclusive, xactInitReadExclusiveDirty -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -383,7 +443,10 @@ class MEICoherence extends CoherencePolicyWithUncached { xactInitReadExclusiveClean -> probeReqInvalidate, xactInitReadExclusiveDirty -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } @@ -403,10 +466,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -419,7 +484,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -450,7 +515,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> tileExclusiveDirty, xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -461,8 +529,12 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -496,15 +568,14 @@ class MSICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -513,7 +584,10 @@ class MSICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -542,10 +616,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: Nil = Enum(4){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: Nil = Enum(5){ UFix() } + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -558,7 +634,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { val (read, write) = cpuCmdToRW(cmd) - (read && (outstanding.x_type === xactInitReadUncached || outstanding.x_type === xactInitWriteUncached)) || + (read && messageIsUncached(outstanding)) || (write && (outstanding.x_type != xactInitReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { @@ -589,7 +665,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), xactReplyReadExclusiveAck -> tileExclusiveDirty, xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid )) } def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { @@ -600,8 +679,12 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getTransactionInitTypeOnUncachedRead() = xactInitReadUncached - def getTransactionInitTypeOnUncachedWrite() = xactInitWriteUncached + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) @@ -635,15 +718,14 @@ class MESICoherence extends CoherencePolicyWithUncached { reply.p_type === probeRepDowngradeData || reply.p_type === probeRepCopyData) } - def messageHasData (init: TransactionInit): Bool = { - (init.x_type === xactInitWriteUncached) - } + def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck) + (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) } def messageUpdatesDataArray (reply: TransactionReply): Bool = { (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) } + def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -652,7 +734,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), xactInitReadExclusive -> xactReplyReadExclusive, xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached )) } @@ -661,7 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { xactInitReadShared -> probeReqDowngrade, xactInitReadExclusive -> probeReqInvalidate, xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate )) } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 8ebf5c49..95b07cc8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,15 +1,33 @@ package uncore import Chisel._ +import scala.math._ object Constants { + val NTILES = 1 + val HAVE_RVC = false + val HAVE_FPU = true + val HAVE_VEC = true - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - + val M_X = Bits("b????", 4); + val M_XRD = Bits("b0000", 4); // int load + val M_XWR = Bits("b0001", 4); // int store + val M_PFR = Bits("b0010", 4); // prefetch with intent to read + val M_PFW = Bits("b0011", 4); // prefetch with intent to write + val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines + val M_FENCE = Bits("b0101", 4); // memory fence + val M_INV = Bits("b0110", 4); // write back and invalidate line + val M_CLN = Bits("b0111", 4); // write back line + val M_XA_ADD = Bits("b1000", 4); + val M_XA_SWAP = Bits("b1001", 4); + val M_XA_AND = Bits("b1010", 4); + val M_XA_OR = Bits("b1011", 4); + val M_XA_MIN = Bits("b1100", 4); + val M_XA_MAX = Bits("b1101", 4); + val M_XA_MINU = Bits("b1110", 4); + val M_XA_MAXU = Bits("b1111", 4); + val PADDR_BITS = 40; val VADDR_BITS = 43; val PGIDX_BITS = 13; @@ -18,6 +36,25 @@ object Constants val ASID_BITS = 7; val PERM_BITS = 6; + // rocketNBDCache parameters + val DCACHE_PORTS = 3 + val CPU_DATA_BITS = 64; + val CPU_TAG_BITS = 9; + val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS + val OFFSET_BITS = 6; // log2(cache line size in bytes) + val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses + val NRPQ = 16; // number of secondary misses + val NSDQ = 17; // number of secondary stores/AMOs + val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) + val IDX_BITS = 7; + val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; + val NWAYS = 4 + require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); + + // coherence parameters + val ENABLE_SHARING = true + val ENABLE_CLEAN_EXCLUSIVE = true + val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 val TILE_ID_BITS = log2Up(NTILES)+1 @@ -25,4 +62,17 @@ object Constants val NGLOBAL_XACTS = 8 val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val X_INIT_TYPE_MAX_BITS = 2 + val X_INIT_WRITE_MASK_BITS = OFFSET_BITS + val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) + val X_INIT_ATOMIC_OP_BITS = 4 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 + + // external memory interface + val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS + } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 56acece0..1220a56f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,14 +3,17 @@ package uncore import Chisel._ import Constants._ +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + class MemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd() extends Bundle +class MemReqCmd() extends PhysicalAddress { val rw = Bool() - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) val tag = Bits(width = MEM_TAG_BITS) } @@ -165,7 +168,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.valid := Bool(false) io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.address := addr_ + io.probe_req.bits.addr := addr_ io.push_p_req := Bits(0, width = ntiles) io.pop_p_rep := Bits(0, width = ntiles) io.pop_p_rep_data := Bits(0, width = ntiles) @@ -178,7 +181,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.address + addr_ := io.alloc_req.bits.xact_init.addr x_type_ := io.alloc_req.bits.xact_init.x_type init_tile_id_ := io.alloc_req.bits.tile_id tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id @@ -272,7 +275,7 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.address + io.mem.req_cmd.bits.addr := x_init.bits.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep @@ -432,7 +435,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.address) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) } x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) From cf8f20584e21560b99970b8edffefa79569dd125 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 1 Oct 2012 16:08:41 -0700 Subject: [PATCH 131/688] factoring out uncore into separate uncore repo --- uncore/llc.scala | 437 ---------------------------------- uncore/memserdes.scala | 1 + uncore/uncore.scala | 519 ----------------------------------------- 3 files changed, 1 insertion(+), 956 deletions(-) delete mode 100644 uncore/llc.scala delete mode 100644 uncore/uncore.scala diff --git a/uncore/llc.scala b/uncore/llc.scala deleted file mode 100644 index a2a5d361..00000000 --- a/uncore/llc.scala +++ /dev/null @@ -1,437 +0,0 @@ -package rocket - -import Chisel._ -import Node._ -import Constants._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component -{ - class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = new PipeIO()(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) - when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = dout1 - } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UFix(width = log2Up(ways)) - val isWriteback = Bool() - - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component -{ - val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - } } - val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) - val request = requests.orR - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int) extends Component -{ - val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } - val mem = new ioMemPipe - } - - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) - } - - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int) extends Component // UNTESTED -{ - val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } - val mem = new ioMem - } - - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } - - val arb = new RRArbiter(n)(new MemReqCmd) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) - for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UFix(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 - when (unlock) { lock := Bool(false) } -} - -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component -{ - val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe - } - - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) - - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } - - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 - val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } - - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) - - io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data -} diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index cdd109c2..712dec16 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -4,6 +4,7 @@ import Chisel._ import Node._ import Constants._ import scala.math._ +import uncore._ class ioMemSerialized extends Bundle { diff --git a/uncore/uncore.scala b/uncore/uncore.scala deleted file mode 100644 index 84b74366..00000000 --- a/uncore/uncore.scala +++ /dev/null @@ -1,519 +0,0 @@ -package rocket - -import Chisel._ -import Constants._ - -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerAllocReq extends Bundle { - val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) -} - -class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) - val send_x_rep_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << tile_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } - val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) - val p_rep_tile_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only - - io.busy := state != s_idle - io.addr := addr_ - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only - io.x_type := x_type_ - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) - io.send_x_rep_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type - init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) - p_req_flags := p_req_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) - } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs - } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id - } - } - is(s_mem) { - when (p_rep_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, - p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) - } . elsewhen(x_init_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, - x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { - state := s_idle - } - } - } -} - -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { - val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip - val mem = new ioMem - } -} - -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) -{ - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr - io.mem.req_data <> io.tiles(0).xact_init_data - - val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) -{ - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type - sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until ntiles ) { - val finish = io.tiles(j).xact_finish - when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) - } - finish.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { - p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) - } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until ntiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) - } - } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) - } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below - abort_state_arr(j) := s_idle - } - } - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until ntiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready - } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req - } - -} From 916c1019af027e7d42133676644f70c4cbb9df06 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Tue, 9 Oct 2012 13:02:58 -0700 Subject: [PATCH 132/688] fixed memdessert unpacking --- uncore/memserdes.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/memserdes.scala b/uncore/memserdes.scala index 712dec16..e8d99514 100644 --- a/uncore/memserdes.scala +++ b/uncore/memserdes.scala @@ -144,8 +144,8 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.addr.width) + io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) + io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) From 3973aef9388edfffb1910ea21c7a2e45723be7ab Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 9 Oct 2012 18:04:55 -0700 Subject: [PATCH 133/688] handle structural hazard on LLC tags --- uncore/src/llc.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2bb2b62a..cb3fbb6a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -361,10 +361,10 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val stall_s1 = Bool() val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.valid && !stall_s1 || replay_s1, resetVal = Bool(false)) + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) replay_s1 := s1_valid && stall_s1 val s1 = Reg() { new MemReqCmd } - when (io.cpu.req_cmd.valid && io.cpu.req_cmd.ready) { s1 := io.cpu.req_cmd.bits } + when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } val stall_s2 = Bool() val s2_valid = Reg(resetVal = Bool(false)) @@ -430,7 +430,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 + io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 9610622ab0500addf535474ef91f3de518e5b947 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 12:41:11 -0700 Subject: [PATCH 134/688] moving memserdes + slowio into src --- uncore/{ => src}/memserdes.scala | 0 uncore/{ => src}/slowio.scala | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename uncore/{ => src}/memserdes.scala (100%) rename uncore/{ => src}/slowio.scala (100%) diff --git a/uncore/memserdes.scala b/uncore/src/memserdes.scala similarity index 100% rename from uncore/memserdes.scala rename to uncore/src/memserdes.scala diff --git a/uncore/slowio.scala b/uncore/src/slowio.scala similarity index 100% rename from uncore/slowio.scala rename to uncore/src/slowio.scala From 08ab076217312efd98bf697964e7db7573edfc94 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Wed, 10 Oct 2012 15:42:39 -0700 Subject: [PATCH 135/688] forgot to change package + using fromBits in memserdes instead of manual unpacking --- uncore/src/consts.scala | 3 +++ uncore/src/memserdes.scala | 6 ++---- uncore/src/slowio.scala | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 95b07cc8..a1e87f41 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,6 +10,9 @@ object Constants val HAVE_FPU = true val HAVE_VEC = true + val HTIF_WIDTH = 16 + val MEM_BACKUP_WIDTH = HTIF_WIDTH + val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load val M_XWR = Bits("b0001", 4); // int store diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index e8d99514..c44e2543 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ @@ -143,9 +143,7 @@ class MemDessert extends Component // test rig side val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) io.wide.req_cmd.valid := state === s_cmd - io.wide.req_cmd.bits.tag := req_cmd - io.wide.req_cmd.bits.addr := req_cmd.toUFix >> UFix(io.wide.req_cmd.bits.tag.width + io.wide.req_cmd.bits.rw.width) - io.wide.req_cmd.bits.rw := req_cmd(io.wide.req_cmd.bits.tag.width) + io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 068e90c5..fa6fef94 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Constants._ From 1418604bf01252c9b88bf337288b0a303ccf8717 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Oct 2012 18:52:48 -0700 Subject: [PATCH 136/688] new constants organization --- uncore/src/consts.scala | 103 +++++++++++++++++++-------------------- uncore/src/package.scala | 10 ++++ 2 files changed, 61 insertions(+), 52 deletions(-) create mode 100644 uncore/src/package.scala diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index a1e87f41..098bc1d6 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -1,17 +1,49 @@ package uncore +package constants import Chisel._ -import scala.math._ -object Constants +abstract trait MulticoreConstants { + val NTILES: Int + val TILE_ID_BITS = log2Up(NTILES)+1 +} + +abstract trait CoherenceConfigConstants { + val ENABLE_SHARING: Boolean + val ENABLE_CLEAN_EXCLUSIVE: Boolean +} + +trait UncoreConstants { + val NGLOBAL_XACTS = 8 + val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 +} + +trait TileLinkTypeConstants { + val X_INIT_TYPE_MAX_BITS = 2 + val X_REP_TYPE_MAX_BITS = 3 + val P_REQ_TYPE_MAX_BITS = 2 + val P_REP_TYPE_MAX_BITS = 3 +} + +trait TileLinkSizeConstants extends + TileLinkTypeConstants { - val NTILES = 1 - val HAVE_RVC = false - val HAVE_FPU = true - val HAVE_VEC = true + val TILE_XACT_ID_BITS = 5 + val X_INIT_WRITE_MASK_BITS = 6 + val X_INIT_SUBWORD_ADDR_BITS = 3 + val X_INIT_ATOMIC_OP_BITS = 4 +} - val HTIF_WIDTH = 16 - val MEM_BACKUP_WIDTH = HTIF_WIDTH +trait MemoryOpConstants { + val MT_X = Bits("b???", 3); + val MT_B = Bits("b000", 3); + val MT_H = Bits("b001", 3); + val MT_W = Bits("b010", 3); + val MT_D = Bits("b011", 3); + val MT_BU = Bits("b100", 3); + val MT_HU = Bits("b101", 3); + val MT_WU = Bits("b110", 3); val M_X = Bits("b????", 4); val M_XRD = Bits("b0000", 4); // int load @@ -30,52 +62,19 @@ object Constants val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); - - val PADDR_BITS = 40; - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; +} - // rocketNBDCache parameters - val DCACHE_PORTS = 3 - val CPU_DATA_BITS = 64; - val CPU_TAG_BITS = 9; - val DCACHE_TAG_BITS = log2Up(DCACHE_PORTS) + CPU_TAG_BITS - val OFFSET_BITS = 6; // log2(cache line size in bytes) - val NMSHR = if (HAVE_VEC) 4 else 2 // number of primary misses - val NRPQ = 16; // number of secondary misses - val NSDQ = 17; // number of secondary stores/AMOs - val LG_REFILL_WIDTH = 4; // log2(cache bus width in bytes) - val IDX_BITS = 7; - val TAG_BITS = PADDR_BITS - OFFSET_BITS - IDX_BITS; - val NWAYS = 4 - require(IDX_BITS+OFFSET_BITS <= PGIDX_BITS); +trait HTIFConstants { + val HTIF_WIDTH = 16 +} - // coherence parameters - val ENABLE_SHARING = true - val ENABLE_CLEAN_EXCLUSIVE = true - - - val COHERENCE_DATA_BITS = (1 << OFFSET_BITS)*8 - val TILE_ID_BITS = log2Up(NTILES)+1 - val TILE_XACT_ID_BITS = log2Up(NMSHR)+3 - val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) - - val X_INIT_TYPE_MAX_BITS = 2 - val X_INIT_WRITE_MASK_BITS = OFFSET_BITS - val X_INIT_SUBWORD_ADDR_BITS = log2Up(OFFSET_BITS) - val X_INIT_ATOMIC_OP_BITS = 4 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 - - // external memory interface +trait MemoryInterfaceConstants extends + HTIFConstants with + UncoreConstants with + TileLinkSizeConstants +{ val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 - val REFILL_CYCLES = (1 << OFFSET_BITS)*8/MEM_DATA_BITS - + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_BACKUP_WIDTH = HTIF_WIDTH } diff --git a/uncore/src/package.scala b/uncore/src/package.scala new file mode 100644 index 00000000..17954b54 --- /dev/null +++ b/uncore/src/package.scala @@ -0,0 +1,10 @@ +package uncore +import uncore.constants._ + +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants +{ + +} + From 8509cda8131b896540b1ddf33c879298c42b5870 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 13:58:18 -0700 Subject: [PATCH 137/688] Refined traits for use with rocket asserts, added UncoreConfiguration to handle ntiles --- uncore/src/consts.scala | 21 +++++-- uncore/src/package.scala | 7 ++- uncore/src/uncore.scala | 125 ++++++++++++++++++++------------------- 3 files changed, 85 insertions(+), 68 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 098bc1d6..b7a70bac 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -2,11 +2,7 @@ package uncore package constants import Chisel._ - -abstract trait MulticoreConstants { - val NTILES: Int - val TILE_ID_BITS = log2Up(NTILES)+1 -} +import scala.math.max abstract trait CoherenceConfigConstants { val ENABLE_SHARING: Boolean @@ -19,6 +15,10 @@ trait UncoreConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } +trait CacheConstants extends UncoreConstants { + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + trait TileLinkTypeConstants { val X_INIT_TYPE_MAX_BITS = 2 val X_REP_TYPE_MAX_BITS = 3 @@ -78,3 +78,14 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS val MEM_BACKUP_WIDTH = HTIF_WIDTH } + +trait AddressConstants { + val PADDR_BITS = 40; + val VADDR_BITS = 43; + val PGIDX_BITS = 13; + val PPN_BITS = PADDR_BITS-PGIDX_BITS; + val VPN_BITS = VADDR_BITS-PGIDX_BITS; + val ASID_BITS = 7; + val PERM_BITS = 6; +} + diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 17954b54..e3a8f554 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,10 +1,13 @@ package uncore import uncore.constants._ +//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to +// package object uncore and remove import Constants._'s from other files object Constants extends MemoryOpConstants with - MemoryInterfaceConstants + MemoryInterfaceConstants with + CacheConstants with + AddressConstants { - } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 1220a56f..dcd1a819 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -36,13 +36,14 @@ class ioMemPipe() extends Bundle val resp = (new PipeIO) { new MemResp() }.flip } -class TrackerProbeData extends Bundle { - val tile_id = Bits(width = TILE_ID_BITS) +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { + val tile_id = Bits(width = conf.tile_id_bits) } -class TrackerAllocReq extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = TILE_ID_BITS) + val tile_id = Bits(width = conf.tile_id_bits) + override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { @@ -61,15 +62,15 @@ class ioTileLink extends Bundle { val incoherent = Bool(OUTPUT) } -class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { +class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, ntiles) - val p_req_cnt_inc = Bits(INPUT, ntiles) - val tile_incoherent = Bits(INPUT, ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) + val p_req_cnt_inc = Bits(INPUT, conf.ntiles) + val tile_incoherent = Bits(INPUT, conf.ntiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -82,18 +83,18 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, TILE_ID_BITS) - val p_rep_tile_id = Bits(OUTPUT, TILE_ID_BITS) + val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, TILE_ID_BITS+1) + val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, ntiles) - val pop_p_rep = Bits(OUTPUT, ntiles) - val pop_p_rep_data = Bits(OUTPUT, ntiles) - val pop_p_rep_dep = Bits(OUTPUT, ntiles) - val pop_x_init = Bits(OUTPUT, ntiles) - val pop_x_init_data = Bits(OUTPUT, ntiles) - val pop_x_init_dep = Bits(OUTPUT, ntiles) + val push_p_req = Bits(OUTPUT, conf.ntiles) + val pop_p_rep = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) + val pop_x_init = Bits(OUTPUT, conf.ntiles) + val pop_x_init_data = Bits(OUTPUT, conf.ntiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -136,8 +137,8 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val x_type_ = Reg{ Bits() } val init_tile_id_ = Reg{ Bits() } val tile_xact_id_ = Reg{ Bits() } - val p_rep_count = if (ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = ntiles)) + val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -147,15 +148,15 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = ntiles) - p_req_initial_flags := (if (ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(ntiles)-1,0)))) //TODO: Broadcast only + val p_req_initial_flags = Bits(width = conf.ntiles) + p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only io.busy := state != s_idle io.addr := addr_ io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := tile_xact_id_ - io.sharer_count := UFix(ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only io.x_type := x_type_ io.mem_req_cmd.valid := Bool(false) @@ -169,13 +170,13 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := addr_ - io.push_p_req := Bits(0, width = ntiles) - io.pop_p_rep := Bits(0, width = ntiles) - io.pop_p_rep_data := Bits(0, width = ntiles) - io.pop_p_rep_dep := Bits(0, width = ntiles) - io.pop_x_init := Bits(0, width = ntiles) - io.pop_x_init_data := Bits(0, width = ntiles) - io.pop_x_init_dep := Bits(0, width = ntiles) + io.push_p_req := Bits(0, width = conf.ntiles) + io.pop_p_rep := Bits(0, width = conf.ntiles) + io.pop_p_rep_data := Bits(0, width = conf.ntiles) + io.pop_p_rep_dep := Bits(0, width = conf.ntiles) + io.pop_x_init := Bits(0, width = conf.ntiles) + io.pop_x_init_data := Bits(0, width = conf.ntiles) + io.pop_x_init_dep := Bits(0, width = conf.ntiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -192,7 +193,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(ntiles > 1) { + if(conf.ntiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -209,7 +210,7 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ntiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -260,14 +261,16 @@ class XactTracker(ntiles: Int, id: Int, co: CoherencePolicy) extends Component { } } -abstract class CoherenceHub(ntiles: Int, co: CoherencePolicy) extends Component { +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) + +abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { - val tiles = Vec(ntiles) { new ioTileLink() }.flip + val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) +class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback @@ -294,23 +297,23 @@ class CoherenceHubNull(co: ThreeStateIncoherence) extends CoherenceHub(1, co) } -class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceHub(ntiles, co) +class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(ntiles, _, co)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_ID_BITS) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width = TILE_ID_BITS) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -331,19 +334,19 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = TILE_ID_BITS) + p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -355,7 +358,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -394,7 +397,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -414,10 +417,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until ntiles) { + for( j <- 0 until conf.ntiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -425,9 +428,9 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(ntiles) { Bool() } - for( j <- 0 until ntiles ) { + val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ntiles) { Bool() } + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -472,7 +475,7 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(ntiles)) { new TrackerAllocReq() } + val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -481,10 +484,10 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until ntiles ) { + for( j <- 0 until conf.ntiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -505,8 +508,8 @@ class CoherenceHubBroadcast(ntiles: Int, co: CoherencePolicy) extends CoherenceH // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until ntiles ) { + val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ntiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 9df5cfa5524249d8b1ce3f955e5090cd73e79980 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 16 Oct 2012 14:26:33 -0700 Subject: [PATCH 138/688] Factored out tilelink classes --- uncore/src/coherence.scala | 64 --------------------- uncore/src/tilelink.scala | 111 +++++++++++++++++++++++++++++++++++++ uncore/src/uncore.scala | 45 --------------- 3 files changed, 111 insertions(+), 109 deletions(-) create mode 100644 uncore/src/tilelink.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7fc03ece..dcf384aa 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,70 +3,6 @@ package uncore import Chisel._ import Constants._ -object TransactionInit -{ - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init - } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init - } -} -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) -} - -class TransactionInitData extends MemData - -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) -} - -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - -class ProbeReplyData extends MemData - -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) - val require_ack = Bool() -} - -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) -} - object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = { val store = (cmd === M_XWR) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala new file mode 100644 index 00000000..bddbc0d3 --- /dev/null +++ b/uncore/src/tilelink.scala @@ -0,0 +1,111 @@ +package uncore + +import Chisel._ +import Constants._ + + +class PhysicalAddress extends Bundle { + val addr = UFix(width = PADDR_BITS - OFFSET_BITS) +} + +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends PhysicalAddress { + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends MemData { + val tag = Bits(width = MEM_TAG_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + +class TransactionInit extends PhysicalAddress { + val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) + val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +} + +object TransactionInit +{ + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.write_mask := write_mask + init + } + def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val init = new TransactionInit + init.x_type := x_type + init.addr := addr + init.tile_xact_id := tile_xact_id + init.subword_addr := subword_addr + init.atomic_opcode := atomic_opcode + init + } +} + +class TransactionInitData extends MemData + +class TransactionAbort extends Bundle { + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +} + +class ProbeRequest extends PhysicalAddress { + val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReply extends Bundle { + val p_type = Bits(width = P_REP_TYPE_MAX_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ProbeReplyData extends MemData + +class TransactionReply extends MemData { + val x_type = Bits(width = X_REP_TYPE_MAX_BITS) + val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val require_ack = Bool() +} + +class TransactionFinish extends Bundle { + val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +} + +class ioTileLink extends Bundle { + val xact_init = (new FIFOIO) { new TransactionInit } + val xact_init_data = (new FIFOIO) { new TransactionInitData } + val xact_abort = (new FIFOIO) { new TransactionAbort }.flip + val probe_req = (new FIFOIO) { new ProbeRequest }.flip + val probe_rep = (new FIFOIO) { new ProbeReply } + val probe_rep_data = (new FIFOIO) { new ProbeReplyData } + val xact_rep = (new FIFOIO) { new TransactionReply }.flip + val xact_finish = (new FIFOIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dcd1a819..ebca0834 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,39 +3,6 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) -} - -class MemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemReqCmd() extends PhysicalAddress -{ - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp () extends MemData -{ - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe() extends Bundle -{ - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip -} - class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val tile_id = Bits(width = conf.tile_id_bits) } @@ -50,18 +17,6 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip From ffda0e41a99e3f344d08e43516ee446d0f28c4b0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:56:36 -0700 Subject: [PATCH 139/688] parameterize width of MemSerdes/MemDesser --- uncore/src/memserdes.scala | 54 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c44e2543..783fb3e1 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -6,17 +6,17 @@ import Constants._ import scala.math._ import uncore._ -class ioMemSerialized extends Bundle +class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = MEM_BACKUP_WIDTH) } - val resp = (new PipeIO) { Bits(width = MEM_BACKUP_WIDTH) }.flip + val req = (new FIFOIO) { Bits(width = w) } + val resp = (new PipeIO) { Bits(width = w) }.flip } -class MemSerdes extends Component +class MemSerdes(w: Int) extends Component { val io = new Bundle { val wide = new ioMem().flip - val narrow = new ioMemSerialized + val narrow = new ioMemSerialized(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -27,14 +27,14 @@ class MemSerdes extends Component val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(MEM_BACKUP_WIDTH) + out_buf := out_buf >> UFix(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -68,19 +68,19 @@ class MemSerdes extends Component send_cnt := UFix(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) val resp_val = Reg(resetVal = Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH)) { + when (recv_cnt === UFix((rbits-1)/w)) { recv_cnt := UFix(0) data_recv_cnt := data_recv_cnt + UFix(1) resp_val := Bool(true) } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.wide.resp.valid := resp_val @@ -88,22 +88,24 @@ class MemSerdes extends Component io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) } -class MemDessert extends Component // test rig side +class MemDesserIO(w: Int) extends Bundle { + val narrow = new ioMemSerialized(w).flip + val wide = new ioMem +} + +class MemDesser(w: Int) extends Component // test rig side { - val io = new Bundle { - val narrow = new ioMemSerialized().flip - val wide = new ioMem - } + val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH))) + val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/MEM_BACKUP_WIDTH) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/MEM_BACKUP_WIDTH) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } val state = Reg(resetVal = s_cmd_recv) @@ -111,7 +113,7 @@ class MemDessert extends Component // test rig side val in_buf = Reg() { Bits() } when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { recv_cnt := recv_cnt + UFix(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH*MEM_BACKUP_WIDTH-1,MEM_BACKUP_WIDTH)) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv @@ -141,17 +143,17 @@ class MemDessert extends Component // test rig side data_recv_cnt := data_recv_cnt + UFix(1) } - val req_cmd = in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (abits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH - (dbits+MEM_BACKUP_WIDTH-1)/MEM_BACKUP_WIDTH)*MEM_BACKUP_WIDTH) + io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/MEM_BACKUP_WIDTH) + dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(MEM_BACKUP_WIDTH)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) } From 2aecb0024fa12caa8a71aa1250bb25b432e0b8aa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 18 Oct 2012 16:57:28 -0700 Subject: [PATCH 140/688] UncoreConfiguration now contains coherence policy --- uncore/src/consts.scala | 6 ------ uncore/src/uncore.scala | 16 ++++++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b7a70bac..757959fb 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -64,19 +64,13 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); } -trait HTIFConstants { - val HTIF_WIDTH = 16 -} - trait MemoryInterfaceConstants extends - HTIFConstants with UncoreConstants with TileLinkSizeConstants { val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_BACKUP_WIDTH = HTIF_WIDTH } trait AddressConstants { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ebca0834..80c088f2 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -17,7 +17,8 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { + val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip @@ -216,17 +217,19 @@ class XactTracker(id: Int, co: CoherencePolicy)(implicit conf: UncoreConfigurati } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int) +case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) -abstract class CoherenceHub(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { val io = new Bundle { val tiles = Vec(conf.ntiles) { new ioTileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub { + val co = conf.co.asInstanceOf[ThreeStateIncoherence] + val x_init = io.tiles(0).xact_init val is_write = x_init.bits.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp @@ -252,9 +255,10 @@ class CoherenceHubNull(co: ThreeStateIncoherence)(implicit conf: UncoreConfigura } -class CoherenceHubBroadcast(co: CoherencePolicy)(implicit conf: UncoreConfiguration) extends CoherenceHub(co)(conf) +class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub { - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_, co)) + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } From 0cd0f8a9db89d34e057008d051b82a2e48ed6c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 23 Oct 2012 18:01:53 -0700 Subject: [PATCH 141/688] Initial version of migratory protocol --- uncore/src/coherence.scala | 174 ++++++++++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 2 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index dcf384aa..7bbeaf48 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -9,7 +9,7 @@ object cpuCmdToRW { val load = (cmd === M_XRD) val amo = cmd(3).toBool val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo + val write = store || amo (read, write) } } @@ -556,7 +556,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { @@ -699,3 +699,173 @@ class MESICoherence extends CoherencePolicyWithUncached { (x_type === xactInitWriteUncached) } } + +class MigratoryCoherence extends CoherencePolicyWithUncached { + + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + + val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } + val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } + val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } + val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + + def isHit (cmd: Bits, state: UFix): Bool = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + } + def isValid (state: UFix): Bool = { + state != tileInvalid + } + + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + val (read, write) = cpuCmdToRW(cmd) + (read && messageIsUncached(outstanding)) || + (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + } + def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + MuxLookup(cmd, (state === tileExclusiveDirty), Array( + M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + )) + } + def needsWriteback (state: UFix): Bool = { + needsTransactionOnCacheControl(M_INV, state) + } + + def newStateOnHit(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + tileExclusiveClean -> tileExclusiveDirty, + tileMigratoryClean -> tileMigratoryDirty)), state) + } + def newStateOnCacheControl(cmd: Bits) = { + MuxLookup(cmd, tileInvalid, Array( + M_INV -> tileInvalid, + M_CLN -> tileShared + )) + } + def newStateOnWriteback() = newStateOnCacheControl(M_INV) + def newStateOnFlush() = newStateOnCacheControl(M_INV) + def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { + MuxLookup(incoming.x_type, tileInvalid, Array( + xactReplyReadShared -> tileShared, + xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( + xactInitReadExclusive -> tileExclusiveDirty, + xactInitReadShared -> tileExclusiveClean)), + xactReplyReadExclusiveAck -> tileExclusiveDirty, + xactReplyReadUncached -> tileInvalid, + xactReplyWriteUncached -> tileInvalid, + xactReplyReadWordUncached -> tileInvalid, + xactReplyWriteWordUncached -> tileInvalid, + xactReplyAtomicUncached -> tileInvalid, + xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( + xactInitInvalidateOthers -> tileMigratoryDirty, + xactInitReadExclusive -> tileMigratoryDirty, + xactInitReadShared -> tileMigratoryClean)) + )) + } + def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + MuxLookup(incoming.p_type, state, Array( + probeReqInvalidate -> tileInvalid, + probeReqInvalidateOthers -> tileInvalid, + probeReqCopy -> state, + probeReqDowngrade -> MuxLookup(state, tileShared, Array( + tileExclusiveClean -> tileSharedByTwo, + tileExclusiveDirty -> tileSharedByTwo, + tileSharedByTwo -> tileShared, + tileMigratoryClean -> tileSharedByTwo, + tileMigratoryDirty -> tileInvalid)) + )) + } + + def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) + def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) + def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) + def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + + def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + } + def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + val (read, write) = cpuCmdToRW(cmd) + Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + } + def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached + def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + + def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { + Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new ProbeReply() + val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( + probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeRepInvalidateDataMigratory, probeRepInvalidateData), + probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), + probeReqCopy -> probeRepCopyData + )) + val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( + probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), + probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), + probeReqCopy -> probeRepCopyAck + )) + reply.p_type := Mux(needsWriteback(state), with_data, without_data) + reply.global_xact_id := incoming.global_xact_id + reply + } + + def messageHasData (reply: ProbeReply): Bool = { + uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + } + def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) + def messageHasData (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + } + def messageUpdatesDataArray (reply: TransactionReply): Bool = { + uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + } + def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + + def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + + def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { + MuxLookup(x_type, xactReplyReadUncached, Array( + xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? + xactInitReadExclusive -> xactReplyReadExclusive, + xactInitReadUncached -> xactReplyReadUncached, + xactInitWriteUncached -> xactReplyWriteUncached, + xactInitReadWordUncached -> xactReplyReadWordUncached, + xactInitWriteWordUncached -> xactReplyWriteWordUncached, + xactInitAtomicUncached -> xactReplyAtomicUncached, + xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + )) + } + + def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { + MuxLookup(x_type, probeReqCopy, Array( + xactInitReadShared -> probeReqDowngrade, + xactInitReadExclusive -> probeReqInvalidate, + xactInitReadUncached -> probeReqCopy, + xactInitWriteUncached -> probeReqInvalidate, + xactInitReadWordUncached -> probeReqCopy, + xactInitWriteWordUncached -> probeReqInvalidate, + xactInitAtomicUncached -> probeReqInvalidate, + xactInitInvalidateOthers -> probeReqInvalidateOthers + )) + } + + def needsMemRead(x_type: UFix, global_state: UFix): Bool = { + (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + } + def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + } + def needsAckReply(x_type: UFix, global_state: UFix): Bool = { + (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + } +} From 3e6dc35809509ddffac67a7d4af1d2c0f2294eff Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 16 Nov 2012 02:37:56 -0800 Subject: [PATCH 142/688] issue self-probes for uncached read transactions this facilitates I$ coherence. but it seems like a hack and perhaps the mechanism should be rethought. --- uncore/src/coherence.scala | 15 +++++++-------- uncore/src/consts.scala | 5 ++++- uncore/src/uncore.scala | 34 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 7bbeaf48..d7cf1ded 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,14 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = { - val store = (cmd === M_XWR) - val load = (cmd === M_XRD) - val amo = cmd(3).toBool - val read = load || amo || (cmd === M_PFR) || (cmd === M_PFW) - val write = store || amo - (read, write) - } + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) } abstract class CoherencePolicy { @@ -56,6 +49,7 @@ trait UncachedTransactions { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit + def isUncachedReadTransaction(xinit: TransactionInit): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions @@ -182,6 +176,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive @@ -317,6 +312,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -470,6 +466,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -620,6 +617,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -787,6 +785,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 757959fb..57390387 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -50,7 +50,6 @@ trait MemoryOpConstants { val M_XWR = Bits("b0001", 4); // int store val M_PFR = Bits("b0010", 4); // prefetch with intent to read val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FLA = Bits("b0100", 4); // write back and invlaidate all lines val M_FENCE = Bits("b0101", 4); // memory fence val M_INV = Bits("b0110", 4); // write back and invalidate line val M_CLN = Bits("b0111", 4); // write back line @@ -62,6 +61,10 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b1101", 4); val M_XA_MINU = Bits("b1110", 4); val M_XA_MAXU = Bits("b1111", 4); + + def isAMO(cmd: Bits) = cmd(3) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } trait MemoryInterfaceConstants extends diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 80c088f2..e2dcfc30 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -89,10 +89,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val addr_ = Reg{ UFix() } - val x_type_ = Reg{ Bits() } + val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val tile_xact_id_ = Reg{ Bits() } val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) val p_rep_tile_id_ = Reg{ Bits() } @@ -105,27 +103,37 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val p_req_initial_flags = Bits(width = conf.ntiles) - p_req_initial_flags := (if (conf.ntiles == 1) Bits(0) else ~(io.tile_incoherent | UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0)))) //TODO: Broadcast only + p_req_initial_flags := Bits(0) + if (conf.ntiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } io.busy := state != s_idle - io.addr := addr_ + io.addr := xact.addr io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ - io.tile_xact_id := tile_xact_id_ + io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only - io.x_type := x_type_ + io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := addr_ + io.mem_req_cmd.bits.addr := xact.addr io.mem_req_cmd.bits.tag := UFix(id) io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(x_type_, UFix(0)) + io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := addr_ + io.probe_req.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ntiles) io.pop_p_rep := Bits(0, width = conf.ntiles) io.pop_p_rep_data := Bits(0, width = conf.ntiles) @@ -138,10 +146,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - addr_ := io.alloc_req.bits.xact_init.addr - x_type_ := io.alloc_req.bits.xact_init.x_type + xact := io.alloc_req.bits.xact_init init_tile_id_ := io.alloc_req.bits.tile_id - tile_xact_id_ := io.alloc_req.bits.xact_init.tile_xact_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags @@ -202,7 +208,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(x_type_, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { From 6bd4f93f8c27044e6dd6244258b3c4082deda354 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sun, 18 Nov 2012 03:13:17 -0800 Subject: [PATCH 143/688] pull out prefetch commands from isRead --- uncore/src/consts.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 57390387..29be0acc 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -63,7 +63,8 @@ trait MemoryOpConstants { val M_XA_MAXU = Bits("b1111", 4); def isAMO(cmd: Bits) = cmd(3) - def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_PFR || cmd === M_PFW || isAMO(cmd) + def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW + def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) } From 56f9b9721df5283e0fb3f854cf6eb42a52f5bbca Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 20 Nov 2012 05:38:49 -0800 Subject: [PATCH 144/688] treat prefetches as read requests --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d7cf1ded..e8aad9d0 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -4,7 +4,7 @@ import Chisel._ import Constants._ object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd), isWrite(cmd)) + def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } abstract class CoherencePolicy { From 8103676b37b40f0a32f46527ad771f3dc0edf54b Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 26 Nov 2012 20:54:56 -0800 Subject: [PATCH 145/688] reduce physical address space to 4GB --- uncore/src/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 29be0acc..0b885352 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -78,7 +78,7 @@ trait MemoryInterfaceConstants extends } trait AddressConstants { - val PADDR_BITS = 40; + val PADDR_BITS = 32 val VADDR_BITS = 43; val PGIDX_BITS = 13; val PPN_BITS = PADDR_BITS-PGIDX_BITS; From 50e9d952e874ae5415d87014966a350347ec23e1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 4 Dec 2012 06:57:53 -0800 Subject: [PATCH 146/688] don't initiate llc refill until writeback drains --- uncore/src/llc.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb3fbb6a..2d3eea16 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -109,6 +109,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val requested = Bool() val old_dirty = Bool() val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } @@ -124,12 +125,13 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).way := io.repl_way mshr(freeId).old_dirty := io.repl_dirty mshr(freeId).old_tag := io.repl_tag + mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) mshr(freeId).refillCount := UFix(0) mshr(freeId).refilled := Bool(false) } - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).requested):_*) + val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) val request = requests.orR val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -149,7 +151,11 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) val writeback = writebacks.orR val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { mshr(writebackId).old_dirty := Bool(false) } + when (writeback && io.data.ready && !replay) { + mshr(writebackId).old_dirty := Bool(false) + mshr(writebackId).wb_busy := Bool(true) + } + mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) io.cpu.ready := !conflicts.orR && !validBits.andR From aae7a677812e20227d8f6f87b56c40765dbd91f5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 6 Dec 2012 02:07:03 -0800 Subject: [PATCH 147/688] fix llc refill/writeback bugs --- uncore/src/llc.scala | 55 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 2d3eea16..7ccdedb8 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -132,7 +132,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component } val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR + val request = requests.orR && io.data.ready // allow in-flight hits to drain val requestId = PriorityEncoder(requests) when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } @@ -365,19 +365,21 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UFix(1) } - val stall_s1 = Bool() - val replay_s1 = Reg(resetVal = Bool(false)) - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s1, resetVal = Bool(false)) - replay_s1 := s1_valid && stall_s1 + val replay_s2 = Reg(resetVal = Bool(false)) + val s2_valid = Reg(resetVal = Bool(false)) + val s2 = Reg() { new MemReqCmd } + val s3_rdy = Bool() + val replay_s2_rdy = Bool() + + val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg() { new MemReqCmd } when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } + when (replay_s2 && replay_s2_rdy) { s1 := s2 } - val stall_s2 = Bool() - val s2_valid = Reg(resetVal = Bool(false)) - s2_valid := s1_valid && !replay_s1 && !stall_s1 || stall_s2 - val s2 = Reg() { new MemReqCmd } + s2_valid := s1_valid + replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } - when (s1_valid && !stall_s1 && !replay_s1) { + when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) @@ -389,26 +391,24 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) val repl_tag = s2_tags(repl_way).toUFix val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - stall_s1 := initialize || stall_s2 - val tag_we = setDirty || mshr.io.tag.valid - val tag_waddr = Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(log2Up(sets)-1,0) - val tag_wdata = Cat(setDirty, Bool(true), Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wway = Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way) - tags.io.in.valid := (io.cpu.req_cmd.valid || replay_s1) && !stall_s1 || initialize || tag_we - tags.io.in.bits.addr := Mux(initialize, initCount, Mux(tag_we, tag_waddr, Mux(replay_s1, s1.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0))) - tags.io.in.bits.rw := initialize || tag_we - tags.io.in.bits.wdata := Mux(initialize, UFix(0), Fill(ways, tag_wdata)) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, Mux(initialize, Fix(-1, ways), UFixToOH(tag_wway))) - when (tag_we && Mux(stall_s2, s2.addr, s1.addr)(log2Up(sets)-1,0) === tag_waddr) { s2_tags(tag_wway) := tag_wdata } + val tag_we = initialize || setDirty || mshr.io.tag.fire() + val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we + tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) + tags.io.in.bits.rw := tag_we + tags.io.in.bits.wdata := Fill(ways, tag_wdata) + tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw && dataArb.io.in(1).ready && writeback.io.req(0).ready // stall_s2 + mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw mshr.io.cpu.bits := s2 mshr.io.repl_way := repl_way mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR mshr.io.repl_tag := repl_tag mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !setDirty + mshr.io.tag.ready := !s1_valid && !s2_valid data.io.req <> dataArb.io.out data.io.mem_resp := io.mem.resp @@ -419,7 +419,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da writeback.io.req(0) <> data.io.writeback writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && dataArb.io.in(1).ready && mshr.io.cpu.ready // stall_s2 + writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready writeback.io.req(1).bits := s2.addr writeback.io.data(1).valid := io.cpu.req_data.valid writeback.io.data(1).bits := io.cpu.req_data.bits @@ -428,15 +428,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da memCmdArb.io.in(1) <> writeback.io.mem.req_cmd dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && writeback.io.req(0).ready && mshr.io.cpu.ready // stall_s2 + dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready dataArb.io.in(1).bits := s2 dataArb.io.in(1).bits.way := s2_hit_way dataArb.io.in(1).bits.isWriteback := Bool(false) - stall_s2 := s2_valid && !(dataArb.io.in(1).ready && writeback.io.req(0).ready && mshr.io.cpu.ready) + s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) + replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !stall_s1 && !replay_s1 && !tag_we + io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data From 12caa55dc7e7aa99c4b2449d01fa773d07b8d863 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 148/688] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 8d69f9b41cebb267d5e8cab7756fb2a6e616e10f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 149/688] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From f359518e52375496f6cde523046310b215bc5955 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 6 Dec 2012 18:51:30 -0800 Subject: [PATCH 150/688] wip: new network classes --- uncore/src/tilelink.scala | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index bddbc0d3..00d40e54 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,6 +3,72 @@ package uncore import Chisel._ import Constants._ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + val temp = UFix(width = conf.idBits) +} + +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asInput +} + + +abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asInput +} + +class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) +class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) From 6d61baa6cd8d977bae9a93e862bb3972017eb6f2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Dec 2012 00:05:28 -0800 Subject: [PATCH 151/688] Initial version of phys/log network compiles --- uncore/src/memserdes.scala | 1 - uncore/src/package.scala | 74 ++++++++++++++++++++++++++++++++------ uncore/src/temp.scala | 13 +++++++ uncore/src/tilelink.scala | 67 ---------------------------------- 4 files changed, 77 insertions(+), 78 deletions(-) create mode 100644 uncore/src/temp.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 783fb3e1..88a6f1b3 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -4,7 +4,6 @@ import Chisel._ import Node._ import Constants._ import scala.math._ -import uncore._ class ioMemSerialized(w: Int) extends Bundle { diff --git a/uncore/src/package.scala b/uncore/src/package.scala index e3a8f554..a3af64e1 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,13 +1,67 @@ -package uncore -import uncore.constants._ +package object uncore { +import Chisel._ -//TODO: When compiler bug SI-5604 is fixed in 2.10, change object Constants to -// package object uncore and remove import Constants._'s from other files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) } +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new PhysicalHeader).asOutput +} + +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip + val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + } + + for(i <- 0 until conf.nEndpoints) { + val rrarb = new RRArbiter(conf.nEndpoints)(data) + (rrarb.io.in, io.in).zipped.map( (arb, io) => { + arb.valid := io.valid && (io.header.dst === UFix(i)) + arb.bits := io.bits + io.ready := arb.ready + }) + io.out(i) <> rrarb.io.out + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { + val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { + val header = (new LogicalHeader).asOutput +} + +class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) +class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} + +class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val xact_init = (new TileIO) { new TransactionInit } + val xact_init_data = (new TileIO) { new TransactionInitData } + val xact_abort = (new HubIO) { new TransactionAbort } + val probe_req = (new HubIO) { new ProbeRequest } + val probe_rep = (new TileIO) { new ProbeReply } + val probe_rep_data = (new TileIO) { new ProbeReplyData } + val xact_rep = (new HubIO) { new TransactionReply } + val xact_finish = (new TileIO) { new TransactionFinish } + val incoherent = Bool(OUTPUT) +} +} diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala new file mode 100644 index 00000000..b2d3fdca --- /dev/null +++ b/uncore/src/temp.scala @@ -0,0 +1,13 @@ +package uncore +import _root_.uncore.constants._ + +//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and +// mixin Constants traits to package object uncore in package.scala and +// remove import Constants._'s from other .scala files +object Constants extends + MemoryOpConstants with + MemoryInterfaceConstants with + CacheConstants with + AddressConstants +{ +} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 00d40e54..a9cbc482 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,73 +3,6 @@ package uncore import Chisel._ import Constants._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - val temp = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asInput -} - - -abstract class PhysicalNetwork(implicit conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - } - - for(i <- 0 until conf.nEndpoints) { - val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asInput -} - -class TileIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) -class HubIO[T <: Data]()(data: => T)(implicit m: Manifest[T], conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(m,conf) - -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) -} - class PhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } From 400d48e3de0a00444cef01e03829cfbb2450ced8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 13 Dec 2012 11:39:14 -0800 Subject: [PATCH 152/688] Refactored uncore conf --- uncore/src/coherence.scala | 4 ++ uncore/src/package.scala | 3 +- uncore/src/uncore.scala | 124 ++++++++++++++++++------------------- 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e8aad9d0..96fd475d 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,6 +3,10 @@ package uncore import Chisel._ import Constants._ +trait CoherenceAgent +trait ClientCoherenceAgent extends CoherenceAgent +trait MasterCoherenceAgent extends CoherenceAgent + object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a3af64e1..137cc74d 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -35,7 +35,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[Component])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -63,5 +63,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) + override def clone = { new TileLink().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e2dcfc30..99136c1b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,13 +3,13 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val tile_id = Bits(width = conf.tile_id_bits) +class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { + val tile_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.tile_id_bits) + val tile_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,16 +17,16 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component { +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val xact_finish = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ntiles) - val p_req_cnt_inc = Bits(INPUT, conf.ntiles) - val tile_incoherent = Bits(INPUT, conf.ntiles) + val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip val x_init_data = (new PipeIO) { new TransactionInitData }.flip val sent_x_rep_ack = Bool(INPUT) @@ -39,18 +39,18 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.tile_id_bits) - val p_rep_tile_id = Bits(OUTPUT, conf.tile_id_bits) + val init_tile_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.tile_id_bits+1) + val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ntiles) - val pop_p_rep = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ntiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ntiles) - val pop_x_init = Bits(OUTPUT, conf.ntiles) - val pop_x_init_data = Bits(OUTPUT, conf.ntiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ntiles) + val push_p_req = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) val send_x_rep_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } val init_tile_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ntiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ntiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ntiles)) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val p_rep_tile_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,16 +102,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ntiles) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) p_req_initial_flags := Bits(0) - if (conf.ntiles > 1) { + if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ntiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } @@ -120,7 +120,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.init_tile_id := init_tile_id_ io.p_rep_tile_id := p_rep_tile_id_ io.tile_xact_id := xact.tile_xact_id - io.sharer_count := UFix(conf.ntiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type io.mem_req_cmd.valid := Bool(false) @@ -134,13 +134,13 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) io.probe_req.bits.global_xact_id := UFix(id) io.probe_req.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ntiles) - io.pop_p_rep := Bits(0, width = conf.ntiles) - io.pop_p_rep_data := Bits(0, width = conf.ntiles) - io.pop_p_rep_dep := Bits(0, width = conf.ntiles) - io.pop_x_init := Bits(0, width = conf.ntiles) - io.pop_x_init_data := Bits(0, width = conf.ntiles) - io.pop_x_init_dep := Bits(0, width = conf.ntiles) + io.push_p_req := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) + io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) + io.pop_x_init := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) + io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) io.send_x_rep_ack := Bool(false) switch (state) { @@ -155,7 +155,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id - if(conf.ntiles > 1) { + if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -172,7 +172,7 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component when(io.p_rep_cnt_dec.orR) { val dec = PopCount(io.p_rep_cnt_dec) io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ntiles > 1) p_rep_count := p_rep_count - dec + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec when(p_rep_count === dec) { state := s_mem } @@ -223,16 +223,16 @@ class XactTracker(id: Int)(implicit conf: UncoreConfiguration) extends Component } } -case class UncoreConfiguration(ntiles: Int, tile_id_bits: Int, co: CoherencePolicy) +case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) -abstract class CoherenceHub(implicit conf: UncoreConfiguration) extends Component { +abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.ntiles) { new ioTileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLink }.flip val mem = new ioMem } } -class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co.asInstanceOf[ThreeStateIncoherence] @@ -261,24 +261,24 @@ class CoherenceHubNull(implicit conf: UncoreConfiguration) extends CoherenceHub } -class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends CoherenceHub +class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ntiles){ Bool()} } + val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.tile_id_bits) } + val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { @@ -299,19 +299,19 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.tile_id_bits) + p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) p_req_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ntiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { do_free_arr(finish.bits.global_xact_id) := Bool(true) @@ -323,7 +323,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep rep.bits.x_type := UFix(0) rep.bits.tile_xact_id := UFix(0) @@ -362,7 +362,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data val idx = p_rep.bits.global_xact_id @@ -382,10 +382,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ntiles) { + for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) } @@ -393,9 +393,9 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Nack conflicting transaction init attempts val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ntiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ntiles) { Bool() } - for( j <- 0 until conf.ntiles ) { + val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } + val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_abort = io.tiles(j).xact_abort @@ -440,7 +440,7 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ntiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -449,10 +449,10 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ntiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ntiles ) { + for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init val x_init_data = io.tiles(j).xact_init_data val x_init_data_dep = x_init_data_dep_list(j).io.deq @@ -473,8 +473,8 @@ class CoherenceHubBroadcast(implicit conf: UncoreConfiguration) extends Coherenc // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ntiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) - for( j <- 0 until conf.ntiles ) { + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits From 418e3fdf5046111bc34edb00462802ebba20ddc9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 7 Jan 2013 13:57:48 -0800 Subject: [PATCH 153/688] Bugfix in crossbar ready sigs, added adapter for old hub, removed incoherent sig from tilelink --- uncore/src/package.scala | 29 +++++++++++++++++---------- uncore/src/uncore.scala | 42 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 137cc74d..9698ffc3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,6 +1,9 @@ package object uncore { import Chisel._ +//TODO: Remove these Networking classes from the package object once Scala bug +//SI-3439 is resolved. + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -22,14 +25,21 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } } - for(i <- 0 until conf.nEndpoints) { + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { val rrarb = new RRArbiter(conf.nEndpoints)(data) - (rrarb.io.in, io.in).zipped.map( (arb, io) => { - arb.valid := io.valid && (io.header.dst === UFix(i)) - arb.bits := io.bits - io.ready := arb.ready - }) - io.out(i) <> rrarb.io.out + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.header.dst === UFix(i)) + }} + out <> rrarb.io.out + out.header.src := rrarb.io.chosen.toUFix + out.header.dst := UFix(i) + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } @@ -53,7 +63,7 @@ abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalN class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} -class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new TileIO) { new TransactionInit } val xact_init_data = (new TileIO) { new TransactionInitData } val xact_abort = (new HubIO) { new TransactionAbort } @@ -62,7 +72,6 @@ class TileLink(implicit conf: LogicalNetworkConfiguration) extends Bundle { val probe_rep_data = (new TileIO) { new ProbeReplyData } val xact_rep = (new HubIO) { new TransactionReply } val xact_finish = (new TileIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) - override def clone = { new TileLink().asInstanceOf[this.type] } + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 99136c1b..daf756f0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -225,9 +225,47 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) +class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val net = (new TileLinkIO).flip + val hub = Vec(conf.nTiles) { new TileLinkIO } + } + + val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[TileIO[Data]] } + val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[HubIO[Data]] } + + val hubTileProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose + val hubHubProducedSubBundles = io.hub.map{ io => { + io.getClass.getMethods.filter( x => + classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + + hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { + net.header.src := UFix(0) + net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.valid := hub.map(_.valid).fold(Bool(false))(_||_) + net.bits := hub(0).bits + hub.foreach( _.ready := net.ready) + }} + hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.header := net.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits := net.bits) + net.ready := hub.map(_.ready).fold(Bool(false))(_||_) + }} +} + abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLink }.flip + val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput val mem = new ioMem } } @@ -295,7 +333,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits - t.tile_incoherent := (Vec(io.tiles.map(_.incoherent)) { Bool() }).toBits + t.tile_incoherent := io.incoherent.toBits t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) From f7c01524099f1880ba395e15f9474c50b9d8272c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 15 Jan 2013 15:52:47 -0800 Subject: [PATCH 154/688] Refactored packet headers/payloads --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 92 ++++++++++++++----- uncore/src/tilelink.scala | 2 + uncore/src/uncore.scala | 179 +++++++++++++++++++------------------ 4 files changed, 164 insertions(+), 115 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 96fd475d..caa2b648 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgent -trait ClientCoherenceAgent extends CoherenceAgent -trait MasterCoherenceAgent extends CoherenceAgent +trait CoherenceAgentRole +trait ClientCoherenceAgent extends CoherenceAgentRole +trait MasterCoherenceAgent extends CoherenceAgentRole object cpuCmdToRW { def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 9698ffc3..3c2a93df 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,5 +1,7 @@ package object uncore { import Chisel._ +import Node._ +import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. @@ -11,32 +13,35 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UFix(width = conf.idBits) } -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new PhysicalHeader).asOutput +abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data } -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) +class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { + override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } +} abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } }.flip - val out = Vec(conf.nEndpoints) { (new BasicCrossbarIO) { data } } + val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} } val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new RRArbiter(conf.nEndpoints)(data) + val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits - rdy := arb.ready && (in.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - out.header.src := rrarb.io.chosen.toUFix - out.header.dst := UFix(i) + //out.bits.header.src := rrarb.io.chosen.toUFix + //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -45,7 +50,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgent])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) @@ -56,22 +61,63 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UFix(width = conf.idBits) } -abstract class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends FIFOIO()(data) { - val header = (new LogicalHeader).asOutput +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.in.ready := io.out.ready } -class TileIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf) -class HubIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends LogicalNetworkIO()(data)(conf){flip()} +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new TileIO) { new TransactionInit } - val xact_init_data = (new TileIO) { new TransactionInitData } - val xact_abort = (new HubIO) { new TransactionAbort } - val probe_req = (new HubIO) { new ProbeRequest } - val probe_rep = (new TileIO) { new ProbeReply } - val probe_rep_data = (new TileIO) { new ProbeReplyData } - val xact_rep = (new HubIO) { new TransactionReply } - val xact_finish = (new TileIO) { new TransactionFinish } + val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} + val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} + val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} + val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} + val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index a9cbc482..f07e7ed1 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -97,6 +97,7 @@ class TransactionFinish extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } +/* class ioTileLink extends Bundle { val xact_init = (new FIFOIO) { new TransactionInit } val xact_init_data = (new FIFOIO) { new TransactionInitData } @@ -108,3 +109,4 @@ class ioTileLink extends Bundle { val xact_finish = (new FIFOIO) { new TransactionFinish } val incoherent = Bool(OUTPUT) } +*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index daf756f0..697b632c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -4,12 +4,12 @@ import Chisel._ import Constants._ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { val xact_init = new TransactionInit() - val tile_id = Bits(width = conf.ln.idBits) + val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } @@ -17,7 +17,7 @@ class TrackerDependency extends Bundle { val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { +class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { val co = conf.co val io = new Bundle { val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip @@ -39,8 +39,8 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe_req = (new FIFOIO) { new ProbeRequest } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_tile_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_tile_id = Bits(OUTPUT, conf.ln.idBits) + val init_client_id = Bits(OUTPUT, conf.ln.idBits) + val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) @@ -54,7 +54,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val send_x_rep_ack = Bool(OUTPUT) } - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, tile_id: UFix) { + def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { req_cmd.bits.rw := Bool(true) req_data.bits := data.bits when(req_cmd.ready && req_cmd.valid) { @@ -66,11 +66,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com when (req_cmd.ready || cmd_sent) { req_data.valid := data.valid when(req_data.ready) { - pop_data := UFix(1) << tile_id + pop_data := UFix(1) << client_id when (data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << tile_id + pop_dep := UFix(1) << client_id trigger := Bool(false) } } @@ -90,10 +90,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new TransactionInit } - val init_tile_id_ = Reg{ Bits() } + val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_tile_id_ = Reg{ Bits() } + val p_rep_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val x_init_data_needs_write = Reg(resetVal = Bool(false)) val p_rep_data_needs_write = Reg(resetVal = Bool(false)) @@ -111,14 +111,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.tile_id(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr - io.init_tile_id := init_tile_id_ - io.p_rep_tile_id := p_rep_tile_id_ + io.init_client_id := init_client_id_ + io.p_rep_client_id := p_rep_client_id_ io.tile_xact_id := xact.tile_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.x_type := xact.x_type @@ -147,14 +147,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.xact_init - init_tile_id_ := io.alloc_req.bits.tile_id + init_client_id_ := io.alloc_req.bits.client_id x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.tile_id + io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { p_rep_count := PopCount(p_req_initial_flags) state := Mux(p_req_initial_flags.orR, s_probe, s_mem) @@ -179,7 +179,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } when(io.p_data.valid) { p_rep_data_needs_write := Bool(true) - p_rep_tile_id_ := io.p_data.bits.tile_id + p_rep_client_id_ := io.p_data.bits.client_id } } is(s_mem) { @@ -193,7 +193,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_p_rep_data, io.pop_p_rep_dep, io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), - p_rep_tile_id_) + p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -204,7 +204,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.pop_x_init_data, io.pop_x_init_dep, io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), - init_tile_id_) + init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { @@ -231,33 +231,33 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co val hub = Vec(conf.nTiles) { new TileLinkIO } } - val netTileProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[TileIO[Data]] } - val netHubProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[HubIO[Data]] } + val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } + val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - val hubTileProducedSubBundles = io.hub.map{ io => { + val hubClientProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[TileIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[TileIO[Data]] }}}.transpose - val hubHubProducedSubBundles = io.hub.map{ io => { + classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose + val hubMasterProducedSubBundles = io.hub.map{ io => { io.getClass.getMethods.filter( x => - classOf[HubIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[HubIO[Data]] }}}.transpose + classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => + m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - hubHubProducedSubBundles.zip(netHubProducedSubBundles).foreach{ case(hub, net) => { - net.header.src := UFix(0) - net.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { + net.bits.header.src := UFix(0) + net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} + net.bits.payload := hub(0).bits.payload net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - net.bits := hub(0).bits hub.foreach( _.ready := net.ready) }} - hubTileProducedSubBundles.zip(netTileProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.header := net.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits := net.bits) + hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { + hub.foreach(_.bits.header := net.bits.header) + hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } + hub.foreach(_.bits.payload := net.bits.payload) net.ready := hub.map(_.ready).fold(Bool(false))(_||_) }} } @@ -275,20 +275,20 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren val co = conf.co.asInstanceOf[ThreeStateIncoherence] val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.x_type === co.xactInitWriteback + val is_write = x_init.bits.payload.x_type === co.xactInitWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.tile_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.addr + io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.addr := x_init.bits.payload.addr io.mem.req_data <> io.tiles(0).xact_init_data val x_rep = io.tiles(0).xact_rep - x_rep.bits.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.tile_xact_id) - x_rep.bits.global_xact_id := UFix(0) // don't care - x_rep.bits.data := io.mem.resp.bits.data - x_rep.bits.require_ack := Bool(true) + x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) + x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) + x_rep.bits.payload.global_xact_id := UFix(0) // don't care + x_rep.bits.payload.data := io.mem.resp.bits.data + x_rep.bits.payload.require_ack := Bool(true) x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready io.tiles(0).xact_abort.valid := Bool(false) @@ -301,12 +301,13 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) { + implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } @@ -316,20 +317,20 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_tile_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } + val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io busy_arr(i) := t.busy addr_arr(i) := t.addr - init_tile_id_arr(i) := t.init_tile_id + init_client_id_arr(i) := t.init_client_id tile_xact_id_arr(i) := t.tile_xact_id x_type_arr(i) := t.x_type sh_count_arr(i) := t.sharer_count send_x_rep_ack_arr(i) := t.send_x_rep_ack t.xact_finish := do_free_arr(i) - t.p_data.bits.tile_id := p_data_tile_id_arr(i) + t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits @@ -337,7 +338,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.sent_x_rep_ack := sent_x_rep_ack_arr(i) do_free_arr(i) := Bool(false) sent_x_rep_ack_arr(i) := Bool(false) - p_data_tile_id_arr(i) := Bits(0, width = conf.ln.idBits) + p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { p_rep_cnt_dec_arr(i)(j) := Bool(false) @@ -352,7 +353,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val finish = io.tiles(j).xact_finish when (finish.valid) { - do_free_arr(finish.bits.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -363,28 +364,28 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { val rep = io.tiles(j).xact_rep - rep.bits.x_type := UFix(0) - rep.bits.tile_xact_id := UFix(0) - rep.bits.global_xact_id := UFix(0) - rep.bits.data := io.mem.resp.bits.data - rep.bits.require_ack := Bool(true) + rep.bits.payload.x_type := UFix(0) + rep.bits.payload.tile_xact_id := UFix(0) + rep.bits.payload.global_xact_id := UFix(0) + rep.bits.payload.data := io.mem.resp.bits.data + rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_tile_id_arr(mem_idx))) { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.global_xact_id := mem_idx + when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) + rep.bits.payload.global_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.global_xact_id := ack_idx - when (UFix(j) === init_tile_id_arr(ack_idx)) { + rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) + rep.bits.payload.global_xact_id := ack_idx + when (UFix(j) === init_client_id_arr(ack_idx)) { rep.valid := send_x_rep_ack_arr.toBits.orR sent_x_rep_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_tile_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -403,29 +404,29 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co for( j <- 0 until conf.ln.nTiles ) { val p_rep = io.tiles(j).probe_rep val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.global_xact_id + val idx = p_rep.bits.payload.global_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.global_xact_id + p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits)) { + when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) - p_data_tile_id_arr(idx) := UFix(j) + p_data_client_id_arr(idx) := UFix(j) } p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_tile_id).probe_rep_data.bits + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_tile_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) + trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.global_xact_id === UFix(i)) + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) } } @@ -441,16 +442,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.addr) + conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.tile_xact_id := x_init.bits.tile_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits))) + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits)) { + when(co.messageHasData(x_init.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send @@ -485,10 +486,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.bits - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_tile_id).xact_init_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_tile_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) + trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { val x_init = io.tiles(j).xact_init @@ -496,11 +497,11 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val x_init_data_dep = x_init_data_dep_list(j).io.deq val x_abort = io.tiles(j).xact_abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits - init_arb.io.in(j).bits.tile_id := UFix(j) + init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits) && (abort_state_arr(j) === s_idle) + x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) @@ -519,7 +520,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - p_req_arb_arr(j).io.out <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req } } From fb2644760ff155c4b4dc14e954d65b5d40157c40 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Jan 2013 23:57:35 -0800 Subject: [PATCH 155/688] single-ported coherence master --- uncore/src/package.scala | 6 +- uncore/src/uncore.scala | 344 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 3c2a93df..2e73a6d7 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -112,11 +112,11 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO) {(new LogicalNetworkIO){new ProbeRequest }} + val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} + val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO) {(new LogicalNetworkIO){new TransactionReply }} + val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 697b632c..179179e8 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -524,3 +524,347 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } } + +abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { + val io = new Bundle { + val network = (new TileLinkIO).flip + val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val mem = new ioMem + } +} + +class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +{ + implicit val lnConf = conf.ln + val co = conf.co + val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i) + t.io.tile_incoherent := io.incoherent.toBits + t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) + t.io.mem_resp.bits := io.mem.resp.bits + } + io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + + // Handle transaction initiation requests + // Only one allocation per cycle + // Init requests may or may not have data + val x_init = io.network.xact_init + val x_init_data = io.network.xact_init_data + val x_abort = io.network.xact_abort + val x_dep_deq = x_init_data_dep_q.io.deq + val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } + val abort_state = Reg(resetVal = s_idle) + val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + + val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } + for( i <- 0 until NGLOBAL_XACTS ) { + alloc_arb.io.in(i).valid := !trackerList(i).io.busy + trackerList(i).io.x_init.bits := x_init.bits + trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + + trackerList(i).io.x_init_data.bits := x_init_data.bits + trackerList(i).io.x_init_data.valid := x_init_data.valid + trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits + trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + } + val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) + x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init + x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) + x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) + x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + + alloc_arb.io.out.ready := x_init.valid + + // Nack conflicting transaction init attempts + x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.valid := Bool(false) + switch(abort_state) { + is(s_idle) { + when(want_to_abort) { + abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + } + } + is(s_abort_drain) { // raises x_init_data.ready below + when(x_init_data.valid) { + abort_cnt := abort_cnt + UFix(1) + when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { + abort_state := s_abort_send + } + } + } + is(s_abort_send) { // nothing is dequeued for now + x_abort.valid := Bool(true) + when(x_abort.ready) { // raises x_init.ready + abort_state := s_idle + } + } + } + + // Handle probe request generation + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + p_req_arb.io.in(i).bits := t.p_req.bits + p_req_arb.io.in(i).valid := t.p_req.valid + t.p_req.ready := p_req_arb.io.in(i).ready + } + io.network.probe_req <> p_req_arb.io.out + + // Handle probe replies, which may or may not have data + val p_rep = io.network.probe_rep + val p_rep_data = io.network.probe_rep_data + val idx = p_rep.bits.payload.global_xact_id + p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) + p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) + p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) + p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.p_rep_data.valid := p_rep_data.valid + trackerList(i).io.p_rep_data.bits := p_rep_data.bits + trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid + trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits + trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) + trackerList(i).io.p_rep.bits := p_rep.bits + } + + // Reply to initial requestor + // Forward memory responses from mem to tile or arbitrate to ack + val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + for( i <- 0 until NGLOBAL_XACTS ) { + val t = trackerList(i).io + x_rep_arb.io.in(i).bits := t.x_rep.bits + x_rep_arb.io.in(i).valid := t.x_rep.valid + t.x_rep.ready := x_rep_arb.io.in(i).ready + } + x_rep_arb.io.out.ready := Bool(false) + io.network.xact_rep.valid := x_rep_arb.io.out.valid + io.network.xact_rep.bits := x_rep_arb.io.out.bits + x_rep_arb.io.out.ready := io.network.xact_rep.ready + when(io.mem.resp.valid) { + io.network.xact_rep.valid := Bool(true) + io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + } + } + + // Free finished transactions + val finish = io.network.xact_finish + for( i <- 0 until NGLOBAL_XACTS ) { + trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + } + finish.ready := Bool(true) + + // Create an arbiter for the one memory port + // We have to arbitrate between the different trackers' memory requests + // and once we have picked a request, get the right write data + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } + for( i <- 0 until NGLOBAL_XACTS ) { + mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd + mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data + mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock + } + io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) + io.mem.req_data <> Queue(mem_req_data_arb.io.out) +} + +class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + val co = conf.co + implicit val ln = conf.ln + val io = new Bundle { + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val free = Bool(INPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val mem_resp = (new FIFOIO) { new MemResp }.flip + + val mem_req_cmd = (new FIFOIO) { new MemReqCmd } + val mem_req_data = (new FIFOIO) { new MemData } + val mem_req_lock = Bool(OUTPUT) + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} + val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val busy = Bool(OUTPUT) + val has_conflict = Bool(OUTPUT) + } + + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new TransactionInit } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + //TODO: Will need id reg for merged release xacts + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val x_needs_read = Reg(resetVal = Bool(false)) + val x_init_data_needs_write = Reg(resetVal = Bool(false)) + val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) + val p_req_initial_flags = Bits(width = conf.ln.nTiles) + p_req_initial_flags := Bits(0) + if (conf.ln.nTiles > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + // TODO: this is hackish; figure out how to do it more systematically + val probe_self = co match { + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case _ => Bool(false) + } + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + p_req_initial_flags := ~(io.tile_incoherent | myflag) + } + val all_x_reps_require_acks = Bool(true) + + io.busy := state != s_idle + io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.p_req.valid := Bool(false) + io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) + io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.addr := xact.addr + io.p_req.bits.header.dst := UFix(0) + io.x_rep.bits.payload.data := io.mem_resp.bits.data + io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) + io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id + io.x_rep.bits.payload.global_xact_id := UFix(id) + io.x_rep.bits.payload.require_ack := all_x_reps_require_acks + io.x_rep.bits.header.dst := init_client_id_ + io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.x_init.ready := Bool(false) + io.x_init_data.ready := Bool(false) + io.x_init_data_dep.ready := Bool(false) + io.p_rep.ready := Bool(false) + io.p_rep_data.ready := Bool(false) + io.p_rep_data_dep.ready := Bool(false) + io.mem_resp.ready := io.x_rep.ready + + switch (state) { + is(s_idle) { + when( io.x_init.valid ) { + xact := io.x_init.bits.payload + init_client_id_ := io.x_init.bits.header.src + init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + p_req_flags := p_req_initial_flags + mem_cnt := UFix(0) + p_w_mem_cmd_sent := Bool(false) + x_w_mem_cmd_sent := Bool(false) + io.x_init.ready := Bool(true) + if(conf.ln.nTiles > 1) { + p_rep_count := PopCount(p_req_initial_flags) + state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + } else state := s_mem + } + } + is(s_probe) { + val curr_p_id = PriorityEncoder(p_req_flags) + when(p_req_flags.orR) { + io.p_req.valid := Bool(true) + io.p_req.bits.header.dst := curr_p_id + } + when(io.p_req.ready) { + p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + } + when(io.p_rep.valid) { + io.p_rep.ready := Bool(true) + if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) + when(p_rep_count === UFix(1)) { + state := s_mem + } + p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + } + } + is(s_mem) { + when (p_rep_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.p_rep_data, + p_rep_data_needs_write, + p_w_mem_cmd_sent, + io.p_rep_data_dep.ready, + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen(x_init_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.x_init_data, + x_init_data_needs_write, + x_w_mem_cmd_sent, + io.x_init_data_dep.ready, + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + } . elsewhen (x_needs_read) { + doMemReqRead(io.mem_req_cmd, x_needs_read) + } . otherwise { + state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, + Mux(all_x_reps_require_acks, s_busy, s_idle)) + } + } + is(s_ack) { + io.x_rep.valid := Bool(true) + when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when (io.free) { + state := s_idle + } + } + } + + def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + req_cmd.bits.rw := Bool(true) + req_data.bits := data.bits.payload + when(req_cmd.ready && req_cmd.valid) { + cmd_sent := Bool(true) + } + when (at_front_of_dep_queue) { + req_cmd.valid := !cmd_sent && req_data.ready && data.valid + lock := data.valid || cmd_sent + when (req_cmd.ready || cmd_sent) { + req_data.valid := data.valid + when(req_data.ready) { + data.ready:= Bool(true) + when (data.valid) { + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + pop_dep := Bool(true) + trigger := Bool(false) + } + } + } + } + } + } + + def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { + req_cmd.valid := Bool(true) + req_cmd.bits.rw := Bool(false) + when(req_cmd.ready) { + trigger := Bool(false) + } + } +} From c211d74e95ed1561cca1827650b93087b76bcc23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 21 Jan 2013 17:17:26 -0800 Subject: [PATCH 156/688] New TileLink names --- uncore/src/coherence.scala | 898 ++++++++++++++++++------------------- uncore/src/consts.scala | 20 +- uncore/src/package.scala | 16 +- uncore/src/tilelink.scala | 114 ++--- uncore/src/uncore.scala | 286 ++++++------ 5 files changed, 668 insertions(+), 666 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index caa2b648..4268fcf8 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -15,7 +15,7 @@ abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool def needsWriteback (state: UFix): Bool @@ -23,71 +23,71 @@ abstract class CoherencePolicy { def newStateOnCacheControl(cmd: Bits): UFix def newStateOnWriteback(): UFix def newStateOnFlush(): UFix - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix + def newStateOnProbe(incoming: Probe, state: UFix): Bits - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits - def getTransactionInitTypeOnWriteback(): Bits + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getAcquireTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnWriteback(): Bits - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply + def newRelease (incoming: Probe, state: UFix): Release - def messageHasData (reply: ProbeReply): Bool - def messageHasData (init: TransactionInit): Bool - def messageHasData (reply: TransactionReply): Bool - def messageUpdatesDataArray (reply: TransactionReply): Bool - def messageIsUncached(init: TransactionInit): Bool + def messageHasData (reply: Release): Bool + def messageHasData (acq: Acquire): Bool + def messageHasData (reply: Grant): Bool + def messageUpdatesDataArray (reply: Grant): Bool + def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool - def getTransactionReplyType(x_type: UFix, count: UFix): Bits - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix - def needsMemRead(x_type: UFix, global_state: UFix): Bool - def needsMemWrite(x_type: UFix, global_state: UFix): Bool - def needsAckReply(x_type: UFix, global_state: UFix): Bool + def getGrantType(a_type: UFix, count: UFix): Bits + def getProbeType(a_type: UFix, global_state: UFix): UFix + def needsMemRead(a_type: UFix, global_state: UFix): Bool + def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsAckReply(a_type: UFix, global_state: UFix): Bool } trait UncachedTransactions { - def getUncachedReadTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix): TransactionInit - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits): TransactionInit - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): TransactionInit - def isUncachedReadTransaction(xinit: TransactionInit): Bool + def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire + def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def isUncachedReadTransaction(acq: Acquire): Bool } abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = state - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - reply.p_type := UFix(0) - reply.global_xact_id := UFix(0) + def newStateOnProbe(incoming: Probe, state: UFix): Bits = state + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + reply.r_type := UFix(0) + reply.master_xact_id := UFix(0) reply } - def messageHasData (reply: ProbeReply) = Bool(false) + def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = Bits(0) - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(x_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val xactInitReadClean :: xactInitReadDirty :: xactInitWriteback :: Nil = Enum(3){ UFix() } - val xactReplyData :: xactReplyAck :: Nil = Enum(2){ UFix() } - val probeRepInvalidateAck :: Nil = Enum(1){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } + val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val releaseInvalidateAck :: Nil = Enum(1){ UFix() } val uncachedTypeList = List() - val hasDataTypeList = List(xactInitWriteback) + val hasDataTypeList = List(acquireWriteback) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit) = Bool(false) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty def needsWriteback (state: UFix): Bool = state === tileDirty @@ -99,28 +99,28 @@ class ThreeStateIncoherence extends IncoherentPolicy { def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit) = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyData -> Mux(outstanding.x_type === xactInitReadDirty, tileDirty, tileClean), - xactReplyAck -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), + grantAck -> tileInvalid )) } - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadDirty, xactInitReadClean) + Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadDirty, outstanding.x_type) + Mux(write, acquireReadDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteback //TODO - def getTransactionInitTypeOnWriteback(): Bits = xactInitWriteback + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO + def getAcquireTypeOnWriteback(): Bits = acquireWriteback - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageUpdatesDataArray (reply: TransactionReply) = (reply.x_type === xactReplyData) - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) } class MICoherence extends CoherencePolicyWithUncached { @@ -128,17 +128,17 @@ class MICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } - val xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(6){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(6){ UFix() } - val probeReqInvalidate :: probeReqCopy :: Nil = Enum(2){ UFix() } - val probeRepInvalidateData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepCopyAck :: Nil = Enum(4){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = (outstanding.x_type != xactInitReadExclusive) + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), @@ -158,95 +158,95 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> tileValid, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> tileValid, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = xactInitReadExclusive - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = xactInitReadExclusive - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -255,20 +255,20 @@ class MEICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } - val xactInitReadExclusiveClean :: xactInitReadExclusiveDirty :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(7){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusiveDirty)) + (write && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -292,108 +292,108 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileExclusiveClean, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileExclusiveClean, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, xactInitReadExclusiveClean) + Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusiveDirty, outstanding.x_type) + Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadExclusiveClean -> xactReplyReadExclusive, - xactInitReadExclusiveDirty -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadExclusiveClean -> grantReadExclusive, + acquireReadExclusiveDirty -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadExclusiveClean -> probeReqInvalidate, - xactInitReadExclusiveDirty -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadExclusiveClean -> probeInvalidate, + acquireReadExclusiveDirty -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -402,12 +402,12 @@ class MSICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactReplyReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -418,10 +418,10 @@ class MSICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -445,106 +445,106 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> tileExclusiveDirty, - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> tileExclusiveDirty, + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -553,12 +553,12 @@ class MESICoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: Nil = Enum(7){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: Nil = Enum(8){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: Nil = Enum(3){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: Nil = Enum(6){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -569,10 +569,10 @@ class MESICoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive)) + (write && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -596,109 +596,109 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> Mux(outstanding.x_type === xactInitReadExclusive, tileExclusiveDirty, tileExclusiveClean), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqDowngrade -> tileShared, - probeReqCopy -> state + probeInvalidate -> tileInvalid, + probeDowngrade -> tileShared, + probeCopy -> state )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, xactInitReadExclusive, xactInitReadShared) + Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, xactInitReadExclusive, outstanding.x_type) + Mux(write, acquireReadExclusive, outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> probeRepInvalidateData, - probeReqDowngrade -> probeRepDowngradeData, - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + val reply = new Release + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> releaseInvalidateData, + probeDowngrade -> releaseDowngradeData, + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> probeRepInvalidateAck, - probeReqDowngrade -> probeRepDowngradeAck, - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> releaseInvalidateAck, + probeDowngrade -> releaseDowngradeAck, + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - (reply.p_type === probeRepInvalidateData || - reply.p_type === probeRepDowngradeData || - reply.p_type === probeRepCopyData) + def messageHasData (reply: Release): Bool = { + (reply.r_type === releaseInvalidateData || + reply.r_type === releaseDowngradeData || + reply.r_type === releaseCopyData) } - def messageHasData (init: TransactionInit): Bool = hasDataTypeList.map(t => init.x_type === t).reduceLeft(_||_) - def messageHasData (reply: TransactionReply): Bool = { - (reply.x_type != xactReplyWriteUncached && reply.x_type != xactReplyReadExclusiveAck && reply.x_type != xactReplyWriteWordUncached) + def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageHasData (reply: Grant): Bool = { + (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - (reply.x_type === xactReplyReadShared || reply.x_type === xactReplyReadExclusive) + def messageUpdatesDataArray (reply: Grant): Bool = { + (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(init: TransactionInit): Bool = uncachedTypeList.map(t => init.x_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached) } } @@ -706,12 +706,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } - val xactInitReadShared :: xactInitReadExclusive :: xactInitReadUncached :: xactInitWriteUncached :: xactInitReadWordUncached :: xactInitWriteWordUncached :: xactInitAtomicUncached :: xactInitInvalidateOthers :: Nil = Enum(8){ UFix() } - val xactReplyReadShared :: xactReplyReadExclusive :: xactReplyReadUncached :: xactReplyWriteUncached :: xactReplyReadExclusiveAck :: xactReplyReadWordUncached :: xactReplyWriteWordUncached :: xactReplyAtomicUncached :: xactReplyReadMigratory :: Nil = Enum(9){ UFix() } - val probeReqInvalidate :: probeReqDowngrade :: probeReqCopy :: probeReqInvalidateOthers :: Nil = Enum(4){ UFix() } - val probeRepInvalidateData :: probeRepDowngradeData :: probeRepCopyData :: probeRepInvalidateAck :: probeRepDowngradeAck :: probeRepCopyAck :: probeRepDowngradeDataMigratory :: probeRepDowngradeAckHasCopy :: probeRepInvalidateDataMigratory :: probeRepInvalidateAckMigratory :: Nil = Enum(10){ UFix() } - val uncachedTypeList = List(xactInitReadUncached, xactInitWriteUncached, xactInitReadWordUncached, xactInitWriteWordUncached, xactInitAtomicUncached) - val hasDataTypeList = List(xactInitWriteUncached, xactInitWriteWordUncached, xactInitAtomicUncached) + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } + val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) @@ -723,10 +723,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: TransactionInit): Bool = { + def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { val (read, write) = cpuCmdToRW(cmd) (read && messageIsUncached(outstanding)) || - (write && (outstanding.x_type != xactInitReadExclusive && outstanding.x_type != xactInitInvalidateOthers)) + (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -752,30 +752,30 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnTransactionReply(incoming: TransactionReply, outstanding: TransactionInit): UFix = { - MuxLookup(incoming.x_type, tileInvalid, Array( - xactReplyReadShared -> tileShared, - xactReplyReadExclusive -> MuxLookup(outstanding.x_type, tileExclusiveDirty, Array( - xactInitReadExclusive -> tileExclusiveDirty, - xactInitReadShared -> tileExclusiveClean)), - xactReplyReadExclusiveAck -> tileExclusiveDirty, - xactReplyReadUncached -> tileInvalid, - xactReplyWriteUncached -> tileInvalid, - xactReplyReadWordUncached -> tileInvalid, - xactReplyWriteWordUncached -> tileInvalid, - xactReplyAtomicUncached -> tileInvalid, - xactReplyReadMigratory -> MuxLookup(outstanding.x_type, tileMigratoryDirty, Array( - xactInitInvalidateOthers -> tileMigratoryDirty, - xactInitReadExclusive -> tileMigratoryDirty, - xactInitReadShared -> tileMigratoryClean)) + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + MuxLookup(incoming.g_type, tileInvalid, Array( + grantReadShared -> tileShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( + acquireReadExclusive -> tileExclusiveDirty, + acquireReadShared -> tileExclusiveClean)), + grantReadExclusiveAck -> tileExclusiveDirty, + grantReadUncached -> tileInvalid, + grantWriteUncached -> tileInvalid, + grantReadWordUncached -> tileInvalid, + grantWriteWordUncached -> tileInvalid, + grantAtomicUncached -> tileInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( + acquireInvalidateOthers -> tileMigratoryDirty, + acquireReadExclusive -> tileMigratoryDirty, + acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbeRequest(incoming: ProbeRequest, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UFix): Bits = { MuxLookup(incoming.p_type, state, Array( - probeReqInvalidate -> tileInvalid, - probeReqInvalidateOthers -> tileInvalid, - probeReqCopy -> state, - probeReqDowngrade -> MuxLookup(state, tileShared, Array( + probeInvalidate -> tileInvalid, + probeInvalidateOthers -> tileInvalid, + probeCopy -> state, + probeDowngrade -> MuxLookup(state, tileShared, Array( tileExclusiveClean -> tileSharedByTwo, tileExclusiveDirty -> tileSharedByTwo, tileSharedByTwo -> tileShared, @@ -784,91 +784,91 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadUncached, addr, id) - def getUncachedWriteTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitWriteUncached, addr, id) - def getUncachedReadWordTransactionInit(addr: UFix, id: UFix) = TransactionInit(xactInitReadWordUncached, addr, id) - def getUncachedWriteWordTransactionInit(addr: UFix, id: UFix, write_mask: Bits) = TransactionInit(xactInitWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicTransactionInit(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = TransactionInit(xactInitAtomicUncached, addr, id, subword_addr, atomic_op) - def isUncachedReadTransaction(xinit: TransactionInit) = xinit.x_type === xactInitReadUncached + def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) + def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) + def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) + def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) + def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached - def getTransactionInitTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), xactInitReadShared) + Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getTransactionInitTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: TransactionInit): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, xactInitReadExclusive, xactInitInvalidateOthers), outstanding.x_type) + Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getTransactionInitTypeOnCacheControl(cmd: Bits): Bits = xactInitWriteUncached - def getTransactionInitTypeOnWriteback(): Bits = getTransactionInitTypeOnCacheControl(M_INV) + def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached + def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) - def newProbeReply (incoming: ProbeRequest, state: UFix): ProbeReply = { - Assert( incoming.p_type === probeReqInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new ProbeReply() - val with_data = MuxLookup(incoming.p_type, probeRepInvalidateData, Array( - probeReqInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), - probeRepInvalidateDataMigratory, probeRepInvalidateData), - probeReqDowngrade -> Mux(state === tileMigratoryDirty, probeRepDowngradeDataMigratory, probeRepDowngradeData), - probeReqCopy -> probeRepCopyData + def newRelease (incoming: Probe, state: UFix): Release = { + Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + val reply = new Release() + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) - val without_data = MuxLookup(incoming.p_type, probeRepInvalidateAck, Array( - probeReqInvalidate -> Mux(tileExclusiveClean === state, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqInvalidateOthers -> Mux(state === tileSharedByTwo, probeRepInvalidateAckMigratory, probeRepInvalidateAck), - probeReqDowngrade -> Mux(state != tileInvalid, probeRepDowngradeAckHasCopy, probeRepDowngradeAck), - probeReqCopy -> probeRepCopyAck + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck )) - reply.p_type := Mux(needsWriteback(state), with_data, without_data) - reply.global_xact_id := incoming.global_xact_id + reply.r_type := Mux(needsWriteback(state), with_data, without_data) + reply.master_xact_id := incoming.master_xact_id reply } - def messageHasData (reply: ProbeReply): Bool = { - uFixListContains(List(probeRepInvalidateData, probeRepDowngradeData, probeRepCopyData, probeRepInvalidateDataMigratory, probeRepDowngradeDataMigratory), reply.p_type) + def messageHasData (reply: Release): Bool = { + uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) } - def messageHasData (init: TransactionInit): Bool = uFixListContains(hasDataTypeList, init.x_type) - def messageHasData (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadUncached, xactReplyReadMigratory, xactReplyReadWordUncached, xactReplyAtomicUncached), reply.x_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) + def messageHasData (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) } - def messageUpdatesDataArray (reply: TransactionReply): Bool = { - uFixListContains(List(xactReplyReadShared, xactReplyReadExclusive, xactReplyReadMigratory), reply.x_type) + def messageUpdatesDataArray (reply: Grant): Bool = { + uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(init: TransactionInit): Bool = uFixListContains(uncachedTypeList, init.x_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getTransactionReplyType(x_type: UFix, count: UFix): Bits = { - MuxLookup(x_type, xactReplyReadUncached, Array( - xactInitReadShared -> Mux(count > UFix(0), xactReplyReadShared, xactReplyReadExclusive), //TODO: what is count? Depend on probeRep.p_type??? - xactInitReadExclusive -> xactReplyReadExclusive, - xactInitReadUncached -> xactReplyReadUncached, - xactInitWriteUncached -> xactReplyWriteUncached, - xactInitReadWordUncached -> xactReplyReadWordUncached, - xactInitWriteWordUncached -> xactReplyWriteWordUncached, - xactInitAtomicUncached -> xactReplyAtomicUncached, - xactInitInvalidateOthers -> xactReplyReadExclusiveAck //TODO: add this to MESI? + def getGrantType(a_type: UFix, count: UFix): Bits = { + MuxLookup(a_type, grantReadUncached, Array( + acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadExclusive -> grantReadExclusive, + acquireReadUncached -> grantReadUncached, + acquireWriteUncached -> grantWriteUncached, + acquireReadWordUncached -> grantReadWordUncached, + acquireWriteWordUncached -> grantWriteWordUncached, + acquireAtomicUncached -> grantAtomicUncached, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getProbeRequestType(x_type: UFix, global_state: UFix): UFix = { - MuxLookup(x_type, probeReqCopy, Array( - xactInitReadShared -> probeReqDowngrade, - xactInitReadExclusive -> probeReqInvalidate, - xactInitReadUncached -> probeReqCopy, - xactInitWriteUncached -> probeReqInvalidate, - xactInitReadWordUncached -> probeReqCopy, - xactInitWriteWordUncached -> probeReqInvalidate, - xactInitAtomicUncached -> probeReqInvalidate, - xactInitInvalidateOthers -> probeReqInvalidateOthers + def getProbeType(a_type: UFix, global_state: UFix): UFix = { + MuxLookup(a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireReadUncached -> probeCopy, + acquireWriteUncached -> probeInvalidate, + acquireReadWordUncached -> probeCopy, + acquireWriteWordUncached -> probeInvalidate, + acquireAtomicUncached -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers )) } - def needsMemRead(x_type: UFix, global_state: UFix): Bool = { - (x_type != xactInitWriteUncached && x_type != xactInitInvalidateOthers) + def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached || x_type === xactInitAtomicUncached) + def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(x_type: UFix, global_state: UFix): Bool = { - (x_type === xactInitWriteUncached || x_type === xactInitWriteWordUncached ||x_type === xactInitInvalidateOthers) + def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } } diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 0b885352..b515c1c8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val GLOBAL_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -20,19 +20,19 @@ trait CacheConstants extends UncoreConstants { } trait TileLinkTypeConstants { - val X_INIT_TYPE_MAX_BITS = 2 - val X_REP_TYPE_MAX_BITS = 3 - val P_REQ_TYPE_MAX_BITS = 2 - val P_REP_TYPE_MAX_BITS = 3 + val ACQUIRE_TYPE_MAX_BITS = 2 + val GRANT_TYPE_MAX_BITS = 3 + val PROBE_TYPE_MAX_BITS = 2 + val RELEASE_TYPE_MAX_BITS = 3 } trait TileLinkSizeConstants extends TileLinkTypeConstants { - val TILE_XACT_ID_BITS = 5 - val X_INIT_WRITE_MASK_BITS = 6 - val X_INIT_SUBWORD_ADDR_BITS = 3 - val X_INIT_ATOMIC_OP_BITS = 4 + val CLIENT_XACT_ID_BITS = 5 + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 } trait MemoryOpConstants { @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(TILE_XACT_ID_BITS, GLOBAL_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 2e73a6d7..0e69ff47 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -110,14 +110,14 @@ class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val xact_init = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInit }} - val xact_init_data = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionInitData }} - val xact_abort = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionAbort }} - val probe_req = (new MasterSourcedIO){(new LogicalNetworkIO){new ProbeRequest }} - val probe_rep = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReply }} - val probe_rep_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ProbeReplyData }} - val xact_rep = (new MasterSourcedIO){(new LogicalNetworkIO){new TransactionReply }} - val xact_finish = (new ClientSourcedIO){(new LogicalNetworkIO){new TransactionFinish }} + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index f07e7ed1..8ff26adc 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -32,81 +32,83 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class TransactionInit extends PhysicalAddress { - val x_type = Bits(width = X_INIT_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val write_mask = Bits(width = X_INIT_WRITE_MASK_BITS) - val subword_addr = Bits(width = X_INIT_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = X_INIT_ATOMIC_OP_BITS) +class Acquire extends PhysicalAddress { + val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -object TransactionInit +object Acquire { - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, write_mask: Bits) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.write_mask := write_mask - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.write_mask := write_mask + acq } - def apply(x_type: Bits, addr: UFix, tile_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { - val init = new TransactionInit - init.x_type := x_type - init.addr := addr - init.tile_xact_id := tile_xact_id - init.subword_addr := subword_addr - init.atomic_opcode := atomic_opcode - init + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + val acq = new Acquire + acq.a_type := a_type + acq.addr := addr + acq.client_xact_id := client_xact_id + acq.subword_addr := subword_addr + acq.atomic_opcode := atomic_opcode + acq } } -class TransactionInitData extends MemData +class AcquireData extends MemData -class TransactionAbort extends Bundle { - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) +class Abort extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) } -class ProbeRequest extends PhysicalAddress { - val p_type = Bits(width = P_REQ_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Probe extends PhysicalAddress { + val p_type = Bits(width = PROBE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReply extends Bundle { - val p_type = Bits(width = P_REP_TYPE_MAX_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Release extends Bundle { + val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ProbeReplyData extends MemData +class ReleaseData extends MemData -class TransactionReply extends MemData { - val x_type = Bits(width = X_REP_TYPE_MAX_BITS) - val tile_xact_id = Bits(width = TILE_XACT_ID_BITS) - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class Grant extends MemData { + val g_type = Bits(width = GRANT_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) val require_ack = Bool() } -class TransactionFinish extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) +class GrantAck extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -/* -class ioTileLink extends Bundle { - val xact_init = (new FIFOIO) { new TransactionInit } - val xact_init_data = (new FIFOIO) { new TransactionInitData } - val xact_abort = (new FIFOIO) { new TransactionAbort }.flip - val probe_req = (new FIFOIO) { new ProbeRequest }.flip - val probe_rep = (new FIFOIO) { new ProbeReply } - val probe_rep_data = (new FIFOIO) { new ProbeReplyData } - val xact_rep = (new FIFOIO) { new TransactionReply }.flip - val xact_finish = (new FIFOIO) { new TransactionFinish } - val incoherent = Bool(OUTPUT) +abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) +class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} + val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} + val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} + val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} + val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -*/ diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 179179e8..690454a1 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -8,13 +8,13 @@ class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle } class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { - val xact_init = new TransactionInit() + val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val global_xact_id = Bits(width = GLOBAL_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { @@ -23,27 +23,27 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) - val xact_finish = Bool(INPUT) + val grant_ack = Bool(INPUT) val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ProbeReplyData }.flip - val x_init_data = (new PipeIO) { new TransactionInitData }.flip - val sent_x_rep_ack = Bool(INPUT) + val p_rep_data = (new PipeIO) { new ReleaseData }.flip + val x_init_data = (new PipeIO) { new AcquireData }.flip + val sent_grant_ack = Bool(INPUT) val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val probe_req = (new FIFOIO) { new ProbeRequest } + val probe = (new FIFOIO) { new Probe } val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) - val tile_xact_id = Bits(OUTPUT, TILE_XACT_ID_BITS) + val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val x_type = Bits(OUTPUT, X_INIT_TYPE_MAX_BITS) + val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) val push_p_req = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) @@ -51,7 +51,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) - val send_x_rep_ack = Bool(OUTPUT) + val send_grant_ack = Bool(OUTPUT) } def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { @@ -89,7 +89,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) @@ -108,7 +108,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.xact_init) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) @@ -119,9 +119,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.addr := xact.addr io.init_client_id := init_client_id_ io.p_rep_client_id := p_rep_client_id_ - io.tile_xact_id := xact.tile_xact_id + io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only - io.x_type := xact.x_type + io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) @@ -130,10 +130,10 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.probe_req.valid := Bool(false) - io.probe_req.bits.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.probe_req.bits.global_xact_id := UFix(id) - io.probe_req.bits.addr := xact.addr + io.probe.valid := Bool(false) + io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.master_xact_id := UFix(id) + io.probe.bits.addr := xact.addr io.push_p_req := Bits(0, width = conf.ln.nTiles) io.pop_p_rep := Bits(0, width = conf.ln.nTiles) io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) @@ -141,15 +141,15 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.pop_x_init := Bits(0, width = conf.ln.nTiles) io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) - io.send_x_rep_ack := Bool(false) + io.send_grant_ack := Bool(false) switch (state) { is(s_idle) { when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.xact_init + xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.xact_init) - x_needs_read := co.needsMemRead(io.alloc_req.bits.xact_init.x_type, UFix(0)) + x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -164,7 +164,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex is(s_probe) { when(p_req_flags.orR) { io.push_p_req := p_req_flags - io.probe_req.valid := Bool(true) + io.probe.valid := Bool(true) } when(io.p_req_cnt_inc.orR) { p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs @@ -192,7 +192,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent, io.pop_p_rep_data, io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id)), + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), p_rep_client_id_) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, @@ -203,20 +203,20 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex x_w_mem_cmd_sent, io.pop_x_init_data, io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id)), + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, s_busy) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) } } is(s_ack) { - io.send_x_rep_ack := Bool(true) - when(io.sent_x_rep_ack) { state := s_busy } + io.send_grant_ack := Bool(true) + when(io.sent_grant_ack) { state := s_busy } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.xact_finish) { + when (io.grant_ack) { state := s_idle } } @@ -274,28 +274,28 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).xact_init - val is_write = x_init.bits.payload.x_type === co.xactInitWriteback + val x_init = io.tiles(0).acquire + val is_write = x_init.bits.payload.a_type === co.acquireWriteback x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.tile_xact_id + io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id io.mem.req_cmd.bits.addr := x_init.bits.payload.addr - io.mem.req_data <> io.tiles(0).xact_init_data + io.mem.req_data <> io.tiles(0).acquire_data - val x_rep = io.tiles(0).xact_rep - x_rep.bits.payload.x_type := Mux(io.mem.resp.valid, co.xactReplyData, co.xactReplyAck) - x_rep.bits.payload.tile_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.tile_xact_id) - x_rep.bits.payload.global_xact_id := UFix(0) // don't care - x_rep.bits.payload.data := io.mem.resp.bits.data - x_rep.bits.payload.require_ack := Bool(true) - x_rep.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + val grant = io.tiles(0).grant + grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.master_xact_id := UFix(0) // don't care + grant.bits.payload.data := io.mem.resp.bits.data + grant.bits.payload.require_ack := Bool(true) + grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).xact_abort.valid := Bool(false) - io.tiles(0).xact_finish.ready := Bool(true) - io.tiles(0).probe_req.valid := Bool(false) - io.tiles(0).probe_rep.ready := Bool(true) - io.tiles(0).probe_rep_data.ready := Bool(true) + io.tiles(0).abort.valid := Bool(false) + io.tiles(0).grant_ack.ready := Bool(true) + io.tiles(0).probe.valid := Bool(false) + io.tiles(0).release.ready := Bool(true) + io.tiles(0).release_data.ready := Bool(true) } @@ -308,15 +308,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val tile_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=TILE_XACT_ID_BITS) } - val x_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=X_INIT_TYPE_MAX_BITS) } + val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } + val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val sent_x_rep_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } + val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -325,19 +325,19 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co busy_arr(i) := t.busy addr_arr(i) := t.addr init_client_id_arr(i) := t.init_client_id - tile_xact_id_arr(i) := t.tile_xact_id - x_type_arr(i) := t.x_type + client_xact_id_arr(i) := t.client_xact_id + a_type_arr(i) := t.a_type sh_count_arr(i) := t.sharer_count - send_x_rep_ack_arr(i) := t.send_x_rep_ack - t.xact_finish := do_free_arr(i) + send_grant_ack_arr(i) := t.send_grant_ack + t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits - t.sent_x_rep_ack := sent_x_rep_ack_arr(i) + t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) - sent_x_rep_ack_arr(i) := Bool(false) + sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { @@ -351,9 +351,9 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).xact_finish + val finish = io.tiles(j).grant_ack when (finish.valid) { - do_free_arr(finish.bits.payload.global_xact_id) := Bool(true) + do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) } finish.ready := Bool(true) } @@ -361,31 +361,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_x_rep_ack_arr.toBits) + val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) for( j <- 0 until conf.ln.nTiles ) { - val rep = io.tiles(j).xact_rep - rep.bits.payload.x_type := UFix(0) - rep.bits.payload.tile_xact_id := UFix(0) - rep.bits.payload.global_xact_id := UFix(0) + val rep = io.tiles(j).grant + rep.bits.payload.g_type := UFix(0) + rep.bits.payload.client_xact_id := UFix(0) + rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(mem_idx) - rep.bits.payload.global_xact_id := mem_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) + rep.bits.payload.master_xact_id := mem_idx rep.valid := Bool(true) } . otherwise { - rep.bits.payload.x_type := co.getTransactionReplyType(x_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.tile_xact_id := tile_xact_id_arr(ack_idx) - rep.bits.payload.global_xact_id := ack_idx + rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) + rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) + rep.bits.payload.master_xact_id := ack_idx when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_x_rep_ack_arr.toBits.orR - sent_x_rep_ack_arr(ack_idx) := rep.ready + rep.valid := send_grant_ack_arr.toBits.orR + sent_grant_ack_arr(ack_idx) := rep.ready } } } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).xact_rep.ready + io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -402,14 +402,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).probe_rep - val p_rep_data = io.tiles(j).probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.tiles(j).release + val p_rep_data = io.tiles(j).release_data + val idx = p_rep.bits.payload.master_xact_id val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) val do_pop = foldR(pop_p_reps)(_ || _) p_rep.ready := Bool(true) p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { p_data_valid_arr(idx) := Bool(true) @@ -418,15 +418,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).probe_rep_data.bits.payload + trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid + trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).probe_rep - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.global_xact_id === UFix(i)) + val p_rep = io.tiles(j).release + p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) } } @@ -435,16 +435,16 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data - val x_abort = io.tiles(j).xact_abort + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data + val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -486,23 +486,23 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).xact_init_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).xact_init_data.valid + trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).xact_init - val x_init_data = io.tiles(j).xact_init_data + val x_init = io.tiles(j).acquire + val x_init_data = io.tiles(j).acquire_data val x_init_data_dep = x_init_data_dep_list(j).io.deq - val x_abort = io.tiles(j).xact_abort + val x_abort = io.tiles(j).abort init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.xact_init := x_init.bits.payload + init_arb.io.in(j).bits.acquire := x_init.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) val do_pop = foldR(pop_x_inits)(_||_) x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.global_xact_id := OHToUFix(pop_x_inits) + x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) x_init.ready := (x_abort.valid && x_abort.ready) || do_pop x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) @@ -512,15 +512,15 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new ProbeRequest() }) + val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe_req.bits - p_req_arb_arr(j).io.in(i).valid := t.probe_req.valid && t.push_p_req(j) + p_req_arb_arr(j).io.in(i).bits := t.probe.bits + p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe_req + FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -552,9 +552,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.xact_init - val x_init_data = io.network.xact_init_data - val x_abort = io.network.xact_abort + val x_init = io.network.acquire + val x_init_data = io.network.acquire_data + val x_abort = io.network.abort val x_dep_deq = x_init_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) @@ -578,14 +578,14 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.global_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) + x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := x_init.valid // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.tile_xact_id := x_init.bits.payload.tile_xact_id + x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { @@ -610,23 +610,23 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new ProbeRequest }} + val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io p_req_arb.io.in(i).bits := t.p_req.bits p_req_arb.io.in(i).valid := t.p_req.valid t.p_req.ready := p_req_arb.io.in(i).ready } - io.network.probe_req <> p_req_arb.io.out + io.network.probe <> p_req_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.probe_rep - val p_rep_data = io.network.probe_rep_data - val idx = p_rep.bits.payload.global_xact_id + val p_rep = io.network.release + val p_rep_data = io.network.release_data + val idx = p_rep.bits.payload.master_xact_id p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.global_xact_id := p_rep.bits.payload.global_xact_id + p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { trackerList(i).io.p_rep_data.valid := p_rep_data.valid @@ -639,29 +639,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val x_rep_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new TransactionReply }} + val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - x_rep_arb.io.in(i).bits := t.x_rep.bits - x_rep_arb.io.in(i).valid := t.x_rep.valid - t.x_rep.ready := x_rep_arb.io.in(i).ready + grant_arb.io.in(i).bits := t.grant.bits + grant_arb.io.in(i).valid := t.grant.valid + t.grant.ready := grant_arb.io.in(i).ready } - x_rep_arb.io.out.ready := Bool(false) - io.network.xact_rep.valid := x_rep_arb.io.out.valid - io.network.xact_rep.bits := x_rep_arb.io.out.bits - x_rep_arb.io.out.ready := io.network.xact_rep.ready + grant_arb.io.out.ready := Bool(false) + io.network.grant.valid := grant_arb.io.out.valid + io.network.grant.bits := grant_arb.io.out.bits + grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { - io.network.xact_rep.valid := Bool(true) - io.network.xact_rep.bits := Vec(trackerList.map(_.io.x_rep.bits)){(new LogicalNetworkIO){new TransactionReply}}(io.mem.resp.bits.tag) + io.network.grant.valid := Bool(true) + io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.x_rep.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.xact_rep.ready + trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions - val finish = io.network.xact_finish + val finish = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.global_xact_id === UFix(i)) + trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) } finish.ready := Bool(true) @@ -683,10 +683,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInit }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new TransactionInitData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReply }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ProbeReplyData }}.flip + val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip @@ -696,15 +696,15 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new ProbeRequest }} - val x_rep = (new FIFOIO) {(new LogicalNetworkIO) { new TransactionReply }} + val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) } val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) - val xact = Reg{ new TransactionInit } + val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) @@ -730,7 +730,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) p_req_initial_flags := ~(io.tile_incoherent | myflag) } - val all_x_reps_require_acks = Bool(true) + val all_grants_require_acks = Bool(true) io.busy := state != s_idle io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) @@ -742,24 +742,24 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeRequestType(xact.x_type, UFix(0)) - io.p_req.bits.payload.global_xact_id := UFix(id) + io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) - io.x_rep.bits.payload.data := io.mem_resp.bits.data - io.x_rep.bits.payload.x_type := co.getTransactionReplyType(xact.x_type, init_sharer_cnt_) - io.x_rep.bits.payload.tile_xact_id := xact.tile_xact_id - io.x_rep.bits.payload.global_xact_id := UFix(id) - io.x_rep.bits.payload.require_ack := all_x_reps_require_acks - io.x_rep.bits.header.dst := init_client_id_ - io.x_rep.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) + io.grant.bits.payload.data := io.mem_resp.bits.data + io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.payload.require_ack := all_grants_require_acks + io.grant.bits.header.dst := init_client_id_ + io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) io.x_init_data_dep.ready := Bool(false) io.p_rep.ready := Bool(false) io.p_rep_data.ready := Bool(false) io.p_rep_data_dep.ready := Bool(false) - io.mem_resp.ready := io.x_rep.ready + io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { @@ -768,7 +768,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com init_client_id_ := io.x_init.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.x_type, UFix(0)) + x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) p_req_flags := p_req_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) @@ -807,7 +807,7 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com p_rep_data_needs_write, p_w_mem_cmd_sent, io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.global_xact_id === UFix(id))) + io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen(x_init_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, @@ -816,17 +816,17 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com x_init_data_needs_write, x_w_mem_cmd_sent, io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.global_xact_id === UFix(id))) + io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.x_type, UFix(0)), s_ack, - Mux(all_x_reps_require_acks, s_busy, s_idle)) + state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + Mux(all_grants_require_acks, s_busy, s_idle)) } } is(s_ack) { - io.x_rep.valid := Bool(true) - when(io.x_rep.ready) { state := Mux(all_x_reps_require_acks, s_busy, s_idle) } + io.grant.valid := Bool(true) + when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { From b5ccdab514aae4b642c5a88b6c6cbe767420133b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 157/688] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 462 ++++++++++++++++++++-------------------- 1 file changed, 231 insertions(+), 231 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..21aaba8c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -402,31 +402,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -435,31 +435,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -468,7 +468,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -486,41 +486,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -538,8 +538,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -552,49 +552,49 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := acquire.bits.header.src + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -603,38 +603,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -659,11 +659,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -683,20 +683,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -708,32 +708,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -741,11 +741,11 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) - io.p_req.valid := Bool(false) - io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.p_req.bits.payload.master_xact_id := UFix(id) - io.p_req.bits.payload.addr := xact.addr - io.p_req.bits.header.dst := UFix(0) + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -753,70 +753,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 1945fa898bb2ab1e242f201112a352eaf4f55dbe Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 24 Jan 2013 23:40:47 -0800 Subject: [PATCH 158/688] make external clock divider programmable --- uncore/src/slowio.scala | 47 ++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index fa6fef94..cf00ae6e 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ import Constants._ -class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { val io = new Bundle { val out_fast = new FIFOIO()(data).flip @@ -13,31 +13,54 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) val in_slow = new FIFOIO()(data).flip val clk_slow = Bool(OUTPUT) + + val set_divisor = new PipeIO()(Bits(width = 32)).flip + val divisor = Bits(OUTPUT, 32) } - val hold_cycles = if (hold_cycles_in == -1) divisor/4 else hold_cycles_in - require((divisor & (divisor-1)) == 0) - require(hold_cycles < divisor/2 && hold_cycles >= 1) + require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) + val divisor = Reg(resetVal = UFix(divisor_max-1)) + val d_shadow = Reg(resetVal = UFix(divisor_max-1)) + val hold = Reg(resetVal = UFix(divisor_max/4-1)) + val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + when (io.set_divisor.valid) { + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + } + io.divisor := hold << UFix(16) | divisor - val cnt = Reg() { UFix(width = log2Up(divisor)) } - cnt := cnt + UFix(1) - val out_en = cnt === UFix(divisor/2+hold_cycles-1) // rising edge + hold time - val in_en = cnt === UFix(divisor/2-1) // rising edge + val count = Reg{UFix(width = log2Up(divisor_max))} + val clock = Reg{Bool()} + count := count + UFix(1) + + val rising = count === (divisor >> UFix(1)) + val falling = count === divisor + val held = count === (divisor >> UFix(1)) + hold + + when (falling) { + divisor := d_shadow + hold := h_shadow + count := UFix(0) + clock := Bool(false) + } + when (rising) { + clock := Bool(true) + } val in_slow_rdy = Reg(resetVal = Bool(false)) val out_slow_val = Reg(resetVal = Bool(false)) val out_slow_bits = Reg() { data } val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := in_en && (io.in_slow.valid && in_slow_rdy || reset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast val tohost_q = new Queue(1)(data) tohost_q.io.enq <> io.out_fast - tohost_q.io.deq.ready := in_en && io.out_slow.ready && out_slow_val + tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val - when (out_en) { + when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) @@ -46,5 +69,5 @@ class slowIO[T <: Data](val divisor: Int, hold_cycles_in: Int = -1)(data: => T) io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := cnt(log2Up(divisor)-1).toBool + io.clk_slow := clock } From 1134bbf1a4dbc9f479fab240410be8ac0ed1c514 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 27 Jan 2013 11:59:17 -0800 Subject: [PATCH 159/688] cleanup disconnected io pins (overwritten network headers) --- uncore/src/package.scala | 8 +++++--- uncore/src/uncore.scala | 7 +++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..46d46661 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -62,21 +62,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 690454a1..01e9c40c 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -369,6 +369,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data rep.bits.payload.require_ack := Bool(true) + rep.bits.header.dst := UFix(0) // DNC + rep.bits.header.src := UFix(0) // DNC rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -445,6 +447,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) } x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.header.dst := UFix(0) // DNC + x_abort.bits.header.src := UFix(0) // DNC want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) x_abort.valid := Bool(false) @@ -585,6 +589,7 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Nack conflicting transaction init attempts x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.src := UFix(0) //DNC x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { @@ -746,12 +751,14 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) + io.p_req.bits.header.src := UFix(0) // DNC io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ + io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.x_init.ready := Bool(false) io.x_init_data.ready := Bool(false) From 47a632cc59cb469458e1a6a8082ea4a0df64d403 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 28 Jan 2013 16:39:45 -0800 Subject: [PATCH 160/688] added support for voluntary wbs over the release network --- uncore/src/coherence.scala | 84 ++++++++++++++++++++--- uncore/src/package.scala | 15 ----- uncore/src/tilelink.scala | 15 ++++- uncore/src/uncore.scala | 135 ++++++++++++++++++++++++++++--------- 4 files changed, 189 insertions(+), 60 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 4268fcf8..e1cc453e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -31,6 +31,7 @@ abstract class CoherencePolicy { def getAcquireTypeOnCacheControl(cmd: Bits): Bits def getAcquireTypeOnWriteback(): Bits + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix): Release def messageHasData (reply: Release): Bool @@ -40,11 +41,15 @@ abstract class CoherencePolicy { def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isVoluntary(rel: Release): Bool def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def requiresAck(grant: Grant): Bool + def requiresAck(release: Release): Bool } trait UncachedTransactions { @@ -70,17 +75,20 @@ abstract class IncoherentPolicy extends CoherencePolicy { def messageHasData (reply: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) + def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } val grantData :: grantAck :: Nil = Enum(2){ UFix() } - val releaseInvalidateAck :: Nil = Enum(1){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } val uncachedTypeList = List() val hasDataTypeList = List(acquireWriteback) @@ -106,6 +114,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) @@ -129,9 +140,9 @@ class MICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(6){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(4){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -180,7 +191,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive @@ -228,6 +241,12 @@ class MICoherence extends CoherencePolicyWithUncached { )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, @@ -248,6 +267,8 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MEICoherence extends CoherencePolicyWithUncached { @@ -256,9 +277,9 @@ class MEICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -316,7 +337,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -373,6 +396,12 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -395,6 +424,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MSICoherence extends CoherencePolicyWithUncached { @@ -403,9 +434,9 @@ class MSICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -470,7 +501,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -527,6 +560,12 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -546,6 +585,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MESICoherence extends CoherencePolicyWithUncached { @@ -554,9 +595,9 @@ class MESICoherence extends CoherencePolicyWithUncached { val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(6){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -621,7 +662,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -678,6 +721,12 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -700,6 +749,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } + + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -707,9 +759,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(9){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(10){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -789,7 +841,9 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) + def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -848,6 +902,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } + def getGrantType(rel: Release, count: UFix): Bits = { + MuxLookup(rel.r_type, grantReadUncached, Array( + releaseVoluntaryInvalidateData -> grantVoluntaryAck + )) + } + def getProbeType(a_type: UFix, global_state: UFix): UFix = { MuxLookup(a_type, probeCopy, Array( @@ -871,4 +931,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } + def requiresAck(grant: Grant) = Bool(true) + def requiresAck(release: Release) = Bool(false) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 0e69ff47..8e511663 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -105,19 +105,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 8ff26adc..51b64b16 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -79,8 +79,20 @@ class Probe extends PhysicalAddress { val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class Release extends Bundle { +object Release +{ + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel.client_xact_id := client_xact_id + rel.master_xact_id := master_xact_id + rel + } +} +class Release extends PhysicalAddress { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) + val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } @@ -90,7 +102,6 @@ class Grant extends MemData { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) - val require_ack = Bool() } class GrantAck extends Bundle { diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 21aaba8c..a439c7b6 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -288,7 +288,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data - grant.bits.payload.require_ack := Bool(true) grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) @@ -368,7 +367,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co rep.bits.payload.client_xact_id := UFix(0) rep.bits.payload.master_xact_id := UFix(0) rep.bits.payload.data := io.mem.resp.bits.data - rep.bits.payload.require_ack := Bool(true) rep.valid := Bool(false) when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) @@ -537,11 +535,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) + val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i) t.io.tile_incoherent := io.incoherent.toBits t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) @@ -559,12 +557,12 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) + val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - for( i <- 0 until NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } + for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready @@ -610,8 +608,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 until NGLOBAL_XACTS ) { + val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb.io.in(i).bits := t.probe.bits probe_arb.io.in(i).valid := t.probe.valid @@ -622,13 +620,16 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle probe replies, which may or may not have data val release = io.network.release val release_data = io.network.release_data - val idx = release.bits.payload.master_xact_id + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} + val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) release.ready := trackerList.map(_.io.release.ready).reduce(_||_) release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.release_data.valid := release_data.valid trackerList(i).io.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid @@ -639,28 +640,27 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Reply to initial requestor // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 until NGLOBAL_XACTS ) { + val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} + for( i <- 0 to NGLOBAL_XACTS ) { val t = trackerList(i).io grant_arb.io.in(i).bits := t.grant.bits grant_arb.io.in(i).valid := t.grant.valid t.grant.ready := grant_arb.io.in(i).ready } - grant_arb.io.out.ready := Bool(false) io.network.grant.valid := grant_arb.io.out.valid io.network.grant.bits := grant_arb.io.out.bits grant_arb.io.out.ready := io.network.grant.ready when(io.mem.resp.valid) { io.network.grant.valid := Bool(true) io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready } } // Free finished transactions val ack = io.network.grant_ack - for( i <- 0 until NGLOBAL_XACTS ) { + for( i <- 0 to NGLOBAL_XACTS ) { trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } ack.ready := Bool(true) @@ -668,9 +668,9 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { + val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } + val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } + for( i <- 0 to NGLOBAL_XACTS ) { mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock @@ -679,7 +679,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren io.mem.req_data <> Queue(mem_req_data_arb.io.out) } -class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { + +abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { @@ -699,15 +700,83 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) - val has_conflict = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } +} +class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } + val state = Reg(resetVal = s_idle) + val xact = Reg{ new Release } + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val release_data_needs_write = Reg(resetVal = Bool(false)) + val mem_cmd_sent = Reg(resetVal = Bool(false)) + + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.mem_resp.ready := Bool(false) + io.probe.valid := Bool(false) + io.busy := Bool(true) + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + + io.mem_req_cmd.valid := Bool(false) + io.mem_req_cmd.bits.rw := Bool(false) + io.mem_req_cmd.bits.addr := xact.addr + io.mem_req_cmd.bits.tag := UFix(id) + io.mem_req_data.valid := Bool(false) + io.mem_req_data.bits.data := UFix(0) + io.mem_req_lock := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) + io.grant.valid := Bool(false) + io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.grant.bits.payload.client_xact_id := xact.client_xact_id + io.grant.bits.payload.master_xact_id := UFix(id) + io.grant.bits.header.dst := init_client_id_ + + switch (state) { + is(s_idle) { + when( io.release.valid ) { + xact := io.release.bits.payload + init_client_id_ := io.release.bits.header.src + release_data_needs_write := co.messageHasData(io.release.bits.payload) + mem_cnt := UFix(0) + mem_cmd_sent := Bool(false) + io.release.ready := Bool(true) + state := s_mem + } + } + is(s_mem) { + when (release_data_needs_write) { + doMemReqWrite(io.mem_req_cmd, + io.mem_req_data, + io.mem_req_lock, + io.release_data, + release_data_needs_write, + mem_cmd_sent, + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . otherwise { state := s_ack } + } + is(s_ack) { + io.grant.valid := Bool(true) + when(io.grant.ready) { state := s_idle } + } + } +} + +class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) @@ -715,9 +784,6 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { @@ -730,10 +796,10 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -747,10 +813,9 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.probe.bits.payload.addr := xact.addr io.probe.bits.header.dst := UFix(0) io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) + io.grant.bits.payload.g_type := grant_type io.grant.bits.payload.client_xact_id := xact.client_xact_id io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.payload.require_ack := all_grants_require_acks io.grant.bits.header.dst := init_client_id_ io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) io.acquire.ready := Bool(false) @@ -821,12 +886,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(all_grants_require_acks, s_busy, s_idle)) + Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(all_grants_require_acks, s_busy, s_idle) } + when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.free) { @@ -834,6 +899,12 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com } } } +} + +trait MemoryRequestGenerator { + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { req_cmd.bits.rw := Bool(true) From 944f56a7660166933e74cbc4942d84e392905601 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 28 Feb 2013 14:55:19 -0800 Subject: [PATCH 161/688] remove duplicate definitions --- uncore/src/package.scala | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46d46661..21ea9433 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -106,20 +106,4 @@ class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkCon val payload = data override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } } - -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} - -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} - override def clone = { new TileLinkIO().asInstanceOf[this.type] } -} } From 9f0ccbeac58c32ae2416e030d53dc656969fb7a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 18:13:41 -0800 Subject: [PATCH 162/688] writebacks on release network pass asm tests and bmarks --- uncore/src/coherence.scala | 213 +++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 113 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index e1cc453e..effba2e9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -28,20 +28,21 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getAcquireTypeOnCacheControl(cmd: Bits): Bits - def getAcquireTypeOnWriteback(): Bits + def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getReleaseTypeOnVoluntaryWriteback(): Bits def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix): Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (reply: Release): Bool + def messageHasData (rel: Release): Bool def messageHasData (acq: Acquire): Bool - def messageHasData (reply: Grant): Bool + def messageHasData (grant: Grant): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool + def isVoluntary(gnt: Grant): Bool def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix @@ -50,6 +51,9 @@ abstract class CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + + def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -66,13 +70,8 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release - reply.r_type := UFix(0) - reply.master_xact_id := UFix(0) - reply - } - def messageHasData (reply: Release) = Bool(false) + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -82,15 +81,19 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + } class ThreeStateIncoherence extends IncoherentPolicy { val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantData :: grantAck :: Nil = Enum(2){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } - val uncachedTypeList = List() - val hasDataTypeList = List(acquireWriteback) + val uncachedAcquireTypeList = List() + val hasDataAcquireTypeList = List(acquireWriteback) + val hasDataReleaseTypeList = List(acquireWriteback) + val hasDataGrantTypeList = List(grantData) def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) def isValid (state: UFix): Bool = state != tileInvalid @@ -116,6 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -125,13 +129,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteback //TODO - def getAcquireTypeOnWriteback(): Bits = acquireWriteback + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (init: Acquire): Bool = hasDataTypeList.map(t => init.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant) = (reply.g_type === grantData) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -143,8 +147,10 @@ class MICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -194,14 +200,14 @@ class MICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,23 +216,16 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -269,6 +268,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -280,8 +280,11 @@ class MEICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid def isValid (state: UFix): Bool = state != tileInvalid @@ -340,6 +343,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -349,11 +353,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusiveDirty, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -364,24 +367,16 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(init: Acquire): Bool = uncachedTypeList.map(t => init.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -426,6 +421,8 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -437,8 +434,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, grantReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -504,6 +503,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -513,11 +513,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -528,24 +527,16 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -587,6 +578,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -598,8 +591,11 @@ class MESICoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -665,6 +661,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -674,11 +671,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, acquireReadExclusive, outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { - val reply = new Release + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -689,24 +685,16 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - (reply.r_type === releaseInvalidateData || - reply.r_type === releaseDowngradeData || - reply.r_type === releaseCopyData) - } - def messageHasData (acq: Acquire): Bool = hasDataTypeList.map(t => acq.a_type === t).reduceLeft(_||_) - def messageHasData (reply: Grant): Bool = { - (reply.g_type != grantWriteUncached && reply.g_type != grantReadExclusiveAck && reply.g_type != grantWriteWordUncached) - } + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uncachedTypeList.map(t => acq.a_type === t).reduceLeft(_||_) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -752,6 +740,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -762,10 +752,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } - val uncachedTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { val (read, write) = cpuCmdToRW(cmd) @@ -844,6 +835,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData + def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -853,12 +845,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val (read, write) = cpuCmdToRW(cmd) Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getAcquireTypeOnCacheControl(cmd: Bits): Bits = acquireWriteUncached - def getAcquireTypeOnWriteback(): Bits = getAcquireTypeOnCacheControl(M_INV) + def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def newRelease (incoming: Probe, state: UFix): Release = { + def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") - val reply = new Release() val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), @@ -871,22 +862,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - reply.r_type := Mux(needsWriteback(state), with_data, without_data) - reply.master_xact_id := incoming.master_xact_id - reply + Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (reply: Release): Bool = { - uFixListContains(List(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory), reply.r_type) - } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataTypeList, acq.a_type) - def messageHasData (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached), reply.g_type) - } + def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) + def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) + def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) @@ -933,4 +918,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + + def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } From 717a78f96435b51de9e99a2b9d0f3f3964d93fda Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:09 -0700 Subject: [PATCH 163/688] fix seqRead inference --- uncore/src/llc.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 7ccdedb8..55d543b7 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -40,7 +40,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val dout1 = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg() { Bits() } else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) @@ -48,15 +48,15 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } - .otherwise { if (postLatency > 0) dout1 := mem(idx) } + .otherwise { if (postLatency > 0) ridx := idx } } if (postLatency == 0) { dout = mem(idx) } else if (postLatency == 1) { - dout = dout1 + dout = mem(ridx) } else - dout = Pipe(reg_ren, dout1, postLatency-1).bits + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits rdata(j) := dout } From f120800aa259592400793f7dfa1b15a141f82e6c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 19 Mar 2013 00:41:28 -0700 Subject: [PATCH 164/688] add DRAMSideLLCNull --- uncore/src/llc.scala | 96 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 55d543b7..1f9f4b2a 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -442,3 +442,99 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data } + +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(resetVal = Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + do_flow := empty && io.deq.ready + + val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} + val ram_addr = Reg{Bits()} + val ram_out_valid = Reg{Bool()} + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +{ + val io = new ioQueue(entries)(data) + + val fq = new HellaFlowQueue(entries)(data) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { + val q = (new HellaQueue(entries)) { enq.bits.clone } + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +{ + val io = new Bundle { + val cpu = new ioMem().flip + val mem = new ioMemPipe + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(resetVal = UFix(numEntries, size)) + val watermark = count >= UFix(refillCycles) + + when (inc && !dec) { + count := count + UFix(1) + } + when (!inc && dec) { + count := count - UFix(refillCycles) + } + when (inc && dec) { + count := count - UFix(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} From 7b019cb0da4f2699e9b1fc45b8f21fc7adf9c96d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 19 Mar 2013 15:30:23 -0700 Subject: [PATCH 165/688] rmeove aborts --- uncore/src/tilelink.scala | 5 -- uncore/src/uncore.scala | 96 ++++----------------------------------- 2 files changed, 8 insertions(+), 93 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 51b64b16..73bf8aff 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -70,10 +70,6 @@ object Acquire class AcquireData extends MemData -class Abort extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) -} - class Probe extends PhysicalAddress { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) @@ -115,7 +111,6 @@ class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} - val abort = (new MasterSourcedIO){(new LogicalNetworkIO){new Abort }} val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index a439c7b6..8d1d396d 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -290,7 +290,6 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren grant.bits.payload.data := io.mem.resp.bits.data grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) io.tiles(0).probe.valid := Bool(false) io.tiles(0).release.ready := Bool(true) @@ -427,51 +426,6 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } - - // Nack conflicting transaction init attempts - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } - val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } - for( j <- 0 until conf.ln.nTiles ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val x_abort = io.tiles(j).abort - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val conflicts = Vec(NGLOBAL_XACTS) { Bool() } - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) - } - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) - - x_abort.valid := Bool(false) - switch(abort_state_arr(j)) { - is(s_idle) { - when(want_to_abort_arr(j)) { - when(co.messageHasData(acquire.bits.payload)) { - abort_state_arr(j) := s_abort_drain - } . otherwise { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state_arr(j) := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready below - abort_state_arr(j) := s_idle - } - } - } - } // Handle transaction initiation requests // Only one allocation per cycle @@ -493,16 +447,14 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq - val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).valid := acquire.valid init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - val do_pop = foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire.ready := foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire.ready := (x_abort.valid && x_abort.ready) || do_pop - acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } @@ -552,61 +504,29 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Init requests may or may not have data val acquire = io.network.acquire val acquire_data = io.network.acquire_data - val x_abort = io.network.abort val x_dep_deq = acquire_data_dep_q.io.deq - val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } - val abort_state = Reg(resetVal = s_idle) - val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = acquire.valid && (any_acquire_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } for( i <- 0 to NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready trackerList(i).io.acquire_data.bits := acquire_data.bits trackerList(i).io.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire - acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid - // Nack conflicting transaction init attempts - x_abort.bits.header.dst := acquire.bits.header.src - x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id - x_abort.valid := Bool(false) - switch(abort_state) { - is(s_idle) { - when(want_to_abort) { - abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) - } - } - is(s_abort_drain) { // raises acquire_data.ready below - when(acquire_data.valid) { - abort_cnt := abort_cnt + UFix(1) - when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { - abort_state := s_abort_send - } - } - } - is(s_abort_send) { // nothing is dequeued for now - x_abort.valid := Bool(true) - when(x_abort.ready) { // raises acquire.ready - abort_state := s_idle - } - } - } - // Handle probe request generation val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 to NGLOBAL_XACTS ) { From a7ae7e5758eea704ed284667df0295e9b304a30c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 19:49:05 -0800 Subject: [PATCH 166/688] Cleaned up self-probes --- uncore/src/coherence.scala | 7 +++++++ uncore/src/uncore.scala | 11 +++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index effba2e9..f7a75afc 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -49,6 +49,7 @@ abstract class CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool def needsMemWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool @@ -79,6 +80,7 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) @@ -268,6 +270,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -421,6 +424,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -578,6 +582,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -740,6 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -918,6 +924,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) + def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 8d1d396d..e3ebbc07 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -105,14 +105,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val probe_initial_flags = Bits(width = conf.ln.nTiles) probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.alloc_req.bits.acquire) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) + val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } io.busy := state != s_idle From 319b4544d757f871a9fcd6437abf62847eeab179 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Feb 2013 21:03:56 -0800 Subject: [PATCH 167/688] nTiles -> nClients in LogicalNetworkConfig --- uncore/src/package.scala | 2 +- uncore/src/uncore.scala | 116 +++++++++++++++++++-------------------- 2 files changed, 59 insertions(+), 59 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 8e511663..7cce3ff3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -48,7 +48,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nHubs: Int, nTiles: Int) +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { val io: Vec[TileLinkType] diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index e3ebbc07..5609b701 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,9 +24,9 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nClients) + val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data = (new PipeIO) { new ReleaseData }.flip val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) @@ -44,13 +44,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nTiles) - val pop_release = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nClients) + val pop_release = Bits(OUTPUT, conf.ln.nClients) + val pop_release_data = Bits(OUTPUT, conf.ln.nClients) + val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) val send_grant_ack = Bool(OUTPUT) } @@ -91,8 +91,8 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) @@ -102,11 +102,11 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) + val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) } @@ -115,7 +115,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.init_client_id := init_client_id_ io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only + io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only io.a_type := xact.a_type io.mem_req_cmd.valid := Bool(false) @@ -129,13 +129,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nTiles) - io.pop_release := Bits(0, width = conf.ln.nTiles) - io.pop_release_data := Bits(0, width = conf.ln.nTiles) - io.pop_release_dep := Bits(0, width = conf.ln.nTiles) - io.pop_acquire := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) - io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nClients) + io.pop_release := Bits(0, width = conf.ln.nClients) + io.pop_release_data := Bits(0, width = conf.ln.nClients) + io.pop_release_dep := Bits(0, width = conf.ln.nClients) + io.pop_acquire := Bits(0, width = conf.ln.nClients) + io.pop_acquire_data := Bits(0, width = conf.ln.nClients) + io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) io.send_grant_ack := Bool(false) switch (state) { @@ -150,7 +150,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -167,7 +167,7 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when(io.release_cnt_dec.orR) { val dec = PopCount(io.release_cnt_dec) io.pop_release := io.release_cnt_dec - if(conf.ln.nTiles > 1) release_count := release_count - dec + if(conf.ln.nClients > 1) release_count := release_count - dec when(release_count === dec) { state := s_mem } @@ -223,7 +223,7 @@ case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConf class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val net = (new TileLinkIO).flip - val hub = Vec(conf.nTiles) { new TileLinkIO } + val hub = Vec(conf.nClients) { new TileLinkIO } } val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => @@ -259,8 +259,8 @@ class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Co abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val tiles = Vec(conf.nTiles) { new TileLinkIO }.flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val tiles = Vec(conf.nClients) { new TileLinkIO }.flip + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -307,8 +307,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -333,17 +333,17 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co sent_grant_ack_arr(i) := Bool(false) p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { release_cnt_dec_arr(i)(j) := Bool(false) probe_cnt_inc_arr(i)(j) := Bool(false) } } - val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val ack = io.tiles(j).grant_ack when (ack.valid) { do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) @@ -355,7 +355,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Forward memory responses from mem to tile or arbitrate to ack val mem_idx = io.mem.resp.bits.tag val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val rep = io.tiles(j).grant rep.bits.payload.g_type := UFix(0) rep.bits.payload.client_xact_id := UFix(0) @@ -393,7 +393,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co io.mem.req_data <> Queue(mem_req_data_arb.io.out) // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val release = io.tiles(j).release val release_data = io.tiles(j).release_data val idx = release.bits.payload.master_xact_id @@ -413,10 +413,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - for( j <- 0 until conf.ln.nTiles) { + for( j <- 0 until conf.ln.nClients) { val release = io.tiles(j).release release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } @@ -426,7 +426,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Only one allocation per cycle // Init requests may or may not have data val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nTiles)) { new TrackerAllocReq } + val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready @@ -435,10 +435,10 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } - for( j <- 0 until conf.ln.nTiles ) { + for( j <- 0 until conf.ln.nClients ) { val acquire = io.tiles(j).acquire val acquire_data = io.tiles(j).acquire_data val acquire_data_dep = acquire_data_dep_list(j).io.deq @@ -457,8 +457,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe request generation // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nTiles ) { + val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + for( j <- 0 until conf.ln.nClients ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io probe_arb_arr(j).io.in(i).bits := t.probe.bits @@ -473,7 +473,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val network = (new TileLinkIO).flip - val incoherent = Vec(conf.nTiles) { Bool() }.asInput + val incoherent = Vec(conf.nClients) { Bool() }.asInput val mem = new ioMem } } @@ -604,7 +604,7 @@ abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) ex val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nTiles) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip @@ -624,7 +624,7 @@ class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extend val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) @@ -688,27 +688,27 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nTiles) + val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nTiles > 1) { + if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -746,7 +746,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends when( io.acquire.valid ) { xact := io.acquire.bits.payload init_client_id_ := io.acquire.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only + init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags @@ -754,7 +754,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) io.acquire.ready := Bool(true) - if(conf.ln.nTiles > 1) { + if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -771,7 +771,7 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends } when(io.release.valid) { io.release.ready := Bool(true) - if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } From c36b1dfa30290b5009925c719c99e243a7f6313a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 20 Mar 2013 14:10:16 -0700 Subject: [PATCH 168/688] Cleaned up uncore and coherence interface. Removed defunct broadcast hub. Trait-ified tilelink bundle components. Added generalized mem arbiter. --- uncore/src/coherence.scala | 86 ++-- uncore/src/consts.scala | 6 +- uncore/src/package.scala | 10 +- uncore/src/tilelink.scala | 132 +++++- uncore/src/uncore.scala | 841 ++++++++----------------------------- 5 files changed, 345 insertions(+), 730 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index f7a75afc..b4c4067f 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -34,9 +34,7 @@ abstract class CoherencePolicy { def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release def newRelease (incoming: Probe, state: UFix, id: UFix): Release - def messageHasData (rel: Release): Bool - def messageHasData (acq: Acquire): Bool - def messageHasData (grant: Grant): Bool + def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool @@ -46,8 +44,8 @@ abstract class CoherencePolicy { def getGrantType(a_type: UFix, count: UFix): Bits def getGrantType(rel: Release, count: UFix): Bits def getProbeType(a_type: UFix, global_state: UFix): UFix - def needsMemRead(a_type: UFix, global_state: UFix): Bool - def needsMemWrite(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UFix, global_state: UFix): Bool + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool @@ -72,13 +70,12 @@ abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) - def messageHasData (rel: Release) = Bool(false) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsMemRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) @@ -134,8 +131,12 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData( msg: SourcedMessage ) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => Bool(false) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) } @@ -221,9 +222,12 @@ class MICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -259,10 +263,10 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -373,9 +377,12 @@ class MEICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } @@ -413,10 +420,10 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -534,9 +541,12 @@ class MSICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -571,10 +581,10 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -693,9 +703,12 @@ class MESICoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } @@ -733,10 +746,10 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { @@ -871,9 +884,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) } - def messageHasData (acq: Acquire): Bool = uFixListContains(hasDataAcquireTypeList, acq.a_type) - def messageHasData (rel: Release): Bool = uFixListContains(hasDataReleaseTypeList, rel.r_type) - def messageHasData (grant: Grant): Bool = uFixListContains(hasDataGrantTypeList, grant.g_type) + def messageHasData(msg: SourcedMessage) = msg match { + case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case _ => Bool(false) + } def messageUpdatesDataArray (reply: Grant): Bool = { uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } @@ -913,10 +929,10 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsMemRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsMemWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } def needsAckReply(a_type: UFix, global_state: UFix): Bool = { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index b515c1c8..d348e0ff 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_BITS = log2Up(NGLOBAL_XACTS) + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } @@ -29,7 +29,7 @@ trait TileLinkTypeConstants { trait TileLinkSizeConstants extends TileLinkTypeConstants { - val CLIENT_XACT_ID_BITS = 5 + val CLIENT_XACT_ID_MAX_BITS = 10 val ACQUIRE_WRITE_MASK_BITS = 6 val ACQUIRE_SUBWORD_ADDR_BITS = 3 val ACQUIRE_ATOMIC_OP_BITS = 4 @@ -72,7 +72,7 @@ trait MemoryInterfaceConstants extends UncoreConstants with TileLinkSizeConstants { - val MEM_TAG_BITS = max(CLIENT_XACT_ID_BITS, MASTER_XACT_ID_BITS) + val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) val MEM_DATA_BITS = 128 val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7cce3ff3..492079cc 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -40,8 +40,6 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi rdy := arb.ready && (in.bits.header.dst === UFix(i)) }} out <> rrarb.io.out - //out.bits.header.src := rrarb.io.chosen.toUFix - //out.bits.header.dst := UFix(i) }} for(i <- 0 until conf.nEndpoints) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) @@ -62,21 +60,23 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper){ in.bits.clone } + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = (new FIFOIO){ data }.flip val out = (new FIFOIO){(new LogicalNetworkIO){ data }} } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src io.in.ready := io.out.ready } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 73bf8aff..c16f8c9b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -3,20 +3,30 @@ package uncore import Chisel._ import Constants._ -class PhysicalAddress extends Bundle { +trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -class MemData extends Bundle { +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +} + +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +} + +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends PhysicalAddress { +class MemData extends Bundle with HasMemData + +class MemReqCmd extends Bundle with HasPhysicalAddress { val rw = Bool() val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends MemData { +class MemResp extends Bundle with HasMemData { val tag = Bits(width = MEM_TAG_BITS) } @@ -32,9 +42,12 @@ class ioMemPipe extends Bundle { val resp = (new PipeIO) { new MemResp() }.flip } -class Acquire extends PhysicalAddress { +trait SourcedMessage extends Bundle +trait ClientSourcedMessage extends SourcedMessage +trait MasterSourcedMessage extends SourcedMessage + +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -47,6 +60,9 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { @@ -55,6 +71,8 @@ object Acquire acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask + acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) + acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) acq } def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { @@ -64,15 +82,15 @@ object Acquire acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode + acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) acq } } -class AcquireData extends MemData +class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends PhysicalAddress { +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = PROBE_TYPE_MAX_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } object Release @@ -86,35 +104,105 @@ object Release rel } } -class Release extends PhysicalAddress { +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class ReleaseData extends MemData +class ReleaseData extends ClientSourcedMessage with HasMemData -class Grant extends MemData { +class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = GRANT_TYPE_MAX_BITS) - val client_xact_id = Bits(width = CLIENT_XACT_ID_BITS) - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) } -class GrantAck extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) -} +class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} + val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } +} + +class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} - val grant = (new MasterSourcedIO){(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } + +object UncachedTileLinkIOArbiterShim { + def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { + val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} + shim.io.in <> in + shim.io.out + } +} +class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip + val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} + } + io.out.bits := io.in.bits + io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) + io.out.valid := io.in.valid + io.in.ready := io.out.ready +} + + +class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + + val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + when(io.out.acquire_data.valid && io.out.acquire_data.ready) { + mem_cnt := mem_cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(mem_cnt_next === UFix(0)) { + locked := Bool(false) + } + } + + val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) + (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) + var acqd_bits = io.in(n-1).acquire_data.bits + for (i <- n-2 to 0 by -1) { + acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) + } + val locked_req = io.in(lock_idx).acquire_data + io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) + io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) + + val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + + val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + val tag = io.out.grant.bits.payload.client_xact_id + io.in(i).grant.valid := Bool(false) + when (tag(log2Up(n)-1,0) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + } +} diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5609b701..14467f7b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,688 +3,199 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { val client_id = Bits(width = conf.ln.idBits) } -class TrackerAllocReq(implicit conf: CoherenceHubConfiguration) extends Bundle { +class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { val acquire = new Acquire() val client_id = Bits(width = conf.ln.idBits) override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } } class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_BITS) + val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) } -class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component { - val co = conf.co - val io = new Bundle { - val alloc_req = (new FIFOIO) { new TrackerAllocReq }.flip - val p_data = (new PipeIO) { new TrackerProbeData }.flip - val can_alloc = Bool(INPUT) - val grant_ack = Bool(INPUT) - val release_cnt_dec = Bits(INPUT, conf.ln.nClients) - val probe_cnt_inc = Bits(INPUT, conf.ln.nClients) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data = (new PipeIO) { new ReleaseData }.flip - val acquire_data = (new PipeIO) { new AcquireData }.flip - val sent_grant_ack = Bool(INPUT) - val release_data_dep = (new PipeIO) { new TrackerDependency }.flip - val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip - - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) { new Probe } - val busy = Bool(OUTPUT) - val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) - val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val release_client_id = Bits(OUTPUT, conf.ln.idBits) - val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) - val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) - val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_probe = Bits(OUTPUT, conf.ln.nClients) - val pop_release = Bits(OUTPUT, conf.ln.nClients) - val pop_release_data = Bits(OUTPUT, conf.ln.nClients) - val pop_release_dep = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_data = Bits(OUTPUT, conf.ln.nClients) - val pop_acquire_dep = Bits(OUTPUT, conf.ln.nClients) - val send_grant_ack = Bool(OUTPUT) - } - - def doMemReqWrite(req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: PipeIO[MemData], trigger: Bool, cmd_sent: Bool, pop_data: Bits, pop_dep: Bits, at_front_of_dep_queue: Bool, client_id: UFix) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits - when(req_cmd.ready && req_cmd.valid) { - cmd_sent := Bool(true) - } - when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - pop_data := UFix(1) << client_id - when (data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := UFix(1) << client_id - trigger := Bool(false) - } - } - } - } - } - } - - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { - trigger := Bool(false) - } - } - - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) - val xact = Reg{ new Acquire } - val init_client_id_ = Reg{ Bits() } - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) - val release_client_id_ = Reg{ Bits() } - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val probe_initial_flags = Bits(width = conf.ln.nClients) - probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { - val probe_self = co.needsSelfProbe(io.alloc_req.bits.acquire) - val probe_self_flag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | probe_self_flag) - } - - io.busy := state != s_idle - io.addr := xact.addr - io.init_client_id := init_client_id_ - io.release_client_id := release_client_id_ - io.client_xact_id := xact.client_xact_id - io.sharer_count := UFix(conf.ln.nClients) // TODO: Broadcast only - io.a_type := xact.a_type - - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.master_xact_id := UFix(id) - io.probe.bits.addr := xact.addr - io.push_probe := Bits(0, width = conf.ln.nClients) - io.pop_release := Bits(0, width = conf.ln.nClients) - io.pop_release_data := Bits(0, width = conf.ln.nClients) - io.pop_release_dep := Bits(0, width = conf.ln.nClients) - io.pop_acquire := Bits(0, width = conf.ln.nClients) - io.pop_acquire_data := Bits(0, width = conf.ln.nClients) - io.pop_acquire_dep := Bits(0, width = conf.ln.nClients) - io.send_grant_ack := Bool(false) - - switch (state) { - is(s_idle) { - when( io.alloc_req.valid && io.can_alloc ) { - xact := io.alloc_req.bits.acquire - init_client_id_ := io.alloc_req.bits.client_id - acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) - x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - probe_flags := probe_initial_flags - mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id - if(conf.ln.nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem - } - } - is(s_probe) { - when(probe_flags.orR) { - io.push_probe := probe_flags - io.probe.valid := Bool(true) - } - when(io.probe_cnt_inc.orR) { - probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs - } - when(io.release_cnt_dec.orR) { - val dec = PopCount(io.release_cnt_dec) - io.pop_release := io.release_cnt_dec - if(conf.ln.nClients > 1) release_count := release_count - dec - when(release_count === dec) { - state := s_mem - } - } - when(io.p_data.valid) { - release_data_needs_write := Bool(true) - release_client_id_ := io.p_data.bits.client_id - } - } - is(s_mem) { - when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, - release_data_needs_write, - p_w_mem_cmd_sent, - io.pop_release_data, - io.pop_release_dep, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), - release_client_id_) - } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, - acquire_data_needs_write, - x_w_mem_cmd_sent, - io.pop_acquire_data, - io.pop_acquire_dep, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), - init_client_id_) - } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) - } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, s_busy) - } - } - is(s_ack) { - io.send_grant_ack := Bool(true) - when(io.sent_grant_ack) { state := s_busy } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.grant_ack) { - state := s_idle - } - } - } -} - -case class CoherenceHubConfiguration(co: CoherencePolicy, ln: LogicalNetworkConfiguration) - -class CoherenceHubAdapter(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val net = (new TileLinkIO).flip - val hub = Vec(conf.nClients) { new TileLinkIO } - } - - val netClientProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] } - val netMasterProducedSubBundles = io.net.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io.net).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] } - - val hubClientProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[ClientSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[ClientSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - val hubMasterProducedSubBundles = io.hub.map{ io => { - io.getClass.getMethods.filter( x => - classOf[MasterSourcedIO[Data]].isAssignableFrom(x.getReturnType)).map{ m => - m.invoke(io).asInstanceOf[MasterSourcedIO[LogicalNetworkIO[Data]]] }}}.transpose - - hubMasterProducedSubBundles.zip(netMasterProducedSubBundles).foreach{ case(hub, net) => { - net.bits.header.src := UFix(0) - net.bits.header.dst := Vec(hub.map(_.valid)){Bool()}.indexWhere{s: Bool => s} - net.bits.payload := hub(0).bits.payload - net.valid := hub.map(_.valid).fold(Bool(false))(_||_) - hub.foreach( _.ready := net.ready) - }} - hubClientProducedSubBundles.zip(netClientProducedSubBundles).foreach{ case(hub, net) => { - hub.foreach(_.bits.header := net.bits.header) - hub.zipWithIndex.foreach{ case(h,i) => h.valid := (net.bits.header.src === UFix(i)) && net.valid } - hub.foreach(_.bits.payload := net.bits.payload) - net.ready := hub.map(_.ready).fold(Bool(false))(_||_) - }} -} - -abstract class CoherenceHub(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { - val io = new Bundle { - val tiles = Vec(conf.nClients) { new TileLinkIO }.flip - val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem - } -} - -class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - val co = conf.co.asInstanceOf[ThreeStateIncoherence] - - val acquire = io.tiles(0).acquire - val is_write = acquire.bits.payload.a_type === co.acquireWriteback - acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) - io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := acquire.bits.payload.addr - io.mem.req_data <> io.tiles(0).acquire_data - - val grant = io.tiles(0).grant - grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) - grant.bits.payload.master_xact_id := UFix(0) // don't care - grant.bits.payload.data := io.mem.resp.bits.data - grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready - - io.tiles(0).grant_ack.ready := Bool(true) - io.tiles(0).probe.valid := Bool(false) - io.tiles(0).release.ready := Bool(true) - io.tiles(0).release_data.ready := Bool(true) -} - - -class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends CoherenceHub()(conf.ln) -{ - implicit val lnConf = conf.ln - val co = conf.co - val trackerList = (0 until NGLOBAL_XACTS).map(new XactTrackerBroadcast(_)) - - val busy_arr = Vec(NGLOBAL_XACTS){ Bool() } - val addr_arr = Vec(NGLOBAL_XACTS){ Bits(width=PADDR_BITS-OFFSET_BITS) } - val init_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val client_xact_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=CLIENT_XACT_ID_BITS) } - val a_type_arr = Vec(NGLOBAL_XACTS){ Bits(width=ACQUIRE_TYPE_MAX_BITS) } - val sh_count_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - - val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nClients){ Bool()} } - val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } - val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } - - for( i <- 0 until NGLOBAL_XACTS) { - val t = trackerList(i).io - busy_arr(i) := t.busy - addr_arr(i) := t.addr - init_client_id_arr(i) := t.init_client_id - client_xact_id_arr(i) := t.client_xact_id - a_type_arr(i) := t.a_type - sh_count_arr(i) := t.sharer_count - send_grant_ack_arr(i) := t.send_grant_ack - t.grant_ack := do_free_arr(i) - t.p_data.bits.client_id := p_data_client_id_arr(i) - t.p_data.valid := p_data_valid_arr(i) - t.release_cnt_dec := release_cnt_dec_arr(i).toBits - t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits - t.tile_incoherent := io.incoherent.toBits - t.sent_grant_ack := sent_grant_ack_arr(i) - do_free_arr(i) := Bool(false) - sent_grant_ack_arr(i) := Bool(false) - p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) - p_data_valid_arr(i) := Bool(false) - for( j <- 0 until conf.ln.nClients) { - release_cnt_dec_arr(i)(j) := Bool(false) - probe_cnt_inc_arr(i)(j) := Bool(false) - } - } - - val release_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val acquire_data_dep_list = List.fill(conf.ln.nClients)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY - - // Free finished transactions - for( j <- 0 until conf.ln.nClients ) { - val ack = io.tiles(j).grant_ack - when (ack.valid) { - do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) - } - ack.ready := Bool(true) - } - - // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val mem_idx = io.mem.resp.bits.tag - val ack_idx = PriorityEncoder(send_grant_ack_arr.toBits) - for( j <- 0 until conf.ln.nClients ) { - val rep = io.tiles(j).grant - rep.bits.payload.g_type := UFix(0) - rep.bits.payload.client_xact_id := UFix(0) - rep.bits.payload.master_xact_id := UFix(0) - rep.bits.payload.data := io.mem.resp.bits.data - rep.valid := Bool(false) - when(io.mem.resp.valid && (UFix(j) === init_client_id_arr(mem_idx))) { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(mem_idx), sh_count_arr(mem_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(mem_idx) - rep.bits.payload.master_xact_id := mem_idx - rep.valid := Bool(true) - } . otherwise { - rep.bits.payload.g_type := co.getGrantType(a_type_arr(ack_idx), sh_count_arr(ack_idx)) - rep.bits.payload.client_xact_id := client_xact_id_arr(ack_idx) - rep.bits.payload.master_xact_id := ack_idx - when (UFix(j) === init_client_id_arr(ack_idx)) { - rep.valid := send_grant_ack_arr.toBits.orR - sent_grant_ack_arr(ack_idx) := rep.ready - } - } - } - io.mem.resp.ready := io.tiles(init_client_id_arr(mem_idx)).grant.ready - - // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS)) { new MemData() } - for( i <- 0 until NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) - - // Handle probe replies, which may or may not have data - for( j <- 0 until conf.ln.nClients ) { - val release = io.tiles(j).release - val release_data = io.tiles(j).release_data - val idx = release.bits.payload.master_xact_id - val pop_releases = trackerList.map(_.io.pop_release(j).toBool) - val do_pop = foldR(pop_releases)(_ || _) - release.ready := Bool(true) - release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id - release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) - when (release.valid && co.messageHasData(release.bits.payload)) { - p_data_valid_arr(idx) := Bool(true) - p_data_client_id_arr(idx) := UFix(j) - } - release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) - } - for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid - trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - - trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) - trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) - - for( j <- 0 until conf.ln.nClients) { - val release = io.tiles(j).release - release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) - } - } - - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data - val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } - val init_arb = (new Arbiter(conf.ln.nClients)) { new TrackerAllocReq } - for( i <- 0 until NGLOBAL_XACTS ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.can_alloc := alloc_arb.io.in(i).ready - trackerList(i).io.alloc_req.bits := init_arb.io.out.bits - trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - - trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) - trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nClients).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) - } - for( j <- 0 until conf.ln.nClients ) { - val acquire = io.tiles(j).acquire - val acquire_data = io.tiles(j).acquire_data - val acquire_data_dep = acquire_data_dep_list(j).io.deq - init_arb.io.in(j).valid := acquire.valid - init_arb.io.in(j).bits.acquire := acquire.bits.payload - init_arb.io.in(j).bits.client_id := UFix(j) - val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) - acquire.ready := foldR(pop_acquires)(_||_) - acquire_data_dep_list(j).io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) - acquire_data.ready := foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) - acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) - } - - alloc_arb.io.out.ready := init_arb.io.out.valid - - // Handle probe request generation - // Must arbitrate for each request port - val probe_arb_arr = List.fill(conf.ln.nClients)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) - for( j <- 0 until conf.ln.nClients ) { - for( i <- 0 until NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb_arr(j).io.in(i).bits := t.probe.bits - probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) - probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready - } - FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe - } - -} +case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { - val network = (new TileLinkIO).flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO val incoherent = Vec(conf.nClients) { Bool() }.asInput - val mem = new ioMem } } -class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends CoherenceAgent()(conf.ln) +class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_)) + val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i) - t.io.tile_incoherent := io.incoherent.toBits - t.io.mem_resp.valid := io.mem.resp.valid && (io.mem.resp.bits.tag === UFix(i)) - t.io.mem_resp.bits := io.mem.resp.bits - } - io.mem.resp.ready := trackerList.map(_.io.mem_resp.ready).reduce(_||_) + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val acquire = io.network.acquire - val acquire_data = io.network.acquire_data - val x_dep_deq = acquire_data_dep_q.io.deq + val acquire = io.client.acquire + val acquire_data = io.client.acquire_data + val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val alloc_arb = (new Arbiter(NGLOBAL_XACTS+1)) { Bool() } - for( i <- 0 to NGLOBAL_XACTS ) { + val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + for( i <- 0 until trackerList.size ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.acquire.bits := acquire.bits - trackerList(i).io.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + val t = trackerList(i).io.client + t.acquire.bits := acquire.bits + t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.acquire_data.bits := acquire_data.bits - trackerList(i).io.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid + t.acquire_data.bits := acquire_data.bits + t.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.acquire.ready).reduce(_||_) - acquire_data.ready := trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) - x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid // Handle probe request generation - val probe_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Probe }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - probe_arb.io.in(i).bits := t.probe.bits - probe_arb.io.in(i).valid := t.probe.valid - t.probe.ready := probe_arb.io.in(i).ready - } - io.network.probe <> probe_arb.io.out + val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } // Handle probe replies, which may or may not have data - val release = io.network.release - val release_data = io.network.release_data + val release = io.client.release + val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) release_data_dep_q.io.enq.bits.master_xact_id := idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.release_data.valid := release_data.valid - trackerList(i).io.release_data.bits := release_data.bits + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.valid := release.valid && (idx === UFix(i)) + t.release.bits := release.bits + t.release_data.valid := release_data.valid + t.release_data.bits := release_data.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) - trackerList(i).io.release.bits := release.bits } // Reply to initial requestor - // Forward memory responses from mem to tile or arbitrate to ack - val grant_arb = (new Arbiter(NGLOBAL_XACTS+1)){(new LogicalNetworkIO){ new Grant }} - for( i <- 0 to NGLOBAL_XACTS ) { - val t = trackerList(i).io - grant_arb.io.in(i).bits := t.grant.bits - grant_arb.io.in(i).valid := t.grant.valid - t.grant.ready := grant_arb.io.in(i).ready - } - io.network.grant.valid := grant_arb.io.out.valid - io.network.grant.bits := grant_arb.io.out.bits - grant_arb.io.out.ready := io.network.grant.ready - when(io.mem.resp.valid) { - io.network.grant.valid := Bool(true) - io.network.grant.bits := Vec(trackerList.map(_.io.grant.bits)){(new LogicalNetworkIO){new Grant}}(io.mem.resp.bits.tag) - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.grant.ready := (io.mem.resp.bits.tag === UFix(i)) && io.network.grant.ready - } - } + val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.network.grant_ack - for( i <- 0 to NGLOBAL_XACTS ) { - trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) - } + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port - // We have to arbitrate between the different trackers' memory requests - // and once we have picked a request, get the right write data - val mem_req_cmd_arb = (new Arbiter(NGLOBAL_XACTS+1)) { new MemReqCmd() } - val mem_req_data_arb = (new LockingArbiter(NGLOBAL_XACTS+1)) { new MemData() } - for( i <- 0 to NGLOBAL_XACTS ) { - mem_req_cmd_arb.io.in(i) <> trackerList(i).io.mem_req_cmd - mem_req_data_arb.io.in(i) <> trackerList(i).io.mem_req_data - mem_req_data_arb.io.lock(i) <> trackerList(i).io.mem_req_lock - } - io.mem.req_cmd <> Queue(mem_req_cmd_arb.io.out) - io.mem.req_data <> Queue(mem_req_data_arb.io.out) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out } -abstract class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Component with MemoryRequestGenerator { +abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip - val free = Bool(INPUT) - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val mem_resp = (new FIFOIO) { new MemResp }.flip + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO - val mem_req_cmd = (new FIFOIO) { new MemReqCmd } - val mem_req_data = (new FIFOIO) { new MemData } - val mem_req_lock = Bool(OUTPUT) - val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} - val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} - val busy = Bool(OUTPUT) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip + + val busy = Bool(OUTPUT) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.mem_resp.ready := Bool(false) - io.probe.valid := Bool(false) + io.release_data_dep.ready := Bool(false) io.busy := Bool(true) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) - io.grant.valid := Bool(false) - io.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := cmd_to_write + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header.src := UFix(bankId) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.release.valid ) { - xact := io.release.bits.payload - init_client_id_ := io.release.bits.header.src - release_data_needs_write := co.messageHasData(io.release.bits.payload) + when( io.client.release.valid ) { + xact := io.client.release.bits.payload + init_client_id_ := io.client.release.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.release.ready := Bool(true) + io.client.release.ready := Bool(true) state := s_mem } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . otherwise { state := s_ack } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := s_idle } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } } } } -class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends XactTracker(id)(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } @@ -697,63 +208,65 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val probe_initial_flags = Bits(width = conf.ln.nClients) probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - // TODO: this is hackish; figure out how to do it more systematically - val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) - case _ => Bool(false) - } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.release.bits.payload.addr) && (state != s_idle) - io.mem_req_cmd.valid := Bool(false) - io.mem_req_cmd.bits.rw := Bool(false) - io.mem_req_cmd.bits.addr := xact.addr - io.mem_req_cmd.bits.tag := UFix(id) - io.mem_req_data.valid := Bool(false) - io.mem_req_data.bits.data := UFix(0) - io.mem_req_lock := Bool(false) - io.probe.valid := Bool(false) - io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.probe.bits.payload.master_xact_id := UFix(id) - io.probe.bits.payload.addr := xact.addr - io.probe.bits.header.dst := UFix(0) - io.grant.bits.payload.data := io.mem_resp.bits.data - io.grant.bits.payload.g_type := grant_type - io.grant.bits.payload.client_xact_id := xact.client_xact_id - io.grant.bits.payload.master_xact_id := UFix(id) - io.grant.bits.header.dst := init_client_id_ - io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.acquire.ready := Bool(false) - io.acquire_data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + //TODO io.master.acquire.bits.header.dst + io.master.acquire.bits.header.src := UFix(bankId) + io.master.acquire_data.valid := Bool(false) + io.master.acquire_data.bits.payload.data := UFix(0) + //TODO io.master.acquire_data.bits.header.dst + io.master.acquire_data.bits.header := UFix(bankId) + io.client.probe.valid := Bool(false) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.addr := xact.addr + io.client.probe.bits.header.dst := UFix(0) + io.client.probe.bits.header.src := UFix(bankId) + io.client.grant.bits.payload.data := io.master.grant.bits.payload.data + io.client.grant.bits.payload.g_type := grant_type + io.client.grant.bits.payload.client_xact_id := xact.client_xact_id + io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.header.dst := init_client_id_ + io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.acquire.ready := Bool(false) + io.client.acquire_data.ready := Bool(false) io.acquire_data_dep.ready := Bool(false) - io.release.ready := Bool(false) - io.release_data.ready := Bool(false) + io.client.release.ready := Bool(false) + io.client.release_data.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.mem_resp.ready := io.grant.ready + io.master.grant.ready := io.client.grant.ready + io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - when( io.acquire.valid ) { - xact := io.acquire.bits.payload - init_client_id_ := io.acquire.bits.header.src + when( io.client.acquire.valid ) { + xact := io.client.acquire.bits.payload + init_client_id_ := io.client.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) - x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) - p_w_mem_cmd_sent := Bool(false) - x_w_mem_cmd_sent := Bool(false) - io.acquire.ready := Bool(true) + r_w_mem_cmd_sent := Bool(false) + a_w_mem_cmd_sent := Bool(false) + io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -763,78 +276,76 @@ class AcquireTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends is(s_probe) { val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { - io.probe.valid := Bool(true) - io.probe.bits.header.dst := curr_p_id + io.client.probe.valid := Bool(true) + io.client.probe.bits.header.dst := curr_p_id } - when(io.probe.ready) { + when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.release.valid) { - io.release.ready := Bool(true) + when(io.client.release.valid) { + io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.release.bits.payload) + release_data_needs_write := co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { when (release_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.release_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, release_data_needs_write, - p_w_mem_cmd_sent, + r_w_mem_cmd_sent, io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen(acquire_data_needs_write) { - doMemReqWrite(io.mem_req_cmd, - io.mem_req_data, - io.mem_req_lock, - io.acquire_data, + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.acquire_data, acquire_data_needs_write, - x_w_mem_cmd_sent, + a_w_mem_cmd_sent, io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) } . elsewhen (x_needs_read) { - doMemReqRead(io.mem_req_cmd, x_needs_read) + doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, - Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle)) + Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } is(s_ack) { - io.grant.valid := Bool(true) - when(io.grant.ready) { state := Mux(co.requiresAck(io.grant.bits.payload), s_busy, s_idle) } + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.free) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { state := s_idle } } } } -trait MemoryRequestGenerator { +abstract trait OuterRequestGenerator { + val cmd_to_write: Acquire + val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - def doMemReqWrite[T <: Data](req_cmd: FIFOIO[MemReqCmd], req_data: FIFOIO[MemData], lock: Bool, data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { - req_cmd.bits.rw := Bool(true) - req_data.bits := data.bits.payload - when(req_cmd.ready && req_cmd.valid) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + master_acq.bits.payload := cmd_to_write + master_acq_data.bits.payload := client_data.bits.payload + when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } when (at_front_of_dep_queue) { - req_cmd.valid := !cmd_sent && req_data.ready && data.valid - lock := data.valid || cmd_sent - when (req_cmd.ready || cmd_sent) { - req_data.valid := data.valid - when(req_data.ready) { - data.ready:= Bool(true) - when (data.valid) { + master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (master_acq.ready || cmd_sent) { + master_acq_data.valid := client_data.valid + when(master_acq_data.ready) { + client_data.ready:= Bool(true) + when (client_data.valid) { mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { pop_dep := Bool(true) @@ -846,10 +357,10 @@ trait MemoryRequestGenerator { } } - def doMemReqRead(req_cmd: FIFOIO[MemReqCmd], trigger: Bool) { - req_cmd.valid := Bool(true) - req_cmd.bits.rw := Bool(false) - when(req_cmd.ready) { + def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { + master_acq.valid := Bool(true) + master_acq.bits.payload := cmd_to_read + when(master_acq.ready) { trigger := Bool(false) } } From 4d007d5c404168104d23473d09615c8aee7930fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Jan 2013 20:09:21 -0800 Subject: [PATCH 169/688] changed val names in hub to match new tilelink names --- uncore/src/uncore.scala | 460 ++++++++++++++++++++-------------------- 1 file changed, 234 insertions(+), 226 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 01e9c40c..9d736e8b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -24,14 +24,14 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val p_data = (new PipeIO) { new TrackerProbeData }.flip val can_alloc = Bool(INPUT) val grant_ack = Bool(INPUT) - val p_rep_cnt_dec = Bits(INPUT, conf.ln.nTiles) - val p_req_cnt_inc = Bits(INPUT, conf.ln.nTiles) + val release_cnt_dec = Bits(INPUT, conf.ln.nTiles) + val probe_cnt_inc = Bits(INPUT, conf.ln.nTiles) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data = (new PipeIO) { new ReleaseData }.flip - val x_init_data = (new PipeIO) { new AcquireData }.flip + val release_data = (new PipeIO) { new ReleaseData }.flip + val acquire_data = (new PipeIO) { new AcquireData }.flip val sent_grant_ack = Bool(INPUT) - val p_rep_data_dep = (new PipeIO) { new TrackerDependency }.flip - val x_init_data_dep = (new PipeIO) { new TrackerDependency }.flip + val release_data_dep = (new PipeIO) { new TrackerDependency }.flip + val acquire_data_dep = (new PipeIO) { new TrackerDependency }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } @@ -40,17 +40,17 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val busy = Bool(OUTPUT) val addr = Bits(OUTPUT, PADDR_BITS - OFFSET_BITS) val init_client_id = Bits(OUTPUT, conf.ln.idBits) - val p_rep_client_id = Bits(OUTPUT, conf.ln.idBits) + val release_client_id = Bits(OUTPUT, conf.ln.idBits) val client_xact_id = Bits(OUTPUT, CLIENT_XACT_ID_BITS) val sharer_count = Bits(OUTPUT, conf.ln.idBits+1) val a_type = Bits(OUTPUT, ACQUIRE_TYPE_MAX_BITS) - val push_p_req = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_p_rep_dep = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_data = Bits(OUTPUT, conf.ln.nTiles) - val pop_x_init_dep = Bits(OUTPUT, conf.ln.nTiles) + val push_probe = Bits(OUTPUT, conf.ln.nTiles) + val pop_release = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_release_dep = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_data = Bits(OUTPUT, conf.ln.nTiles) + val pop_acquire_dep = Bits(OUTPUT, conf.ln.nTiles) val send_grant_ack = Bool(OUTPUT) } @@ -91,19 +91,19 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg{ Bits() } - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) - val p_rep_client_id_ = Reg{ Bits() } + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_client_id_ = Reg{ Bits() } val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically @@ -112,13 +112,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex case _ => Bool(false) } val myflag = Mux(probe_self, Bits(0), UFixToOH(io.alloc_req.bits.client_id(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + probe_initial_flags := ~(io.tile_incoherent | myflag) } io.busy := state != s_idle io.addr := xact.addr io.init_client_id := init_client_id_ - io.p_rep_client_id := p_rep_client_id_ + io.release_client_id := release_client_id_ io.client_xact_id := xact.client_xact_id io.sharer_count := UFix(conf.ln.nTiles) // TODO: Broadcast only io.a_type := xact.a_type @@ -134,13 +134,13 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex io.probe.bits.p_type := co.getProbeType(xact.a_type, UFix(0)) io.probe.bits.master_xact_id := UFix(id) io.probe.bits.addr := xact.addr - io.push_p_req := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_data := Bits(0, width = conf.ln.nTiles) - io.pop_p_rep_dep := Bits(0, width = conf.ln.nTiles) - io.pop_x_init := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_data := Bits(0, width = conf.ln.nTiles) - io.pop_x_init_dep := Bits(0, width = conf.ln.nTiles) + io.push_probe := Bits(0, width = conf.ln.nTiles) + io.pop_release := Bits(0, width = conf.ln.nTiles) + io.pop_release_data := Bits(0, width = conf.ln.nTiles) + io.pop_release_dep := Bits(0, width = conf.ln.nTiles) + io.pop_acquire := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_data := Bits(0, width = conf.ln.nTiles) + io.pop_acquire_dep := Bits(0, width = conf.ln.nTiles) io.send_grant_ack := Bool(false) switch (state) { @@ -148,62 +148,62 @@ class XactTrackerBroadcast(id: Int)(implicit conf: CoherenceHubConfiguration) ex when( io.alloc_req.valid && io.can_alloc ) { xact := io.alloc_req.bits.acquire init_client_id_ := io.alloc_req.bits.client_id - x_init_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) + acquire_data_needs_write := co.messageHasData(io.alloc_req.bits.acquire) x_needs_read := co.needsMemRead(io.alloc_req.bits.acquire.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.pop_x_init := UFix(1) << io.alloc_req.bits.client_id + io.pop_acquire := UFix(1) << io.alloc_req.bits.client_id if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - when(p_req_flags.orR) { - io.push_p_req := p_req_flags + when(probe_flags.orR) { + io.push_probe := probe_flags io.probe.valid := Bool(true) } - when(io.p_req_cnt_inc.orR) { - p_req_flags := p_req_flags & ~io.p_req_cnt_inc // unflag sent reqs + when(io.probe_cnt_inc.orR) { + probe_flags := probe_flags & ~io.probe_cnt_inc // unflag sent reqs } - when(io.p_rep_cnt_dec.orR) { - val dec = PopCount(io.p_rep_cnt_dec) - io.pop_p_rep := io.p_rep_cnt_dec - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - dec - when(p_rep_count === dec) { + when(io.release_cnt_dec.orR) { + val dec = PopCount(io.release_cnt_dec) + io.pop_release := io.release_cnt_dec + if(conf.ln.nTiles > 1) release_count := release_count - dec + when(release_count === dec) { state := s_mem } } when(io.p_data.valid) { - p_rep_data_needs_write := Bool(true) - p_rep_client_id_ := io.p_data.bits.client_id + release_data_needs_write := Bool(true) + release_client_id_ := io.p_data.bits.client_id } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.pop_p_rep_data, - io.pop_p_rep_dep, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id)), - p_rep_client_id_) - } . elsewhen(x_init_data_needs_write) { + io.pop_release_data, + io.pop_release_dep, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id)), + release_client_id_) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.pop_x_init_data, - io.pop_x_init_dep, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id)), + io.pop_acquire_data, + io.pop_acquire_dep, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id)), init_client_id_) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) @@ -274,22 +274,22 @@ class CoherenceHubNull(implicit conf: CoherenceHubConfiguration) extends Coheren { val co = conf.co.asInstanceOf[ThreeStateIncoherence] - val x_init = io.tiles(0).acquire - val is_write = x_init.bits.payload.a_type === co.acquireWriteback - x_init.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp - io.mem.req_cmd.valid := x_init.valid && !(is_write && io.mem.resp.valid) + val acquire = io.tiles(0).acquire + val is_write = acquire.bits.payload.a_type === co.acquireWriteback + acquire.ready := io.mem.req_cmd.ready && !(is_write && io.mem.resp.valid) //stall write req/resp to handle previous read resp + io.mem.req_cmd.valid := acquire.valid && !(is_write && io.mem.resp.valid) io.mem.req_cmd.bits.rw := is_write - io.mem.req_cmd.bits.tag := x_init.bits.payload.client_xact_id - io.mem.req_cmd.bits.addr := x_init.bits.payload.addr + io.mem.req_cmd.bits.tag := acquire.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := acquire.bits.payload.addr io.mem.req_data <> io.tiles(0).acquire_data val grant = io.tiles(0).grant grant.bits.payload.g_type := Mux(io.mem.resp.valid, co.grantData, co.grantAck) - grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, x_init.bits.payload.client_xact_id) + grant.bits.payload.client_xact_id := Mux(io.mem.resp.valid, io.mem.resp.bits.tag, acquire.bits.payload.client_xact_id) grant.bits.payload.master_xact_id := UFix(0) // don't care grant.bits.payload.data := io.mem.resp.bits.data grant.bits.payload.require_ack := Bool(true) - grant.valid := io.mem.resp.valid || x_init.valid && is_write && io.mem.req_cmd.ready + grant.valid := io.mem.resp.valid || acquire.valid && is_write && io.mem.req_cmd.ready io.tiles(0).abort.valid := Bool(false) io.tiles(0).grant_ack.ready := Bool(true) @@ -314,8 +314,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val send_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val do_free_arr = Vec(NGLOBAL_XACTS){ Bool() } - val p_rep_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } - val p_req_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val release_cnt_dec_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } + val probe_cnt_inc_arr = VecBuf(NGLOBAL_XACTS){ Vec(conf.ln.nTiles){ Bool()} } val sent_grant_ack_arr = Vec(NGLOBAL_XACTS){ Bool() } val p_data_client_id_arr = Vec(NGLOBAL_XACTS){ Bits(width=conf.ln.idBits) } val p_data_valid_arr = Vec(NGLOBAL_XACTS){ Bool() } @@ -332,8 +332,8 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co t.grant_ack := do_free_arr(i) t.p_data.bits.client_id := p_data_client_id_arr(i) t.p_data.valid := p_data_valid_arr(i) - t.p_rep_cnt_dec := p_rep_cnt_dec_arr(i).toBits - t.p_req_cnt_inc := p_req_cnt_inc_arr(i).toBits + t.release_cnt_dec := release_cnt_dec_arr(i).toBits + t.probe_cnt_inc := probe_cnt_inc_arr(i).toBits t.tile_incoherent := io.incoherent.toBits t.sent_grant_ack := sent_grant_ack_arr(i) do_free_arr(i) := Bool(false) @@ -341,21 +341,21 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co p_data_client_id_arr(i) := Bits(0, width = conf.ln.idBits) p_data_valid_arr(i) := Bool(false) for( j <- 0 until conf.ln.nTiles) { - p_rep_cnt_dec_arr(i)(j) := Bool(false) - p_req_cnt_inc_arr(i)(j) := Bool(false) + release_cnt_dec_arr(i)(j) := Bool(false) + probe_cnt_inc_arr(i)(j) := Bool(false) } } - val p_rep_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY - val x_init_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY + val release_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth must >= NPRIMARY + val acquire_data_dep_list = List.fill(conf.ln.nTiles)((new Queue(NGLOBAL_XACTS)){new TrackerDependency}) // depth should >= NPRIMARY // Free finished transactions for( j <- 0 until conf.ln.nTiles ) { - val finish = io.tiles(j).grant_ack - when (finish.valid) { - do_free_arr(finish.bits.payload.master_xact_id) := Bool(true) + val ack = io.tiles(j).grant_ack + when (ack.valid) { + do_free_arr(ack.bits.payload.master_xact_id) := Bool(true) } - finish.ready := Bool(true) + ack.ready := Bool(true) } // Reply to initial requestor @@ -404,31 +404,31 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co // Handle probe replies, which may or may not have data for( j <- 0 until conf.ln.nTiles ) { - val p_rep = io.tiles(j).release - val p_rep_data = io.tiles(j).release_data - val idx = p_rep.bits.payload.master_xact_id - val pop_p_reps = trackerList.map(_.io.pop_p_rep(j).toBool) - val do_pop = foldR(pop_p_reps)(_ || _) - p_rep.ready := Bool(true) - p_rep_data_dep_list(j).io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_list(j).io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data.ready := foldR(trackerList.map(_.io.pop_p_rep_data(j)))(_ || _) - when (p_rep.valid && co.messageHasData(p_rep.bits.payload)) { + val release = io.tiles(j).release + val release_data = io.tiles(j).release_data + val idx = release.bits.payload.master_xact_id + val pop_releases = trackerList.map(_.io.pop_release(j).toBool) + val do_pop = foldR(pop_releases)(_ || _) + release.ready := Bool(true) + release_data_dep_list(j).io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_list(j).io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data.ready := foldR(trackerList.map(_.io.pop_release_data(j)))(_ || _) + when (release.valid && co.messageHasData(release.bits.payload)) { p_data_valid_arr(idx) := Bool(true) p_data_client_id_arr(idx) := UFix(j) } - p_rep_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_p_rep_dep(j).toBool))(_||_) + release_data_dep_list(j).io.deq.ready := foldR(trackerList.map(_.io.pop_release_dep(j).toBool))(_||_) } for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := io.tiles(trackerList(i).io.p_rep_client_id).release_data.valid - trackerList(i).io.p_rep_data.bits := io.tiles(trackerList(i).io.p_rep_client_id).release_data.bits.payload + trackerList(i).io.release_data.valid := io.tiles(trackerList(i).io.release_client_id).release_data.valid + trackerList(i).io.release_data.bits := io.tiles(trackerList(i).io.release_client_id).release_data.bits.payload - trackerList(i).io.p_rep_data_dep.valid := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.valid)) - trackerList(i).io.p_rep_data_dep.bits := MuxLookup(trackerList(i).io.p_rep_client_id, p_rep_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> p_rep_data_dep_list(j).io.deq.bits)) + trackerList(i).io.release_data_dep.valid := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.valid)) + trackerList(i).io.release_data_dep.bits := MuxLookup(trackerList(i).io.release_client_id, release_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> release_data_dep_list(j).io.deq.bits)) for( j <- 0 until conf.ln.nTiles) { - val p_rep = io.tiles(j).release - p_rep_cnt_dec_arr(i)(j) := p_rep.valid && (p_rep.bits.payload.master_xact_id === UFix(i)) + val release = io.tiles(j).release + release_cnt_dec_arr(i)(j) := release.valid && (release.bits.payload.master_xact_id === UFix(i)) } } @@ -437,33 +437,33 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co val abort_state_arr = Vec(conf.ln.nTiles) { Reg(resetVal = s_idle) } val want_to_abort_arr = Vec(conf.ln.nTiles) { Bool() } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data val x_abort = io.tiles(j).abort val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val conflicts = Vec(NGLOBAL_XACTS) { Bool() } for( i <- 0 until NGLOBAL_XACTS) { val t = trackerList(i).io - conflicts(i) := t.busy && x_init.valid && co.isCoherenceConflict(t.addr, x_init.bits.payload.addr) + conflicts(i) := t.busy && acquire.valid && co.isCoherenceConflict(t.addr, acquire.bits.payload.addr) } - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.bits.header.dst := UFix(0) // DNC x_abort.bits.header.src := UFix(0) // DNC - want_to_abort_arr(j) := x_init.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!x_init_data_dep_list(j).io.enq.ready && co.messageHasData(x_init.bits.payload))) + want_to_abort_arr(j) := acquire.valid && (conflicts.toBits.orR || busy_arr.toBits.andR || (!acquire_data_dep_list(j).io.enq.ready && co.messageHasData(acquire.bits.payload))) x_abort.valid := Bool(false) switch(abort_state_arr(j)) { is(s_idle) { when(want_to_abort_arr(j)) { - when(co.messageHasData(x_init.bits.payload)) { + when(co.messageHasData(acquire.bits.payload)) { abort_state_arr(j) := s_abort_drain } . otherwise { abort_state_arr(j) := s_abort_send } } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state_arr(j) := s_abort_send @@ -472,7 +472,7 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready below + when(x_abort.ready) { // raises acquire.ready below abort_state_arr(j) := s_idle } } @@ -490,41 +490,41 @@ class CoherenceHubBroadcast(implicit conf: CoherenceHubConfiguration) extends Co trackerList(i).io.alloc_req.bits := init_arb.io.out.bits trackerList(i).io.alloc_req.valid := init_arb.io.out.valid - trackerList(i).io.x_init_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload - trackerList(i).io.x_init_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid - trackerList(i).io.x_init_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.bits)) - trackerList(i).io.x_init_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, x_init_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> x_init_data_dep_list(j).io.deq.valid)) + trackerList(i).io.acquire_data.bits := io.tiles(trackerList(i).io.init_client_id).acquire_data.bits.payload + trackerList(i).io.acquire_data.valid := io.tiles(trackerList(i).io.init_client_id).acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.bits, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.bits)) + trackerList(i).io.acquire_data_dep.valid := MuxLookup(trackerList(i).io.init_client_id, acquire_data_dep_list(0).io.deq.valid, (0 until conf.ln.nTiles).map( j => UFix(j) -> acquire_data_dep_list(j).io.deq.valid)) } for( j <- 0 until conf.ln.nTiles ) { - val x_init = io.tiles(j).acquire - val x_init_data = io.tiles(j).acquire_data - val x_init_data_dep = x_init_data_dep_list(j).io.deq + val acquire = io.tiles(j).acquire + val acquire_data = io.tiles(j).acquire_data + val acquire_data_dep = acquire_data_dep_list(j).io.deq val x_abort = io.tiles(j).abort - init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && x_init.valid - init_arb.io.in(j).bits.acquire := x_init.bits.payload + init_arb.io.in(j).valid := (abort_state_arr(j) === s_idle) && !want_to_abort_arr(j) && acquire.valid + init_arb.io.in(j).bits.acquire := acquire.bits.payload init_arb.io.in(j).bits.client_id := UFix(j) - val pop_x_inits = trackerList.map(_.io.pop_x_init(j).toBool) - val do_pop = foldR(pop_x_inits)(_||_) - x_init_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(x_init.bits.payload) && (abort_state_arr(j) === s_idle) - x_init_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_x_inits) - x_init.ready := (x_abort.valid && x_abort.ready) || do_pop - x_init_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_x_init_data(j).toBool))(_||_) - x_init_data_dep.ready := foldR(trackerList.map(_.io.pop_x_init_dep(j).toBool))(_||_) + val pop_acquires = trackerList.map(_.io.pop_acquire(j).toBool) + val do_pop = foldR(pop_acquires)(_||_) + acquire_data_dep_list(j).io.enq.valid := do_pop && co.messageHasData(acquire.bits.payload) && (abort_state_arr(j) === s_idle) + acquire_data_dep_list(j).io.enq.bits.master_xact_id := OHToUFix(pop_acquires) + acquire.ready := (x_abort.valid && x_abort.ready) || do_pop + acquire_data.ready := (abort_state_arr(j) === s_abort_drain) || foldR(trackerList.map(_.io.pop_acquire_data(j).toBool))(_||_) + acquire_data_dep.ready := foldR(trackerList.map(_.io.pop_acquire_dep(j).toBool))(_||_) } alloc_arb.io.out.ready := init_arb.io.out.valid // Handle probe request generation // Must arbitrate for each request port - val p_req_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) + val probe_arb_arr = List.fill(conf.ln.nTiles)((new Arbiter(NGLOBAL_XACTS)) { new Probe() }) for( j <- 0 until conf.ln.nTiles ) { for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb_arr(j).io.in(i).bits := t.probe.bits - p_req_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_p_req(j) - p_req_cnt_inc_arr(i)(j) := p_req_arb_arr(j).io.in(i).ready + probe_arb_arr(j).io.in(i).bits := t.probe.bits + probe_arb_arr(j).io.in(i).valid := t.probe.valid && t.push_probe(j) + probe_cnt_inc_arr(i)(j) := probe_arb_arr(j).io.in(i).ready } - FIFOedLogicalNetworkIOWrapper(p_req_arb_arr(j).io.out) <> io.tiles(j).probe + FIFOedLogicalNetworkIOWrapper(probe_arb_arr(j).io.out) <> io.tiles(j).probe } } @@ -542,8 +542,8 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren implicit val lnConf = conf.ln val co = conf.co val trackerList = (0 until NGLOBAL_XACTS).map(new XactTracker(_)) - val p_rep_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val x_init_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY + val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i) @@ -556,50 +556,50 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren // Handle transaction initiation requests // Only one allocation per cycle // Init requests may or may not have data - val x_init = io.network.acquire - val x_init_data = io.network.acquire_data + val acquire = io.network.acquire + val acquire_data = io.network.acquire_data val x_abort = io.network.abort - val x_dep_deq = x_init_data_dep_q.io.deq + val x_dep_deq = acquire_data_dep_q.io.deq val s_idle :: s_abort_drain :: s_abort_send :: Nil = Enum(3){ UFix() } val abort_state = Reg(resetVal = s_idle) val abort_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val any_conflict = trackerList.map(_.io.has_conflict).reduce(_||_) val all_busy = trackerList.map(_.io.busy).reduce(_&&_) - val want_to_abort = x_init.valid && (any_conflict || all_busy || (!x_init_data_dep_q.io.enq.ready && co.messageHasData(x_init.bits.payload))) + val want_to_abort = acquire.valid && (any_conflict || all_busy || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload))) val alloc_arb = (new Arbiter(NGLOBAL_XACTS)) { Bool() } for( i <- 0 until NGLOBAL_XACTS ) { alloc_arb.io.in(i).valid := !trackerList(i).io.busy - trackerList(i).io.x_init.bits := x_init.bits - trackerList(i).io.x_init.valid := (abort_state === s_idle) && !want_to_abort && x_init.valid && alloc_arb.io.in(i).ready + trackerList(i).io.acquire.bits := acquire.bits + trackerList(i).io.acquire.valid := (abort_state === s_idle) && !want_to_abort && acquire.valid && alloc_arb.io.in(i).ready - trackerList(i).io.x_init_data.bits := x_init_data.bits - trackerList(i).io.x_init_data.valid := x_init_data.valid - trackerList(i).io.x_init_data_dep.bits := x_dep_deq.bits - trackerList(i).io.x_init_data_dep.valid := x_dep_deq.valid + trackerList(i).io.acquire_data.bits := acquire_data.bits + trackerList(i).io.acquire_data.valid := acquire_data.valid + trackerList(i).io.acquire_data_dep.bits := x_dep_deq.bits + trackerList(i).io.acquire_data_dep.valid := x_dep_deq.valid } - val pop_x_init = trackerList.map(_.io.x_init.ready).reduce(_||_) - x_init.ready := (x_abort.valid && x_abort.ready) || pop_x_init - x_init_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.x_init_data.ready).reduce(_||_) - x_init_data_dep_q.io.enq.valid := pop_x_init && co.messageHasData(x_init.bits.payload) && (abort_state === s_idle) - x_init_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.x_init.ready)) - x_dep_deq.ready := trackerList.map(_.io.x_init_data_dep.ready).reduce(_||_) + val pop_acquire = trackerList.map(_.io.acquire.ready).reduce(_||_) + acquire.ready := (x_abort.valid && x_abort.ready) || pop_acquire + acquire_data.ready := (abort_state === s_abort_drain) || trackerList.map(_.io.acquire_data.ready).reduce(_||_) + acquire_data_dep_q.io.enq.valid := pop_acquire && co.messageHasData(acquire.bits.payload) && (abort_state === s_idle) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.acquire.ready)) + x_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := x_init.valid + alloc_arb.io.out.ready := acquire.valid // Nack conflicting transaction init attempts - x_abort.bits.header.dst := x_init.bits.header.src + x_abort.bits.header.dst := acquire.bits.header.src x_abort.bits.header.src := UFix(0) //DNC - x_abort.bits.payload.client_xact_id := x_init.bits.payload.client_xact_id + x_abort.bits.payload.client_xact_id := acquire.bits.payload.client_xact_id x_abort.valid := Bool(false) switch(abort_state) { is(s_idle) { when(want_to_abort) { - abort_state := Mux( co.messageHasData(x_init.bits.payload), s_abort_drain, s_abort_send) + abort_state := Mux( co.messageHasData(acquire.bits.payload), s_abort_drain, s_abort_send) } } - is(s_abort_drain) { // raises x_init_data.ready below - when(x_init_data.valid) { + is(s_abort_drain) { // raises acquire_data.ready below + when(acquire_data.valid) { abort_cnt := abort_cnt + UFix(1) when(abort_cnt === ~UFix(0, width = log2Up(REFILL_CYCLES))) { abort_state := s_abort_send @@ -608,38 +608,38 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } is(s_abort_send) { // nothing is dequeued for now x_abort.valid := Bool(true) - when(x_abort.ready) { // raises x_init.ready + when(x_abort.ready) { // raises acquire.ready abort_state := s_idle } } } // Handle probe request generation - val p_req_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = (new Arbiter(NGLOBAL_XACTS)){(new LogicalNetworkIO){ new Probe }} for( i <- 0 until NGLOBAL_XACTS ) { val t = trackerList(i).io - p_req_arb.io.in(i).bits := t.p_req.bits - p_req_arb.io.in(i).valid := t.p_req.valid - t.p_req.ready := p_req_arb.io.in(i).ready + probe_arb.io.in(i).bits := t.probe.bits + probe_arb.io.in(i).valid := t.probe.valid + t.probe.ready := probe_arb.io.in(i).ready } - io.network.probe <> p_req_arb.io.out + io.network.probe <> probe_arb.io.out // Handle probe replies, which may or may not have data - val p_rep = io.network.release - val p_rep_data = io.network.release_data - val idx = p_rep.bits.payload.master_xact_id - p_rep.ready := trackerList.map(_.io.p_rep.ready).reduce(_||_) - p_rep_data.ready := trackerList.map(_.io.p_rep_data.ready).reduce(_||_) - p_rep_data_dep_q.io.enq.valid := p_rep.valid && co.messageHasData(p_rep.bits.payload) - p_rep_data_dep_q.io.enq.bits.master_xact_id := p_rep.bits.payload.master_xact_id - p_rep_data_dep_q.io.deq.ready := trackerList.map(_.io.p_rep_data_dep.ready).reduce(_||_) + val release = io.network.release + val release_data = io.network.release_data + val idx = release.bits.payload.master_xact_id + release.ready := trackerList.map(_.io.release.ready).reduce(_||_) + release_data.ready := trackerList.map(_.io.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := release.bits.payload.master_xact_id + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.p_rep_data.valid := p_rep_data.valid - trackerList(i).io.p_rep_data.bits := p_rep_data.bits - trackerList(i).io.p_rep_data_dep.valid := p_rep_data_dep_q.io.deq.valid - trackerList(i).io.p_rep_data_dep.bits := p_rep_data_dep_q.io.deq.bits - trackerList(i).io.p_rep.valid := p_rep.valid && (idx === UFix(i)) - trackerList(i).io.p_rep.bits := p_rep.bits + trackerList(i).io.release_data.valid := release_data.valid + trackerList(i).io.release_data.bits := release_data.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release.valid := release.valid && (idx === UFix(i)) + trackerList(i).io.release.bits := release.bits } // Reply to initial requestor @@ -664,11 +664,11 @@ class L2CoherenceAgent(implicit conf: CoherenceHubConfiguration) extends Coheren } // Free finished transactions - val finish = io.network.grant_ack + val ack = io.network.grant_ack for( i <- 0 until NGLOBAL_XACTS ) { - trackerList(i).io.free := finish.valid && (finish.bits.payload.master_xact_id === UFix(i)) + trackerList(i).io.free := ack.valid && (ack.bits.payload.master_xact_id === UFix(i)) } - finish.ready := Bool(true) + ack.ready := Bool(true) // Create an arbiter for the one memory port // We have to arbitrate between the different trackers' memory requests @@ -688,20 +688,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val co = conf.co implicit val ln = conf.ln val io = new Bundle { - val x_init = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip - val x_init_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip - val p_rep = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip - val p_rep_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip + val acquire = (new FIFOIO){(new LogicalNetworkIO) { new Acquire }}.flip + val acquire_data = (new FIFOIO){(new LogicalNetworkIO) { new AcquireData }}.flip + val release = (new FIFOIO){(new LogicalNetworkIO) { new Release }}.flip + val release_data = (new FIFOIO){(new LogicalNetworkIO) { new ReleaseData }}.flip val free = Bool(INPUT) val tile_incoherent = Bits(INPUT, conf.ln.nTiles) - val p_rep_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val x_init_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip + val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip val mem_resp = (new FIFOIO) { new MemResp }.flip val mem_req_cmd = (new FIFOIO) { new MemReqCmd } val mem_req_data = (new FIFOIO) { new MemData } val mem_req_lock = Bool(OUTPUT) - val p_req = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} + val probe = (new FIFOIO) {(new LogicalNetworkIO) { new Probe }} val grant = (new FIFOIO) {(new LogicalNetworkIO) { new Grant }} val busy = Bool(OUTPUT) val has_conflict = Bool(OUTPUT) @@ -713,32 +713,32 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_rep_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) - val p_req_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) + val release_count = if (conf.ln.nTiles == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nTiles))) + val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nTiles)) val x_needs_read = Reg(resetVal = Bool(false)) - val x_init_data_needs_write = Reg(resetVal = Bool(false)) - val p_rep_data_needs_write = Reg(resetVal = Bool(false)) + val acquire_data_needs_write = Reg(resetVal = Bool(false)) + val release_data_needs_write = Reg(resetVal = Bool(false)) val x_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val p_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) val mem_cnt_max = ~UFix(0, width = log2Up(REFILL_CYCLES)) - val p_req_initial_flags = Bits(width = conf.ln.nTiles) - p_req_initial_flags := Bits(0) + val probe_initial_flags = Bits(width = conf.ln.nTiles) + probe_initial_flags := Bits(0) if (conf.ln.nTiles > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence // TODO: this is hackish; figure out how to do it more systematically val probe_self = co match { - case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.x_init.bits.payload) + case u: CoherencePolicyWithUncached => u.isUncachedReadTransaction(io.acquire.bits.payload) case _ => Bool(false) } - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.x_init.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) - p_req_initial_flags := ~(io.tile_incoherent | myflag) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.acquire.bits.header.src(log2Up(conf.ln.nTiles)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) } val all_grants_require_acks = Bool(true) io.busy := state != s_idle - io.has_conflict := co.isCoherenceConflict(xact.addr, io.x_init.bits.payload.addr) && (state != s_idle) + io.has_conflict := co.isCoherenceConflict(xact.addr, io.acquire.bits.payload.addr) && (state != s_idle) io.mem_req_cmd.valid := Bool(false) io.mem_req_cmd.bits.rw := Bool(false) io.mem_req_cmd.bits.addr := xact.addr @@ -746,12 +746,20 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.mem_req_data.valid := Bool(false) io.mem_req_data.bits.data := UFix(0) io.mem_req_lock := Bool(false) +<<<<<<< HEAD io.p_req.valid := Bool(false) io.p_req.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.p_req.bits.payload.master_xact_id := UFix(id) io.p_req.bits.payload.addr := xact.addr io.p_req.bits.header.dst := UFix(0) io.p_req.bits.header.src := UFix(0) // DNC +======= + io.probe.valid := Bool(false) + io.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) + io.probe.bits.payload.master_xact_id := UFix(id) + io.probe.bits.payload.addr := xact.addr + io.probe.bits.header.dst := UFix(0) +>>>>>>> changed val names in hub to match new tilelink names io.grant.bits.payload.data := io.mem_resp.bits.data io.grant.bits.payload.g_type := co.getGrantType(xact.a_type, init_sharer_cnt_) io.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -760,70 +768,70 @@ class XactTracker(id: Int)(implicit conf: CoherenceHubConfiguration) extends Com io.grant.bits.header.dst := init_client_id_ io.grant.bits.header.src := UFix(0) // DNC io.grant.valid := (io.mem_resp.valid && (UFix(id) === io.mem_resp.bits.tag)) - io.x_init.ready := Bool(false) - io.x_init_data.ready := Bool(false) - io.x_init_data_dep.ready := Bool(false) - io.p_rep.ready := Bool(false) - io.p_rep_data.ready := Bool(false) - io.p_rep_data_dep.ready := Bool(false) + io.acquire.ready := Bool(false) + io.acquire_data.ready := Bool(false) + io.acquire_data_dep.ready := Bool(false) + io.release.ready := Bool(false) + io.release_data.ready := Bool(false) + io.release_data_dep.ready := Bool(false) io.mem_resp.ready := io.grant.ready switch (state) { is(s_idle) { - when( io.x_init.valid ) { - xact := io.x_init.bits.payload - init_client_id_ := io.x_init.bits.header.src + when( io.acquire.valid ) { + xact := io.acquire.bits.payload + init_client_id_ := io.acquire.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nTiles) // TODO: Broadcast only - x_init_data_needs_write := co.messageHasData(io.x_init.bits.payload) - x_needs_read := co.needsMemRead(io.x_init.bits.payload.a_type, UFix(0)) - p_req_flags := p_req_initial_flags + acquire_data_needs_write := co.messageHasData(io.acquire.bits.payload) + x_needs_read := co.needsMemRead(io.acquire.bits.payload.a_type, UFix(0)) + probe_flags := probe_initial_flags mem_cnt := UFix(0) p_w_mem_cmd_sent := Bool(false) x_w_mem_cmd_sent := Bool(false) - io.x_init.ready := Bool(true) + io.acquire.ready := Bool(true) if(conf.ln.nTiles > 1) { - p_rep_count := PopCount(p_req_initial_flags) - state := Mux(p_req_initial_flags.orR, s_probe, s_mem) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem } } is(s_probe) { - val curr_p_id = PriorityEncoder(p_req_flags) - when(p_req_flags.orR) { - io.p_req.valid := Bool(true) - io.p_req.bits.header.dst := curr_p_id + val curr_p_id = PriorityEncoder(probe_flags) + when(probe_flags.orR) { + io.probe.valid := Bool(true) + io.probe.bits.header.dst := curr_p_id } - when(io.p_req.ready) { - p_req_flags := p_req_flags & ~(UFixToOH(curr_p_id)) + when(io.probe.ready) { + probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - when(io.p_rep.valid) { - io.p_rep.ready := Bool(true) - if(conf.ln.nTiles > 1) p_rep_count := p_rep_count - UFix(1) - when(p_rep_count === UFix(1)) { + when(io.release.valid) { + io.release.ready := Bool(true) + if(conf.ln.nTiles > 1) release_count := release_count - UFix(1) + when(release_count === UFix(1)) { state := s_mem } - p_rep_data_needs_write := co.messageHasData(io.p_rep.bits.payload) + release_data_needs_write := co.messageHasData(io.release.bits.payload) } } is(s_mem) { - when (p_rep_data_needs_write) { + when (release_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.p_rep_data, - p_rep_data_needs_write, + io.release_data, + release_data_needs_write, p_w_mem_cmd_sent, - io.p_rep_data_dep.ready, - io.p_rep_data_dep.valid && (io.p_rep_data_dep.bits.master_xact_id === UFix(id))) - } . elsewhen(x_init_data_needs_write) { + io.release_data_dep.ready, + io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(id))) + } . elsewhen(acquire_data_needs_write) { doMemReqWrite(io.mem_req_cmd, io.mem_req_data, io.mem_req_lock, - io.x_init_data, - x_init_data_needs_write, + io.acquire_data, + acquire_data_needs_write, x_w_mem_cmd_sent, - io.x_init_data_dep.ready, - io.x_init_data_dep.valid && (io.x_init_data_dep.bits.master_xact_id === UFix(id))) + io.acquire_data_dep.ready, + io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(id))) } . elsewhen (x_needs_read) { doMemReqRead(io.mem_req_cmd, x_needs_read) } . otherwise { From 67fc09f62ec7c5487d33fd851ef4c96477add9d3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 25 Mar 2013 16:20:12 -0700 Subject: [PATCH 170/688] Fixes after merge, and always self probe. --- uncore/src/coherence.scala | 10 ++++---- uncore/src/uncore.scala | 52 ++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b4c4067f..1fa09fe6 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -274,7 +274,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -431,7 +431,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -592,7 +592,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -758,7 +758,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -940,7 +940,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = acq.a_type === acquireReadUncached + def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 14467f7b..ea3c20c4 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -37,33 +37,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle transaction initiation requests - // Only one allocation per cycle - // Init requests may or may not have data + // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val all_busy = trackerList.map(_.io.busy).reduce(_&&_) + val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { - alloc_arb.io.in(i).valid := !trackerList(i).io.busy val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.valid := acquire.valid && alloc_arb.io.in(i).ready + t.acquire.valid := alloc_arb.io.in(i).ready t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(trackerList.map(_.io.client.acquire.ready)) + acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) + acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -75,22 +73,23 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) - release.ready := trackerList.map(_.io.client.release.ready).reduce(_||_) - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.valid := release.valid && (idx === UFix(i)) t.release.bits := release.bits - t.release_data.valid := release_data.valid + t.release.valid := release.valid && (idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid + t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits + trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) + release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -121,7 +120,6 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val busy = Bool(OUTPUT) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -139,7 +137,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.acquire_data_dep.ready := Bool(false) io.release_data_dep.ready := Bool(false) - io.busy := Bool(true) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -156,7 +153,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.acquire_data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.release_data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -167,13 +164,13 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu switch (state) { is(s_idle) { + io.client.release.ready := Bool(true) when( io.client.release.valid ) { xact := io.client.release.bits.payload init_client_id_ := io.client.release.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) - io.client.release.ready := Bool(true) state := s_mem } } @@ -205,6 +202,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) @@ -216,12 +214,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura probe_initial_flags := Bits(0) if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.busy := state != s_idle io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) io.master.acquire.valid := Bool(false) @@ -256,6 +253,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura switch (state) { is(s_idle) { + io.client.acquire.ready := Bool(true) when( io.client.acquire.valid ) { xact := io.client.acquire.bits.payload init_client_id_ := io.client.acquire.bits.header.src @@ -266,7 +264,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - io.client.acquire.ready := Bool(true) if(conf.ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) @@ -274,7 +271,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } } is(s_probe) { - val curr_p_id = PriorityEncoder(probe_flags) when(probe_flags.orR) { io.client.probe.valid := Bool(true) io.client.probe.bits.header.dst := curr_p_id @@ -282,13 +278,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } + io.client.release.ready := Bool(true) when(io.client.release.valid) { - io.client.release.ready := Bool(true) if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) } } is(s_mem) { From b6cc08e8ca563617607c07e37c5427f2d2813fef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 28 Mar 2013 14:09:08 -0700 Subject: [PATCH 171/688] override io in LogicalNetwork --- uncore/src/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 492079cc..f353a524 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -49,7 +49,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - val io: Vec[TileLinkType] + override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } From 3479f1c6cd86bc0d555b1cb0c20660a274a098c6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 3 Apr 2013 22:13:51 -0700 Subject: [PATCH 172/688] add LR/SC support --- uncore/src/coherence.scala | 12 ++++-------- uncore/src/consts.scala | 40 +++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 1fa09fe6..d97cc2da 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -121,8 +121,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadDirty, acquireReadClean) + Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -517,8 +516,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -679,8 +677,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, acquireReadExclusive, acquireReadShared) + Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) @@ -857,8 +854,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write || cmd === M_PFW, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { val (read, write) = cpuCmdToRW(cmd) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index d348e0ff..cde8d3b9 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -45,27 +45,31 @@ trait MemoryOpConstants { val MT_HU = Bits("b101", 3); val MT_WU = Bits("b110", 3); - val M_X = Bits("b????", 4); - val M_XRD = Bits("b0000", 4); // int load - val M_XWR = Bits("b0001", 4); // int store - val M_PFR = Bits("b0010", 4); // prefetch with intent to read - val M_PFW = Bits("b0011", 4); // prefetch with intent to write - val M_FENCE = Bits("b0101", 4); // memory fence - val M_INV = Bits("b0110", 4); // write back and invalidate line - val M_CLN = Bits("b0111", 4); // write back line - val M_XA_ADD = Bits("b1000", 4); - val M_XA_SWAP = Bits("b1001", 4); - val M_XA_AND = Bits("b1010", 4); - val M_XA_OR = Bits("b1011", 4); - val M_XA_MIN = Bits("b1100", 4); - val M_XA_MAX = Bits("b1101", 4); - val M_XA_MINU = Bits("b1110", 4); - val M_XA_MAXU = Bits("b1111", 4); + val M_SZ = 5 + val M_X = Bits("b?????"); + val M_XRD = Bits("b00000"); // int load + val M_XWR = Bits("b00001"); // int store + val M_PFR = Bits("b00010"); // prefetch with intent to read + val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_FENCE = Bits("b00101"); // memory fence + val M_XLR = Bits("b00110"); + val M_XSC = Bits("b00111"); + val M_XA_ADD = Bits("b01000"); + val M_XA_SWAP = Bits("b01001"); + val M_XA_AND = Bits("b01010"); + val M_XA_OR = Bits("b01011"); + val M_XA_MIN = Bits("b01100"); + val M_XA_MAX = Bits("b01101"); + val M_XA_MINU = Bits("b01110"); + val M_XA_MAXU = Bits("b01111"); + val M_INV = Bits("b10000"); // write back and invalidate line + val M_CLN = Bits("b10001"); // write back line def isAMO(cmd: Bits) = cmd(3) def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW - def isRead(cmd: Bits) = cmd === M_XRD || isAMO(cmd) - def isWrite(cmd: Bits) = cmd === M_XWR || isAMO(cmd) + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) + def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } trait MemoryInterfaceConstants extends From 7ff5b5b86f1dfa508b4c67a3d2c1127da7611f61 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 7 Apr 2013 19:23:44 -0700 Subject: [PATCH 173/688] treat load-reserved as a non-dirtying store --- uncore/src/coherence.scala | 66 +++++++++++++------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index d97cc2da..20157456 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -7,10 +7,6 @@ trait CoherenceAgentRole trait ClientCoherenceAgent extends CoherenceAgentRole trait MasterCoherenceAgent extends CoherenceAgentRole -object cpuCmdToRW { - def apply(cmd: Bits): (Bool, Bool) = (isRead(cmd) || isPrefetch(cmd), isWrite(cmd)) -} - abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -102,8 +98,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { def needsWriteback (state: UFix): Bool = state === tileDirty def newState(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileDirty, Mux(read, Mux(state === tileDirty, tileDirty, tileClean), state)) + Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO @@ -124,8 +119,7 @@ class ThreeStateIncoherence extends IncoherentPolicy { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData @@ -296,9 +290,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def isValid (state: UFix): Bool = state != tileInvalid def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusiveDirty)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -311,8 +304,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -352,12 +344,10 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, acquireReadExclusiveClean) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusiveDirty, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -450,8 +440,7 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -459,9 +448,8 @@ class MSICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -474,8 +462,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -519,8 +506,7 @@ class MSICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -611,8 +597,7 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, (state === tileExclusiveClean || state === tileExclusiveDirty), + Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } def isValid (state: UFix): Bool = { @@ -620,9 +605,8 @@ class MESICoherence extends CoherencePolicyWithUncached { } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -635,8 +619,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, tileExclusiveDirty, state) + Mux(isWrite(cmd), tileExclusiveDirty, state) } def newStateOnCacheControl(cmd: Bits) = { MuxLookup(cmd, tileInvalid, Array( @@ -680,8 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, acquireReadExclusive, outstanding.a_type) + Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) @@ -775,17 +757,15 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: Bits, state: UFix): Bool = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UFix): Bool = { state != tileInvalid } def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { - val (read, write) = cpuCmdToRW(cmd) - (read && messageIsUncached(outstanding)) || - (write && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) + (isRead(cmd) && messageIsUncached(outstanding)) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( @@ -798,8 +778,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnHit(cmd: Bits, state: UFix): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, MuxLookup(state, tileExclusiveDirty, Array( + Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } @@ -857,8 +836,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { - val (read, write) = cpuCmdToRW(cmd) - Mux(write, Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) From 74187c2068144b640347fa674014bfb868e3428e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Apr 2013 14:09:36 -0700 Subject: [PATCH 174/688] Always route voluntary releases to ReleaseTracker to ensure all grants are sent --- uncore/src/uncore.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index ea3c20c4..3e54b9e3 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -68,27 +68,28 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } - // Handle probe replies, which may or may not have data + // Handle releases, which might be voluntary and might have data val release = io.client.release val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - val idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits - t.release.valid := release.valid && (idx === UFix(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := idx + release_data_dep_q.io.enq.bits.master_xact_id := release_idx release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor From 766d5622b17d432286a11c1a98ee46423ad55912 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 10 Apr 2013 13:46:31 -0700 Subject: [PATCH 175/688] Prevent messages from becoming interleaved in the BasicCrossbar. Remove dependency trackers from the uncore, use msg headers instead. Have one ReleaseHnadler per core for now. --- uncore/src/coherence.scala | 10 ++--- uncore/src/package.scala | 53 +++++++++++++++++++++++- uncore/src/tilelink.scala | 5 ++- uncore/src/uncore.scala | 82 +++++++++++++------------------------- 4 files changed, 86 insertions(+), 64 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 20157456..ec932ec9 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -265,7 +265,7 @@ class MICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) @@ -418,7 +418,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -574,7 +574,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -735,7 +735,7 @@ class MESICoherence extends CoherencePolicyWithUncached { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) @@ -912,7 +912,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsAckReply(a_type: UFix, global_state: UFix): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = Bool(true) + def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index f353a524..4133a9d3 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -24,7 +24,56 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { + val io = new ioArbiter(n)(data) + + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + val locked = Reg(resetVal = Bool(false)) + val lock_idx = Reg(resetVal = UFix(n)) + + if(count > 1){ + when(io.out.valid && io.out.ready) { + cnt := cnt_next + when(!locked) { + locked := Bool(true) + lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} + } + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + } else { + locked := Bool(false) + lock_idx := UFix(n) + cnt := UFix(0) + } + + + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) + val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) + (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(locked, lock_idx, choose) + when (io.out.valid && io.out.ready) { + last_grant := choose + } + + val dvec = Vec(n) { data } + (0 until n).map(i => dvec(i) := io.in(i).bits ) + + io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) + io.out.bits := dvec(choose) + io.chosen := choose +} + +class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} @@ -33,7 +82,7 @@ class BasicCrossbar[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfi val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new RRArbiter(conf.nEndpoints)){io.in(0).bits.clone} + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UFix(i)) arb.bits := in.bits diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16f8c9b..c7f82da2 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -118,11 +118,12 @@ class GrantAck extends ClientSourcedMessage with HasMasterTransactionId abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) +class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedIO){(new LogicalNetworkIO){new AcquireData }} + val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -131,7 +132,7 @@ class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bun class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedIO){(new LogicalNetworkIO){new ReleaseData }} + val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} override def clone = { new TileLinkIO().asInstanceOf[this.type] } } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3e54b9e3..445444db 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -3,18 +3,10 @@ package uncore import Chisel._ import Constants._ -class TrackerProbeData(implicit conf: UncoreConfiguration) extends Bundle { - val client_id = Bits(width = conf.ln.idBits) -} - -class TrackerAllocReq(implicit conf: UncoreConfiguration) extends Bundle { - val acquire = new Acquire() - val client_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerAllocReq().asInstanceOf[this.type] } -} - -class TrackerDependency extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { + val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) + val data_src_id = Bits(width = conf.ln.idBits) + override def clone = { new TrackerDependency().asInstanceOf[this.type] } } case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) @@ -31,18 +23,16 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = new WritebackTracker(0, bankId) +: (1 to NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) - val release_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth must >= NPRIMARY - val acquire_data_dep_q = (new Queue(NGLOBAL_XACTS)){new TrackerDependency} // depth should >= NPRIMARY + val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.client.acquire val acquire_data = io.client.acquire_data - val acq_dep_deq = acquire_data_dep_q.io.deq val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict || (!acquire_data_dep_q.io.enq.ready && co.messageHasData(acquire.bits.payload)) + val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { @@ -53,14 +43,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends t.acquire_data.bits := acquire_data.bits t.acquire_data.valid := acquire_data.valid - trackerList(i).io.acquire_data_dep.bits := acq_dep_deq.bits - trackerList(i).io.acquire_data_dep.valid := acq_dep_deq.valid } acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - acquire_data_dep_q.io.enq.valid := acquire.valid && acquire.ready && co.messageHasData(acquire.bits.payload) - acquire_data_dep_q.io.enq.bits.master_xact_id := OHToUFix(alloc_arb.io.in.map(_.ready)) - acq_dep_deq.ready := trackerList.map(_.io.acquire_data_dep.ready).reduce(_||_) alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation @@ -73,24 +58,19 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val release_data = io.client.release_data val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = (!release_data_dep_q.io.enq.ready && co.messageHasData(release.bits.payload)) + val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases t.release_data.bits := release_data.bits t.release_data.valid := release_data.valid - trackerList(i).io.release_data_dep.bits := release_data_dep_q.io.deq.bits - trackerList(i).io.release_data_dep.valid := release_data_dep_q.io.deq.valid } release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) - release_data_dep_q.io.enq.valid := release.valid && release.ready && co.messageHasData(release.bits.payload) - release_data_dep_q.io.enq.bits.master_xact_id := release_idx - release_data_dep_q.io.deq.ready := trackerList.map(_.io.release_data_dep.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -116,17 +96,13 @@ abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Compone val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) - val release_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val acquire_data_dep = (new FIFOIO) { new TrackerDependency }.flip - val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } @@ -136,8 +112,6 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) - io.acquire_data_dep.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) @@ -159,6 +133,7 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.data := UFix(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant_ack.valid := Bool(false) @@ -181,9 +156,8 @@ class WritebackTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigu io.master.acquire_data, io.client.release_data, release_data_needs_write, - mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + mem_cmd_sent, + init_client_id_) } . otherwise { state := s_ack } } is(s_ack) { @@ -198,6 +172,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) //TODO: Will need id reg for merged release xacts val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) @@ -245,10 +220,8 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.ready := Bool(false) io.client.acquire_data.ready := Bool(false) - io.acquire_data_dep.ready := Bool(false) io.client.release.ready := Bool(false) io.client.release_data.ready := Bool(false) - io.release_data_dep.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) @@ -285,7 +258,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(release_count === UFix(1)) { state := s_mem } - release_data_needs_write := release_data_needs_write || co.messageHasData(io.client.release.bits.payload) + when( co.messageHasData(io.client.release.bits.payload)) { + release_data_needs_write := Bool(true) + release_data_client_id := io.client.release.bits.header.src + } } } is(s_mem) { @@ -295,16 +271,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.release_data, release_data_needs_write, r_w_mem_cmd_sent, - io.release_data_dep.ready, - io.release_data_dep.valid && (io.release_data_dep.bits.master_xact_id === UFix(trackerId))) + release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.master.acquire_data, io.client.acquire_data, acquire_data_needs_write, a_w_mem_cmd_sent, - io.acquire_data_dep.ready, - io.acquire_data_dep.valid && (io.acquire_data_dep.bits.master_xact_id === UFix(trackerId))) + init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) } . otherwise { @@ -330,24 +304,22 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, pop_dep: Bool, at_front_of_dep_queue: Bool) { + def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.bits.payload := cmd_to_write master_acq_data.bits.payload := client_data.bits.payload when(master_acq.ready && master_acq.valid) { cmd_sent := Bool(true) } - when (at_front_of_dep_queue) { - master_acq.valid := !cmd_sent && master_acq_data.ready && client_data.valid + when (do_write) { + master_acq.valid := !cmd_sent && master_acq_data.ready when (master_acq.ready || cmd_sent) { master_acq_data.valid := client_data.valid when(master_acq_data.ready) { client_data.ready:= Bool(true) - when (client_data.valid) { - mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { - pop_dep := Bool(true) - trigger := Bool(false) - } + mem_cnt := mem_cnt_next + when(mem_cnt === UFix(REFILL_CYCLES-1)) { + trigger := Bool(false) } } } From 12d394811efccb7e1f83b3935695235b24be447d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Apr 2013 18:47:37 -0700 Subject: [PATCH 176/688] Allow release data to be written out even before all releases have been collected --- uncore/src/uncore.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 445444db..5dcb192f 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -263,6 +263,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura release_data_client_id := io.client.release.bits.header.src } } + when (release_data_needs_write) { + doOuterReqWrite(io.master.acquire, + io.master.acquire_data, + io.client.release_data, + release_data_needs_write, + r_w_mem_cmd_sent, + release_data_client_id) + } } is(s_mem) { when (release_data_needs_write) { From fedc2753e4c54476f34c4ff1b559c1b8baf40b46 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Apr 2013 11:00:23 -0700 Subject: [PATCH 177/688] make sure master_xact_id field is large enough for temporary extra release trackers --- uncore/src/consts.scala | 5 +++-- uncore/src/uncore.scala | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index cde8d3b9..9d343b3f 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -10,8 +10,9 @@ abstract trait CoherenceConfigConstants { } trait UncoreConstants { - val NGLOBAL_XACTS = 8 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_XACTS) + val NGLOBAL_ACQ_XACTS = 8 + val NGLOBAL_REL_XACTS = 4 + val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5dcb192f..f0571d89 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,8 +23,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - val trackerList = (0 until conf.ln.nClients).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.ln.nClients until conf.ln.nClients + NGLOBAL_XACTS).map(new AcquireTracker(_, bankId)) + require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) From 9a258e7fb467e5d3135fc7b372ef73cb9428ede5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 178/688] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From 9a3b2e7006c370b6c37bee57272a0d1788d50e5b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 1 May 2013 16:48:01 -0700 Subject: [PATCH 179/688] new paired meta/data IO type, and matching arbiter --- uncore/src/package.scala | 62 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 7e1fc7a0..46c39d51 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -8,6 +8,68 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = (new FIFOIO()){ m } + val data = (new FIFOIO()){ d } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { From 0672773c1a762db43ad66cf6e3e70f2bff6e061f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 9 May 2013 02:14:44 -0700 Subject: [PATCH 180/688] for now, don't use asserts outside of components --- uncore/src/coherence.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..42b3495e 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -842,7 +842,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), From 4c1f105ce94eb9dd3ae7902272f5301119278c15 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 May 2013 17:19:07 -0700 Subject: [PATCH 181/688] added PairedData link type with matching crossbar, ported tilelink and uncore to use --- uncore/src/coherence.scala | 6 +- uncore/src/package.scala | 50 +++++++++-- uncore/src/tilelink.scala | 93 ++++++++------------ uncore/src/uncore.scala | 168 ++++++++++++++++++------------------- 4 files changed, 158 insertions(+), 159 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index ec932ec9..6a1eaf80 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -3,9 +3,9 @@ package uncore import Chisel._ import Constants._ -trait CoherenceAgentRole -trait ClientCoherenceAgent extends CoherenceAgentRole -trait MasterCoherenceAgent extends CoherenceAgentRole +abstract trait CoherenceAgentRole +abstract trait ClientCoherenceAgent extends CoherenceAgentRole +abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { def isHit (cmd: Bits, state: UFix): Bool diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 46c39d51..754621ad 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -9,8 +9,9 @@ import scala.collection.mutable.Stack implicit def toOption[A](a: A) = Option(a) class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = (new FIFOIO()){ m } - val data = (new FIFOIO()){ d } + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { @@ -18,9 +19,10 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) @@ -28,11 +30,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) - val chosen_meta_has_data = io.out.meta.valid && needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true)) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && chosen_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && chosen_meta_has_data) + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) io.meta_chosen := meta_chosen @@ -70,6 +74,34 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -88,7 +120,7 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c7f82da2..589254f9 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -116,82 +116,55 @@ class Grant extends MasterSourcedMessage with HasMemData with HasClientTransacti class GrantAck extends ClientSourcedMessage with HasMasterTransactionId -abstract class DirectionalFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) -class ClientSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) -class ClientSourcedDataIO[T <: Data]()(data: => T) extends ClientSourcedIO()(data) -class MasterSourcedIO[T <: Data]()(data: => T) extends DirectionalFIFOIO()(data) {flip()} +trait DirectionalIO +trait ClientSourcedIO extends DirectionalIO +trait MasterSourcedIO extends DirectionalIO +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } +} +class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { + flip() + override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } +} class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val acquire = (new ClientSourcedIO){(new LogicalNetworkIO){new Acquire }} - val acquire_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new AcquireData }} - val grant = (new MasterSourcedIO) {(new LogicalNetworkIO){new Grant }} - val grant_ack = (new ClientSourcedIO){(new LogicalNetworkIO){new GrantAck }} + val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) + val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) + val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = (new MasterSourcedIO){(new LogicalNetworkIO){new Probe }} - val release = (new ClientSourcedIO){(new LogicalNetworkIO){new Release }} - val release_data = (new ClientSourcedDataIO){(new LogicalNetworkIO){new ReleaseData }} + val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) + val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -object UncachedTileLinkIOArbiterShim { - def apply[T <: HasClientTransactionId](in: ClientSourcedIO[LogicalNetworkIO[T]], id: Int, max: Int)(implicit lconf: LogicalNetworkConfiguration) = { - val shim = (new UncachedTileLinkIOArbiterShim(id, max)){in.bits.payload.clone} - shim.io.in <> in - shim.io.out - } -} -class UncachedTileLinkIOArbiterShim[T <: HasClientTransactionId](id: Int, max: Int)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new ClientSourcedIO){(new LogicalNetworkIO){ data }}.flip - val out = (new ClientSourcedIO){(new LogicalNetworkIO){ data }} - } - io.out.bits := io.in.bits - io.out.bits.payload.client_xact_id := Cat(io.in.bits.payload.client_xact_id, UFix(id, log2Up(max))) - io.out.valid := io.in.valid - io.in.ready := io.out.ready -} - - -class UncachedTileLinkIOArbiter(n: Int)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - when(io.out.acquire_data.valid && io.out.acquire_data.ready) { - mem_cnt := mem_cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.acquire_data.ready && in.acquire_data.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(mem_cnt_next === UFix(0)) { - locked := Bool(false) - } - } - - val acqd_grant = ArbiterCtrl(io.in.map(_.acquire_data.valid)) - (0 until n).map(i => io.in(i).acquire_data.ready := Mux(locked, UFix(i) === lock_idx, acqd_grant(i)) && io.out.acquire_data.ready) - var acqd_bits = io.in(n-1).acquire_data.bits - for (i <- n-2 to 0 by -1) { - acqd_bits = Mux(io.in(i).acquire_data.valid, io.in(i).acquire_data.bits, acqd_bits) - } - val locked_req = io.in(lock_idx).acquire_data - io.out.acquire_data.bits := Mux(locked, locked_req.bits, acqd_bits) - io.out.acquire_data.valid := Mux(locked, locked_req.valid, io.in.map(_.acquire_data.valid).reduce(_||_)) - - val acq_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new Acquire} } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.map{ case(acq, id) => UncachedTileLinkIOArbiterShim(acq, id, n) }.zip(acq_arb.io.in).map{ case (req, arb) => req <> arb} + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + req.meta.ready := arb.meta.ready + }} - val grant_ack_arb = (new Arbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index f0571d89..3a3c36ae 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -31,23 +31,22 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle acquire transaction initiation val acquire = io.client.acquire - val acquire_data = io.client.acquire_data val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + alloc_arb.io.in(i).valid := t.acquire.meta.ready + t.acquire.meta.bits := acquire.meta.bits + t.acquire.meta.valid := alloc_arb.io.in(i).ready - t.acquire_data.bits := acquire_data.bits - t.acquire_data.valid := acquire_data.valid + t.acquire.data.bits := acquire.data.bits + t.acquire.data.valid := acquire.data.valid } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires - acquire_data.ready := trackerList.map(_.io.client.acquire_data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires + acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) + alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} @@ -56,22 +55,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends // Handle releases, which might be voluntary and might have data val release = io.client.release - val release_data = io.client.release_data - val voluntary = co.isVoluntary(release.bits.payload) + val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.bits.header.src, release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UFix(i)) && !block_releases - t.release_data.bits := release_data.bits - t.release_data.valid := release_data.valid + t.release.meta.bits := release.meta.bits + t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.data.bits := release.data.bits + t.release.data.valid := release.data.valid } - release.ready := Vec(trackerList.map(_.io.client.release.ready)){Bool()}(release_idx) && !block_releases - release_data.ready := trackerList.map(_.io.client.release_data.ready).reduce(_||_) + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} @@ -85,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size) + val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } @@ -114,22 +112,22 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := cmd_to_write + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header.src := UFix(bankId) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) + io.master.acquire.data.bits.header.src := UFix(bankId) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) // DNC + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id @@ -141,11 +139,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { - xact := io.client.release.bits.payload - init_client_id_ := io.client.release.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.bits.payload) + io.client.release.meta.ready := Bool(true) + when( io.client.release.meta.valid ) { + xact := io.client.release.meta.bits.payload + init_client_id_ := io.client.release.meta.bits.header.src + release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) mem_cnt := UFix(0) mem_cmd_sent := Bool(false) state := s_mem @@ -153,10 +151,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, mem_cmd_sent, init_client_id_) } . otherwise { state := s_ack } @@ -192,20 +189,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura if (conf.ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.bits.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.master.acquire.meta.valid := Bool(false) + io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst - io.master.acquire.bits.header.src := UFix(bankId) - io.master.acquire_data.valid := Bool(false) - io.master.acquire_data.bits.payload.data := UFix(0) + io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.data.valid := Bool(false) + io.master.acquire.data.bits.payload.data := UFix(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire_data.bits.header := UFix(bankId) + io.master.acquire.data.bits.header := UFix(bankId) io.client.probe.valid := Bool(false) io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) io.client.probe.bits.payload.master_xact_id := UFix(trackerId) @@ -219,22 +216,22 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UFix(bankId) io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.ready := Bool(false) - io.client.acquire_data.ready := Bool(false) - io.client.release.ready := Bool(false) - io.client.release_data.ready := Bool(false) + io.client.acquire.meta.ready := Bool(false) + io.client.acquire.data.ready := Bool(false) + io.client.release.meta.ready := Bool(false) + io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) - when( io.client.acquire.valid ) { - xact := io.client.acquire.bits.payload - init_client_id_ := io.client.acquire.bits.header.src + io.client.acquire.meta.ready := Bool(true) + when( io.client.acquire.meta.valid ) { + xact := io.client.acquire.meta.bits.payload + init_client_id_ := io.client.acquire.meta.bits.header.src init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.bits.payload.a_type, UFix(0)) + acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) @@ -253,21 +250,20 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when(io.client.probe.ready) { probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) } - io.client.release.ready := Bool(true) - when(io.client.release.valid) { + io.client.release.meta.ready := Bool(true) + when(io.client.release.meta.valid) { if(conf.ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } - when( co.messageHasData(io.client.release.bits.payload)) { + when( co.messageHasData(io.client.release.meta.bits.payload)) { release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.bits.header.src + release_data_client_id := io.client.release.meta.bits.header.src } } when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, + doOuterReqWrite(io.master.acquire, + io.client.release.data, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -275,18 +271,16 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } is(s_mem) { when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.release_data, - release_data_needs_write, - r_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.release.data, + release_data_needs_write, + r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.master.acquire_data, - io.client.acquire_data, - acquire_data_needs_write, - a_w_mem_cmd_sent, + doOuterReqWrite(io.master.acquire, + io.client.acquire.data, + acquire_data_needs_write, + a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, x_needs_read) @@ -313,18 +307,18 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: Data](master_acq: FIFOIO[LogicalNetworkIO[Acquire]], master_acq_data: FIFOIO[LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.bits.payload := cmd_to_write - master_acq_data.bits.payload := client_data.bits.payload - when(master_acq.ready && master_acq.valid) { + master_acq.meta.bits.payload := cmd_to_write + master_acq.data.bits.payload := client_data.bits.payload + when(master_acq.meta.fire()) { cmd_sent := Bool(true) } when (do_write) { - master_acq.valid := !cmd_sent && master_acq_data.ready - when (master_acq.ready || cmd_sent) { - master_acq_data.valid := client_data.valid - when(master_acq_data.ready) { + master_acq.meta.valid := !cmd_sent + when (master_acq.meta.ready || cmd_sent) { + master_acq.data.valid := client_data.valid + when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next when(mem_cnt === UFix(REFILL_CYCLES-1)) { @@ -335,10 +329,10 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: FIFOIO[LogicalNetworkIO[Acquire]], trigger: Bool) { - master_acq.valid := Bool(true) - master_acq.bits.payload := cmd_to_read - when(master_acq.ready) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + master_acq.meta.valid := Bool(true) + master_acq.meta.bits.payload := cmd_to_read + when(master_acq.meta.ready) { trigger := Bool(false) } } From cf02f1ef014ab7f644f1657ecd97f37152cda36f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Apr 2013 16:48:55 -0700 Subject: [PATCH 182/688] use new locking round robin arbiter --- uncore/src/package.scala | 51 ++-------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 4133a9d3..7e1fc7a0 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -6,6 +6,8 @@ import scala.collection.mutable.Stack //TODO: Remove these Networking classes from the package object once Scala bug //SI-3439 is resolved. +implicit def toOption[A](a: A) = Option(a) + case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -24,55 +26,6 @@ class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkCon abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component -class LockingRRArbiter[T <: Data](n: Int, count: Int)(data: => T) extends Component { - val io = new ioArbiter(n)(data) - - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - val locked = Reg(resetVal = Bool(false)) - val lock_idx = Reg(resetVal = UFix(n)) - - if(count > 1){ - when(io.out.valid && io.out.ready) { - cnt := cnt_next - when(!locked) { - locked := Bool(true) - lock_idx := Vec(io.in.map{ in => in.ready && in.valid}){Bool()}.indexWhere{i: Bool => i} - } - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - } else { - locked := Bool(false) - lock_idx := UFix(n) - cnt := UFix(0) - } - - - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val g = ArbiterCtrl((0 until n).map(i => io.in(i).valid && UFix(i) > last_grant) ++ io.in.map(_.valid)) - val grant = (0 until n).map(i => g(i) && UFix(i) > last_grant || g(i+n)) - (0 until n).map(i => io.in(i).ready := Mux(locked, lock_idx === UFix(i), grant(i)) && io.out.ready) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).valid && UFix(i) > last_grant, Bits(i), choose) - choose = Mux(locked, lock_idx, choose) - when (io.out.valid && io.out.ready) { - last_grant := choose - } - - val dvec = Vec(n) { data } - (0 until n).map(i => dvec(i) := io.in(i).bits ) - - io.out.valid := Mux(locked, io.in(lock_idx).valid, io.in.map(_.valid).foldLeft(Bool(false))( _ || _)) - io.out.bits := dvec(choose) - io.chosen := choose -} - class BasicCrossbar[T <: Data](count: Int)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip From db8e5fda9b908e908f78110012836dbab5b80c72 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 9 Jul 2013 15:37:42 -0700 Subject: [PATCH 183/688] new tilelink arbiter types, reduced release xact trackers --- uncore/src/consts.scala | 2 +- uncore/src/tilelink.scala | 96 ++++++++++++++++++++++++++++++++++++--- uncore/src/uncore.scala | 6 +-- 3 files changed, 94 insertions(+), 10 deletions(-) diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 9d343b3f..aaa7b7a8 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -11,7 +11,7 @@ abstract trait CoherenceConfigConstants { trait UncoreConstants { val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 4 + val NGLOBAL_REL_XACTS = 1 val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 589254f9..2e595c34 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -147,12 +147,24 @@ class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTil override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +/* + * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int): Bits + def grantClientXactId(in: Grant): Bits + def arbIdx(in: Grant): UFix +} +*/ + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + val io = new Bundle { val in = Vec(n) { new UncachedTileLinkIO }.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) io.out.acquire <> acq_arb.io.out @@ -160,7 +172,7 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi arb.data <> req.data arb.meta.valid := req.meta.valid arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := Cat(req.meta.bits.payload.client_xact_id, UFix(id, log2Up(n))) + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) req.meta.ready := arb.meta.ready }} @@ -170,13 +182,85 @@ class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: Logi io.out.grant.ready := Bool(false) for (i <- 0 until n) { - val tag = io.out.grant.bits.payload.client_xact_id io.in(i).grant.valid := Bool(false) - when (tag(log2Up(n)-1,0) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := tag >> UFix(log2Up(n)) + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) } } + +class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id + def grantClientXactId(in: Grant) = in.client_xact_id + def arbIdx(in: Grant): UFix = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { + def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) + def grantClientXactId(in: Grant) = UFix(0) // DNC + def arbIdx(in: Grant) = in.client_xact_id + + val io = new Bundle { + val in = Vec(n) { new UncachedTileLinkIO }.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 3a3c36ae..9d0de31b 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -23,7 +23,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends { implicit val lnConf = conf.ln val co = conf.co - require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config + //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) @@ -60,7 +60,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, release.meta.bits.header.src, release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits @@ -83,7 +83,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiter(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } From d8440b042a4d74c3f5f437ab73b328c5ff1ad94c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Jul 2013 23:22:36 -0700 Subject: [PATCH 184/688] Make compatible with scala 2.10. Refactor constants into package object. Remove networking primitives from package object. Clean up request generators. Chnage ++ to +: for appending to io.incoherent. --- uncore/src/coherence.scala | 2 - uncore/src/consts.scala | 2 + uncore/src/llc.scala | 3 - uncore/src/memserdes.scala | 3 - uncore/src/network.scala | 193 ++++++++++++++++++++++++++++++++++ uncore/src/package.scala | 210 ++----------------------------------- uncore/src/slowio.scala | 2 - uncore/src/temp.scala | 13 --- uncore/src/tilelink.scala | 2 - uncore/src/uncore.scala | 19 ++-- 10 files changed, 211 insertions(+), 238 deletions(-) create mode 100644 uncore/src/network.scala delete mode 100644 uncore/src/temp.scala diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index 03c5c8f1..b869436a 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ abstract trait CoherenceAgentRole abstract trait ClientCoherenceAgent extends CoherenceAgentRole diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index aaa7b7a8..4e3c814e 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -36,6 +36,7 @@ trait TileLinkSizeConstants extends val ACQUIRE_ATOMIC_OP_BITS = 4 } +object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); val MT_B = Bits("b000", 3); @@ -82,6 +83,7 @@ trait MemoryInterfaceConstants extends val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS } +object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 val VADDR_BITS = 43; diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 1f9f4b2a..6b0c77f2 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 88a6f1b3..c7c8e773 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -1,8 +1,5 @@ package uncore - import Chisel._ -import Node._ -import Constants._ import scala.math._ class ioMemSerialized(w: Int) extends Bundle diff --git a/uncore/src/network.scala b/uncore/src/network.scala new file mode 100644 index 00000000..cdaa8118 --- /dev/null +++ b/uncore/src/network.scala @@ -0,0 +1,193 @@ +package uncore +import Chisel._ +import scala.collection.mutable.Stack + +class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { + val meta = new FIFOIO()(m) + val data = new FIFOIO()(d) + override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +} + +class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { + val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val out = new PairedDataIO()(m,d) + val meta_chosen = Bits(OUTPUT, log2Up(n)) + val data_chosen = Bits(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +} + +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { + require(isPow2(count)) + val io = new PairedArbiterIO(n)(meta,data) + val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val grant = List.fill(n)(Bool()) + val meta_chosen = Bits(width = log2Up(n)) + + val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) + val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data + val grant_chosen_meta = !(locked && chosen_meta_has_data) + (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta + io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) + io.out.meta.bits := io.in(meta_chosen).meta.bits + io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) + io.meta_chosen := meta_chosen + io.data_chosen := Mux(locked, lockIdx, meta_chosen) + + if(count > 1){ + val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) + val cnt_next = cnt + UFix(1) + when(io.out.data.fire()){ + cnt := cnt_next + when(cnt_next === UFix(0)) { + locked := Bool(false) + } + } + when(io.out.meta.fire()) { + when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { + when(!locked) { + locked := Bool(true) + lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + } + } + } + } + val last_grant = Reg(resetVal = Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + + var choose = Bits(n-1) + for (i <- n-2 to 0 by -1) + choose = Mux(io.in(i).meta.valid, Bits(i), choose) + for (i <- n-1 to 1 by -1) + choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + meta_chosen := choose + + when (io.out.meta.fire()) { last_grant := meta_chosen } +} + +class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + } + + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = metaRdyVecs zip dataRdyVecs + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.bits := in.meta.bits + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + arb.data.bits := in.data.bits + data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) + io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) + } +} + +case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) + +class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = (new PhysicalHeader) + val payload = data + override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +} + +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component + +class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new Bundle { + val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip + val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + } + + val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.bits := in.bits + rdy := arb.ready && (in.bits.header.dst === UFix(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until conf.nEndpoints) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) + } +} + +case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) + +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { + override val io: Vec[TileLinkType] + val physicalNetworks: Seq[PhysicalNetwork] + require(endpoints.length == conf.nEndpoints) +} + +class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val src = UFix(width = conf.idBits) + val dst = UFix(width = conf.idBits) +} + +object FIFOedLogicalNetworkIOWrapper { + def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){ data }.flip + val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + } + io.out.valid := io.in.valid + io.out.bits.payload := io.in.bits + io.out.bits.header.dst := dst + io.out.bits.header.src := src + io.in.ready := io.out.ready +} + +object FIFOedLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + shim.io.in.valid := in.valid + shim.io.in.bits := in.bits + in.ready := shim.io.in.ready + shim.io.out + } +} +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { + val io = new Bundle { + val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip + val out = (new FIFOIO){ data } + } + io.out.valid := io.in.valid + io.out.bits := io.in.bits.payload + io.in.ready := io.out.ready +} + +class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = data + override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } +} diff --git a/uncore/src/package.scala b/uncore/src/package.scala index 754621ad..a92f739c 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,204 +1,8 @@ -package object uncore { -import Chisel._ -import Node._ -import scala.collection.mutable.Stack - -//TODO: Remove these Networking classes from the package object once Scala bug -//SI-3439 is resolved. - -implicit def toOption[A](a: A) = Option(a) - -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { - require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UFix(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[BasicCrossbarIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new BasicCrossbarIO()(meta),new BasicCrossbarIO()(data))} - } - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} - -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) - -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -abstract class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data -} - -class BasicCrossbarIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetworkIO()(data)(conf) { - override def clone = { new BasicCrossbarIO()(data).asInstanceOf[this.type] } -} - -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component - -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new BasicCrossbarIO){data}}} - } - - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} - (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) - arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) - } -} - -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) -} - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) -} - -object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} - -object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out - } -} -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { - val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} - -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } -} - +package object uncore extends + uncore.constants.MemoryOpConstants with + uncore.constants.MemoryInterfaceConstants with + uncore.constants.CacheConstants with + uncore.constants.AddressConstants +{ + implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index cf00ae6e..b56b1396 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component { diff --git a/uncore/src/temp.scala b/uncore/src/temp.scala deleted file mode 100644 index b2d3fdca..00000000 --- a/uncore/src/temp.scala +++ /dev/null @@ -1,13 +0,0 @@ -package uncore -import _root_.uncore.constants._ - -//TODO: When compiler bug SI-5604 is fixed in 2.10, remove object Constants and -// mixin Constants traits to package object uncore in package.scala and -// remove import Constants._'s from other .scala files -object Constants extends - MemoryOpConstants with - MemoryInterfaceConstants with - CacheConstants with - AddressConstants -{ -} diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 2e595c34..3621e73b 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ trait HasPhysicalAddress extends Bundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 9d0de31b..5f87f724 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,7 +1,5 @@ package uncore - import Chisel._ -import Constants._ class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) @@ -109,7 +107,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -153,6 +150,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, mem_cmd_sent, init_client_id_) @@ -264,6 +262,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) @@ -273,17 +272,19 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when (release_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.release.data, + cmd_to_write, release_data_needs_write, r_w_mem_cmd_sent, release_data_client_id) } . elsewhen(acquire_data_needs_write) { doOuterReqWrite(io.master.acquire, io.client.acquire.data, + cmd_to_write, acquire_data_needs_write, a_w_mem_cmd_sent, init_client_id_) } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, x_needs_read) + doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) @@ -302,14 +303,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } abstract trait OuterRequestGenerator { - val cmd_to_write: Acquire - val cmd_to_read: Acquire val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd_to_write + master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload when(master_acq.meta.fire()) { cmd_sent := Bool(true) @@ -329,9 +328,9 @@ abstract trait OuterRequestGenerator { } } - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], trigger: Bool) { + def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd_to_read + master_acq.meta.bits.payload := cmd when(master_acq.meta.ready) { trigger := Bool(false) } From bc2b45da129883748d197abab5fd50ec3a885a9a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Aug 2013 14:55:06 -0700 Subject: [PATCH 185/688] Fold uncore constants into TileLinkConfiguration, update coherence API --- uncore/src/coherence.scala | 229 +++++++++++++++++++++---------------- uncore/src/consts.scala | 61 ++++------ uncore/src/llc.scala | 12 +- uncore/src/memserdes.scala | 27 +++++ uncore/src/package.scala | 5 +- uncore/src/tilelink.scala | 122 ++++++++++---------- uncore/src/uncore.scala | 75 ++++++------ 7 files changed, 283 insertions(+), 248 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b869436a..b8c3e4ad 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -6,6 +6,19 @@ abstract trait ClientCoherenceAgent extends CoherenceAgentRole abstract trait MasterCoherenceAgent extends CoherenceAgentRole abstract class CoherencePolicy { + def nClientStates: Int + def nMasterStates: Int + def nAcquireTypes: Int + def nProbeTypes: Int + def nReleaseTypes: Int + def nGrantTypes: Int + def clientStateBits = log2Up(nClientStates) + def masterStateBits = log2Up(nMasterStates) + def acquireTypeBits = log2Up(nAcquireTypes) + def probeTypeBits = log2Up(nProbeTypes) + def releaseTypeBits = log2Up(nReleaseTypes) + def grantTypeBits = log2Up(nGrantTypes) + def isHit (cmd: Bits, state: UFix): Bool def isValid (state: UFix): Bool @@ -22,11 +35,12 @@ abstract class CoherencePolicy { def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix + def getProbeType(a_type: UFix, global_state: UFix): UFix def getReleaseTypeOnCacheControl(cmd: Bits): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix): Release - def newRelease (incoming: Probe, state: UFix, id: UFix): Release + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits + def getGrantType(a_type: UFix, count: UFix): Bits + def getGrantType(rel: Release, count: UFix): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -35,9 +49,6 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits - def getProbeType(a_type: UFix, global_state: UFix): UFix def needsOuterRead(a_type: UFix, global_state: UFix): Bool def needsOuterWrite(a_type: UFix, global_state: UFix): Bool def needsAckReply(a_type: UFix, global_state: UFix): Bool @@ -50,11 +61,11 @@ abstract class CoherencePolicy { } trait UncachedTransactions { - def getUncachedReadAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteAcquire(addr: UFix, id: UFix): Acquire - def getUncachedReadWordAcquire(addr: UFix, id: UFix): Acquire - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits): Acquire - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix): Acquire + def getUncachedReadAcquireType: Bits + def getUncachedWriteAcquireType: Bits + def getUncachedReadWordAcquireType: Bits + def getUncachedWriteWordAcquireType: Bits + def getUncachedAtomicAcquireType: Bits def isUncachedReadTransaction(acq: Acquire): Bool } @@ -63,7 +74,7 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = Release( UFix(0), UFix(0), UFix(0), UFix(0)) + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) def getGrantType(rel: Release, count: UFix): Bits = Bits(0) @@ -79,10 +90,16 @@ abstract class IncoherentPolicy extends CoherencePolicy { } class ThreeStateIncoherence extends IncoherentPolicy { - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(3){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(3){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(2){ UFix() } + def nClientStates = 3 + def nMasterStates = 0 + def nAcquireTypes = 3 + def nProbeTypes = 0 + def nReleaseTypes = 2 + def nGrantTypes = 3 + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -109,7 +126,6 @@ class ThreeStateIncoherence extends IncoherentPolicy { )) } - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -133,14 +149,21 @@ class ThreeStateIncoherence extends IncoherentPolicy { } class MICoherence extends CoherencePolicyWithUncached { + def nClientStates = 2 + def nMasterStates = 2 + def nAcquireTypes = 6 + def nProbeTypes = 2 + def nReleaseTypes = 5 + def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(2){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(6){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(7){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(2){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(5){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) @@ -186,12 +209,11 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -200,8 +222,7 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -210,7 +231,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -270,14 +291,20 @@ class MICoherence extends CoherencePolicyWithUncached { } class MEICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 2 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(2){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(8){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -331,12 +358,11 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -349,8 +375,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -361,7 +386,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -424,14 +449,21 @@ class MEICoherence extends CoherencePolicyWithUncached { } class MSICoherence extends CoherencePolicyWithUncached { + def nClientStates = 3 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(3){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) @@ -490,12 +522,11 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -508,8 +539,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -520,7 +550,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -580,14 +610,20 @@ class MSICoherence extends CoherencePolicyWithUncached { } class MESICoherence extends CoherencePolicyWithUncached { + def nClientStates = 4 + def nMasterStates = 3 + def nAcquireTypes = 7 + def nProbeTypes = 3 + def nReleaseTypes = 7 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(4){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(3){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(7){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(9){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(3){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(7){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -647,12 +683,11 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -665,8 +700,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -677,7 +711,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -741,13 +775,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } class MigratoryCoherence extends CoherencePolicyWithUncached { + def nClientStates = 7 + def nMasterStates = 0 + def nAcquireTypes = 8 + def nProbeTypes = 4 + def nReleaseTypes = 11 + def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(7){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(8){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(10){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(4){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(11){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -820,12 +860,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def getUncachedReadAcquire(addr: UFix, id: UFix) = Acquire(acquireReadUncached, addr, id) - def getUncachedWriteAcquire(addr: UFix, id: UFix) = Acquire(acquireWriteUncached, addr, id) - def getUncachedReadWordAcquire(addr: UFix, id: UFix) = Acquire(acquireReadWordUncached, addr, id) - def getUncachedWriteWordAcquire(addr: UFix, id: UFix, write_mask: Bits) = Acquire(acquireWriteWordUncached, addr, id, write_mask) - def getUncachedAtomicAcquire(addr: UFix, id: UFix, subword_addr: UFix, atomic_op: UFix) = Acquire(acquireAtomicUncached, addr, id, subword_addr, atomic_op) - def getVoluntaryWriteback(addr: UFix, client_id: UFix, master_id: UFix) = Release(releaseVoluntaryInvalidateData, addr, client_id, master_id) + def getUncachedReadAcquireType = acquireReadUncached + def getUncachedWriteAcquireType = acquireWriteUncached + def getUncachedReadWordAcquireType = acquireReadWordUncached + def getUncachedWriteWordAcquireType = acquireWriteWordUncached + def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -838,22 +877,20 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - - def newRelease (incoming: Probe, state: UFix, id: UFix): Release = { - //Assert( incoming.p_type === probeInvalidateOthers && needsWriteback(state), "Bad probe request type, should be impossible.") + def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData + probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Release( Mux(needsWriteback(state), with_data, without_data), incoming.addr, id, incoming.master_xact_id) + Mux(needsWriteback(state), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { diff --git a/uncore/src/consts.scala b/uncore/src/consts.scala index 4e3c814e..15bfee4d 100644 --- a/uncore/src/consts.scala +++ b/uncore/src/consts.scala @@ -4,38 +4,6 @@ package constants import Chisel._ import scala.math.max -abstract trait CoherenceConfigConstants { - val ENABLE_SHARING: Boolean - val ENABLE_CLEAN_EXCLUSIVE: Boolean -} - -trait UncoreConstants { - val NGLOBAL_ACQ_XACTS = 8 - val NGLOBAL_REL_XACTS = 1 - val MASTER_XACT_ID_MAX_BITS = log2Up(NGLOBAL_ACQ_XACTS+NGLOBAL_REL_XACTS) - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 -} - -trait CacheConstants extends UncoreConstants { - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkTypeConstants { - val ACQUIRE_TYPE_MAX_BITS = 2 - val GRANT_TYPE_MAX_BITS = 3 - val PROBE_TYPE_MAX_BITS = 2 - val RELEASE_TYPE_MAX_BITS = 3 -} - -trait TileLinkSizeConstants extends - TileLinkTypeConstants -{ - val CLIENT_XACT_ID_MAX_BITS = 10 - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_X = Bits("b???", 3); @@ -74,15 +42,6 @@ trait MemoryOpConstants { def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } -trait MemoryInterfaceConstants extends - UncoreConstants with - TileLinkSizeConstants -{ - val MEM_TAG_BITS = max(CLIENT_XACT_ID_MAX_BITS, MASTER_XACT_ID_MAX_BITS) - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS -} - object AddressConstants extends AddressConstants trait AddressConstants { val PADDR_BITS = 32 @@ -94,3 +53,23 @@ trait AddressConstants { val PERM_BITS = 6; } +trait CacheConstants { + val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 + val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) +} + +trait TileLinkSizeConstants { + val ACQUIRE_WRITE_MASK_BITS = 6 + val ACQUIRE_SUBWORD_ADDR_BITS = 3 + val ACQUIRE_ATOMIC_OP_BITS = 4 +} + +trait MemoryInterfaceConstants extends + CacheConstants with + AddressConstants +{ + val MEM_TAG_BITS = 5 + val MEM_DATA_BITS = 128 + val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS + val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS +} diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 6b0c77f2..cb6465c5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -86,10 +86,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val cpu = (new FIFOIO) { new MemReqCmd }.flip val repl_way = UFix(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) val data = (new FIFOIO) { new LLCDataReq(ways) } val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UFix(width = MEM_ADDR_BITS) val way = UFix(width = log2Up(ways)) } } val mem = new ioMemPipe @@ -105,7 +105,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillCount = UFix(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = PADDR_BITS - OFFSET_BITS - log2Up(sets)) + val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -184,7 +184,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component class LLCWriteback(requestors: Int) extends Component { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) }.flip } + val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } val mem = new ioMemPipe } @@ -235,7 +235,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo val io = new Bundle { val req = (new FIFOIO) { new LLCDataReq(ways) }.flip val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = PADDR_BITS - OFFSET_BITS) } + val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } val writeback_data = (new FIFOIO) { new MemData } val resp = (new FIFOIO) { new MemResp } val mem_resp = (new PipeIO) { new MemResp }.flip @@ -348,7 +348,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val mem = new ioMemPipe } - val tagWidth = PADDR_BITS - OFFSET_BITS - log2Up(sets) + val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = (new Arbiter(2)) { new MemReqCmd } diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index c7c8e773..b1272b28 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,6 +2,33 @@ package uncore import Chisel._ import scala.math._ +class MemData extends Bundle { + val data = Bits(width = MEM_DATA_BITS) +} + +class MemReqCmd extends Bundle { + val addr = UFix(width = MEM_ADDR_BITS) + val rw = Bool() + val tag = Bits(width = MEM_TAG_BITS) +} + +class MemResp extends Bundle { + val tag = Bits(width = MEM_TAG_BITS) + val data = Bits(width = MEM_DATA_BITS) +} + +class ioMem extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new FIFOIO) { new MemResp() }.flip +} + +class ioMemPipe extends Bundle { + val req_cmd = (new FIFOIO) { new MemReqCmd() } + val req_data = (new FIFOIO) { new MemData() } + val resp = (new PipeIO) { new MemResp() }.flip +} + class ioMemSerialized(w: Int) extends Bundle { val req = (new FIFOIO) { Bits(width = w) } diff --git a/uncore/src/package.scala b/uncore/src/package.scala index a92f739c..4ac83d0b 100644 --- a/uncore/src/package.scala +++ b/uncore/src/package.scala @@ -1,8 +1,7 @@ package object uncore extends uncore.constants.MemoryOpConstants with - uncore.constants.MemoryInterfaceConstants with - uncore.constants.CacheConstants with - uncore.constants.AddressConstants + uncore.constants.TileLinkSizeConstants with + uncore.constants.MemoryInterfaceConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 3621e73b..6f7e6175 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -1,99 +1,86 @@ package uncore import Chisel._ -trait HasPhysicalAddress extends Bundle { +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) + +abstract trait TileLinkSubBundle extends Bundle { + implicit val conf: TileLinkConfiguration +} + +trait HasPhysicalAddress extends TileLinkSubBundle { val addr = UFix(width = PADDR_BITS - OFFSET_BITS) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = CLIENT_XACT_ID_MAX_BITS) +trait HasClientTransactionId extends TileLinkSubBundle { + val client_xact_id = Bits(width = conf.clientXactIdBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = MASTER_XACT_ID_MAX_BITS) +trait HasMasterTransactionId extends TileLinkSubBundle { + val master_xact_id = Bits(width = conf.masterXactIdBits) } -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) -} - -class MemData extends Bundle with HasMemData - -class MemReqCmd extends Bundle with HasPhysicalAddress { - val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) -} - -class MemResp extends Bundle with HasMemData { - val tag = Bits(width = MEM_TAG_BITS) -} - -class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip -} - -class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip +trait HasTileLinkData extends TileLinkSubBundle { + val data = Bits(width = conf.dataBits) } trait SourcedMessage extends Bundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = ACQUIRE_TYPE_MAX_BITS) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) -} - object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.write_mask := Bits(0) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.write_mask := write_mask - acq.subword_addr := Bits(0, width = ACQUIRE_SUBWORD_ADDR_BITS) - acq.atomic_opcode := Bits(0, width = ACQUIRE_ATOMIC_OP_BITS) + acq.subword_addr := Bits(0) + acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix) = { + def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0, width = ACQUIRE_WRITE_MASK_BITS) + acq.write_mask := Bits(0) acq } } +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + val a_type = Bits(width = conf.co.acquireTypeBits) + val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) + val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) + val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + override def clone = { (new Acquire).asInstanceOf[this.type] } +} -class AcquireData extends ClientSourcedMessage with HasMemData -class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = PROBE_TYPE_MAX_BITS) +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new AcquireData).asInstanceOf[this.type] } +} + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + val p_type = Bits(width = conf.co.probeTypeBits) + override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix) = { + def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -102,17 +89,24 @@ object Release rel } } -class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = RELEASE_TYPE_MAX_BITS) +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + val r_type = Bits(width = conf.co.releaseTypeBits) + override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData extends ClientSourcedMessage with HasMemData - -class Grant extends MasterSourcedMessage with HasMemData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = GRANT_TYPE_MAX_BITS) +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { + override def clone = { (new ReleaseData).asInstanceOf[this.type] } +} + +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { + val g_type = Bits(width = conf.co.grantTypeBits) + override def clone = { (new Grant).asInstanceOf[this.type] } +} + +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { + override def clone = { (new GrantAck).asInstanceOf[this.type] } } -class GrantAck extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO @@ -132,14 +126,15 @@ class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) exten override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class UncachedTileLinkIO(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { + implicit val ln = conf.ln val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: LogicalNetworkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } @@ -154,7 +149,8 @@ abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit c } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix @@ -190,7 +186,8 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int, co: CoherencePolicy) } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UFix = in.client_xact_id @@ -226,7 +223,8 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int, co: CoherencePolicy)(implici } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { + implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) def grantClientXactId(in: Grant) = UFix(0) // DNC def arbIdx(in: Grant) = in.client_xact_id diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 5f87f724..60e100e0 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,30 +1,26 @@ package uncore import Chisel._ -class TrackerDependency(implicit conf: UncoreConfiguration) extends Bundle { - val tracker_id = Bits(width = MASTER_XACT_ID_MAX_BITS) - val data_src_id = Bits(width = conf.ln.idBits) - override def clone = { new TrackerDependency().asInstanceOf[this.type] } -} - -case class UncoreConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration) - -abstract class CoherenceAgent(implicit conf: LogicalNetworkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.nClients) { Bool() }.asInput + val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput } } -class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends CoherenceAgent()(conf.ln) +case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) + +class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) { - implicit val lnConf = conf.ln - val co = conf.co - //require(conf.ln.nClients < NGLOBAL_REL_XACTS) //TODO: handle in config - val trackerList = (0 until NGLOBAL_REL_XACTS).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (NGLOBAL_REL_XACTS until NGLOBAL_REL_XACTS + NGLOBAL_ACQ_XACTS).map(new AcquireTracker(_, bankId)) + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ + (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation @@ -81,32 +77,31 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: UncoreConfiguration) extends ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size, conf.co) + val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: UncoreConfiguration) extends Component with OuterRequestGenerator { - val co = conf.co - implicit val ln = conf.ln +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, conf.ln.nClients) + val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val release_data_needs_write = Reg(resetVal = Bool(false)) val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -163,38 +158,38 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Uncore } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } val state = Reg(resetVal = s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) + val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (conf.ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(conf.ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = conf.ln.nClients)) + val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) val x_needs_read = Reg(resetVal = Bool(false)) val acquire_data_needs_write = Reg(resetVal = Bool(false)) val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = co.getUncachedWriteAcquire(xact.addr, UFix(trackerId)) - val cmd_to_read = co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val probe_initial_flags = Bits(width = conf.ln.nClients) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) - if (conf.ln.nClients > 1) { + if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(conf.ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := co.getUncachedReadAcquire(xact.addr, UFix(trackerId)) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) //TODO io.master.acquire.bits.header.dst io.master.acquire.meta.bits.header.src := UFix(bankId) io.master.acquire.data.valid := Bool(false) @@ -227,14 +222,14 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(conf.ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) probe_flags := probe_initial_flags mem_cnt := UFix(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) - if(conf.ln.nClients > 1) { + if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, s_mem) } else state := s_mem @@ -250,7 +245,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: UncoreConfigura } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(conf.ln.nClients > 1) release_count := release_count - UFix(1) + if(ln.nClients > 1) release_count := release_count - UFix(1) when(release_count === UFix(1)) { state := s_mem } @@ -306,7 +301,7 @@ abstract trait OuterRequestGenerator { val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UFix(1) - def doOuterReqWrite[T <: HasMemData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload From 5c7a1f5cd6b4331807e5899f30e1a9348cc8d428 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 10:36:44 -0700 Subject: [PATCH 186/688] initial attempt at upgrade --- uncore/src/coherence.scala | 426 ++++++++++++++++++------------------- uncore/src/llc.scala | 234 ++++++++++---------- uncore/src/memserdes.scala | 129 +++++------ uncore/src/network.scala | 92 ++++---- uncore/src/slowio.scala | 61 +++--- uncore/src/tilelink.scala | 69 +++--- uncore/src/uncore.scala | 134 ++++++------ 7 files changed, 574 insertions(+), 571 deletions(-) diff --git a/uncore/src/coherence.scala b/uncore/src/coherence.scala index b8c3e4ad..7109be94 100644 --- a/uncore/src/coherence.scala +++ b/uncore/src/coherence.scala @@ -12,35 +12,35 @@ abstract class CoherencePolicy { def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int - def clientStateBits = log2Up(nClientStates) - def masterStateBits = log2Up(nMasterStates) - def acquireTypeBits = log2Up(nAcquireTypes) - def probeTypeBits = log2Up(nProbeTypes) - def releaseTypeBits = log2Up(nReleaseTypes) - def grantTypeBits = log2Up(nGrantTypes) + def clientStateWidth = log2Up(nClientStates) + def masterStateWidth = log2Up(nMasterStates) + def acquireTypeWidth = log2Up(nAcquireTypes) + def probeTypeWidth = log2Up(nProbeTypes) + def releaseTypeWidth = log2Up(nReleaseTypes) + def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: Bits, state: UFix): Bool - def isValid (state: UFix): Bool + def isHit (cmd: UInt, state: UInt): Bool + def isValid (state: UInt): Bool - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool - def needsWriteback (state: UFix): Bool + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool + def needsWriteback (state: UInt): Bool - def newStateOnHit(cmd: Bits, state: UFix): UFix - def newStateOnCacheControl(cmd: Bits): UFix - def newStateOnWriteback(): UFix - def newStateOnFlush(): UFix - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix - def newStateOnProbe(incoming: Probe, state: UFix): Bits + def newStateOnHit(cmd: UInt, state: UInt): UInt + def newStateOnCacheControl(cmd: UInt): UInt + def newStateOnWriteback(): UInt + def newStateOnFlush(): UInt + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt + def newStateOnProbe(incoming: Probe, state: UInt): Bits - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix - def getProbeType(a_type: UFix, global_state: UFix): UFix - def getReleaseTypeOnCacheControl(cmd: Bits): Bits + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, global_state: UInt): UInt + def getReleaseTypeOnCacheControl(cmd: UInt): Bits def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits - def getGrantType(a_type: UFix, count: UFix): Bits - def getGrantType(rel: Release, count: UFix): Bits + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits + def getGrantType(a_type: UInt, count: UInt): Bits + def getGrantType(rel: Release, count: UInt): Bits def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -49,15 +49,15 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UFix, global_state: UFix): Bool - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool - def needsAckReply(a_type: UFix, global_state: UFix): Bool + def needsOuterRead(a_type: UInt, global_state: UInt): Bool + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool + def needsAckReply(a_type: UInt, global_state: UInt): Bool def needsSelfProbe(acq: Acquire): Bool def requiresAck(grant: Grant): Bool def requiresAck(release: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uFixListContains(list: List[UFix], elem: UFix): Bool = list.map(elem === _).reduceLeft(_||_) + def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -73,19 +73,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UFix): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): Bits = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UFix, count: UFix): Bits = Bits(0) - def getGrantType(rel: Release, count: UFix): Bits = Bits(0) - def getProbeType(a_type: UFix, global_state: UFix): UFix = UFix(0) - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = Bool(false) - def needsAckReply(a_type: UFix, global_state: UFix): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) + def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) + def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) def requiresAck(grant: Grant) = Bool(true) def requiresAck(release: Release) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = Bool(false) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -96,27 +96,27 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UFix() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UFix() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) val hasDataGrantTypeList = List(grantData) - def isHit ( cmd: Bits, state: UFix): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UFix): Bool = state != tileInvalid + def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = state === tileDirty - def needsWriteback (state: UFix): Bool = state === tileDirty + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty + def needsWriteback (state: UInt): Bool = state === tileDirty - def newState(cmd: Bits, state: UFix): UFix = { + def newState(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = newState(cmd, state) - def newStateOnCacheControl(cmd: Bits) = tileInvalid //TODO + def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) + def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO def newStateOnWriteback() = tileInvalid def newStateOnFlush() = tileInvalid def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { @@ -129,23 +129,23 @@ class ThreeStateIncoherence extends IncoherentPolicy { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -156,35 +156,35 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileValid), Array( M_INV -> (state === tileValid), M_CLN -> (state === tileValid) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = state - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = state + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileValid @@ -192,7 +192,7 @@ class MICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> tileValid, grantReadUncached -> tileInvalid, @@ -202,7 +202,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -218,11 +218,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +235,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,13 +258,13 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -275,19 +275,19 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MEICoherence extends CoherencePolicyWithUncached { @@ -298,40 +298,40 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = state != tileInvalid - def isValid (state: UFix): Bool = state != tileInvalid + def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid + def isValid (state: UInt): Bool = state != tileInvalid - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileExclusiveClean @@ -339,7 +339,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), grantReadExclusiveAck -> tileExclusiveDirty, @@ -350,7 +350,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -367,15 +367,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +390,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,14 +413,14 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -432,20 +432,20 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence extends CoherencePolicyWithUncached { @@ -456,45 +456,45 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -502,7 +502,7 @@ class MSICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> tileExclusiveDirty, @@ -514,7 +514,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -531,15 +531,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,21 +554,21 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -577,14 +577,14 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -593,20 +593,20 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence extends CoherencePolicyWithUncached { @@ -617,45 +617,45 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UFix() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: Bits, state: UFix): Bool = { + def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( M_INV -> (state === tileExclusiveDirty), M_CLN -> (state === tileExclusiveDirty) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), tileExclusiveDirty, state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -663,7 +663,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), @@ -675,7 +675,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -692,15 +692,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,21 +715,21 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -738,14 +738,14 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -757,13 +757,13 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } @@ -771,7 +771,7 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence extends CoherencePolicyWithUncached { @@ -782,45 +782,45 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UFix() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UFix() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UFix() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UFix() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UFix() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: Bits, state: UFix): Bool = { - Mux(isWriteIntent(cmd), uFixListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + def isHit (cmd: UInt, state: UInt): Bool = { + Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } - def isValid (state: UFix): Bool = { + def isValid (state: UInt): Bool = { state != tileInvalid } - def needsTransactionOnSecondaryMiss(cmd: Bits, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: Bits, state: UFix): Bool = { + def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uFixListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } - def needsWriteback (state: UFix): Bool = { + def needsWriteback (state: UInt): Bool = { needsTransactionOnCacheControl(M_INV, state) } - def newStateOnHit(cmd: Bits, state: UFix): UFix = { + def newStateOnHit(cmd: UInt, state: UInt): UInt = { Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( tileExclusiveClean -> tileExclusiveDirty, tileMigratoryClean -> tileMigratoryDirty)), state) } - def newStateOnCacheControl(cmd: Bits) = { + def newStateOnCacheControl(cmd: UInt) = { MuxLookup(cmd, tileInvalid, Array( M_INV -> tileInvalid, M_CLN -> tileShared @@ -828,7 +828,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def newStateOnWriteback() = newStateOnCacheControl(M_INV) def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UFix = { + def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { MuxLookup(incoming.g_type, tileInvalid, Array( grantReadShared -> tileShared, grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( @@ -846,7 +846,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UFix): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): Bits = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -869,17 +869,17 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: Bits, state: UFix): UFix = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: Bits, state: UFix, outstanding: Acquire): UFix = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: Bits): Bits = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UFix): Bits = { + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uFixListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,21 +894,21 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uFixListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uFixListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uFixListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uFixListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uFixListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) - def getGrantType(a_type: UFix, count: UFix): Bits = { + def getGrantType(a_type: UInt, count: UInt): Bits = { MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UFix(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -918,14 +918,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UFix): Bits = { + def getGrantType(rel: Release, count: UInt): Bits = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UFix, global_state: UFix): UFix = { + def getProbeType(a_type: UInt, global_state: UInt): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -938,18 +938,18 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UFix, global_state: UFix): Bool = { + def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UFix, global_state: UFix): Bool = { + def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UFix, global_state: UFix): Bool = { + def needsAckReply(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck def requiresAck(release: Release) = Bool(false) def needsSelfProbe(acq: Acquire) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UFix, p_type: UFix): Bool = (r_type === releaseVoluntaryInvalidateData) + def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cb6465c5..3e7c2cd5 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -1,17 +1,17 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bits])(gen: => T) extends Component +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module { class Inputs extends Bundle { - val addr = UFix(INPUT, log2Up(n)) + val addr = UInt(INPUT, log2Up(n)) val rw = Bool(INPUT) val wdata = gen.asInput val wmask = gen.asInput override def clone = new Inputs().asInstanceOf[this.type] } val io = new Bundle { - val in = new PipeIO()(new Inputs).flip + val in = Valid(new Inputs).flip val rdata = gen.asOutput } val data = gen @@ -21,27 +21,27 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) - val rdataDeep = Vec(nDeep) { Bits() } - val rdataSel = Vec(nDeep) { Bool() } + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UFix(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = Reg(ren) - val rdata = Vec(nWide) { Bits() } + val reg_ren = RegUpdate(ren) + val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) for (j <- 0 until nWide) { val mem = leaf.clone var dout: Bits = null - val ridx = if (postLatency > 0) Reg() { Bits() } else null + val ridx = if (postLatency > 0) Reg(Bits()) else null var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UFixToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) { Bits() } + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -74,47 +74,49 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[Bit class LLCDataReq(ways: Int) extends MemReqCmd { - val way = UFix(width = log2Up(ways)) + val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component +class LLCTagReq(ways: Int) extends HasMemAddr +{ + val way = UInt(width = log2Up(ways)) + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] +} + +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module { val io = new Bundle { - val cpu = (new FIFOIO) { new MemReqCmd }.flip - val repl_way = UFix(INPUT, log2Up(ways)) + val cpu = Decoupled(new MemReqCmd).flip + val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UFix(INPUT, MEM_ADDR_BITS - log2Up(sets)) - val data = (new FIFOIO) { new LLCDataReq(ways) } - val tag = (new FIFOIO) { new Bundle { - val addr = UFix(width = MEM_ADDR_BITS) - val way = UFix(width = log2Up(ways)) - } } + val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val data = Decoupled(new LLCDataReq(ways)) + val tag = Decoupled(new LLCTagReq(ways)) val mem = new ioMemPipe - val mem_resp_set = UFix(OUTPUT, log2Up(sets)) - val mem_resp_way = UFix(OUTPUT, log2Up(ways)) + val mem_resp_set = UInt(OUTPUT, log2Up(sets)) + val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) - val way = UFix(width = log2Up(ways)) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UFix(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(REFILL_CYCLES)) val requested = Bool() val old_dirty = Bool() - val old_tag = UFix(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec(outstanding) { Reg(resetVal = Bool(false)) } + val valid = Vec.fill(outstanding){RegReset(Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) - val mshr = Vec(outstanding) { Reg() { new MSHR } } + val mshr = Vec.fill(outstanding){Reg(new MSHR)} when (io.cpu.valid && io.cpu.ready) { valid(freeId) := Bool(true) mshr(freeId).addr := io.cpu.bits.addr @@ -124,7 +126,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component mshr(freeId).old_tag := io.repl_tag mshr(freeId).wb_busy := Bool(false) mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UFix(0) + mshr(freeId).refillCount := UInt(0) mshr(freeId).refilled := Bool(false) } @@ -136,8 +138,8 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UFix(1) - when (refillCount === UFix(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + mshr(refillId).refillCount := refillCount + UInt(1) + when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -161,7 +163,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.data.bits.rw := Bool(false) io.data.bits.tag := mshr(replayId).tag io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUFix + io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt io.data.bits.way := mshr(writebackId).way when (replay) { io.data.valid := io.tag.ready @@ -181,25 +183,25 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Component io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Component +class LLCWriteback(requestors: Int) extends Module { val io = new Bundle { - val req = Vec(requestors) { (new FIFOIO) { UFix(width = MEM_ADDR_BITS) }.flip } - val data = Vec(requestors) { (new FIFOIO) { new MemData }.flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new ioMemPipe } - val valid = Reg(resetVal = Bool(false)) - val who = Reg() { UFix() } - val addr = Reg() { UFix() } - val cmd_sent = Reg() { Bool() } - val data_sent = Reg() { Bool() } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val valid = RegReset(Bool(false)) + val who = Reg(UInt()) + val addr = Reg(UInt()) + val cmd_sent = Reg(Bool()) + val data_sent = Reg(Bool()) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UFix(i) && io.mem.req_data.ready + io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready anyReq = anyReq || io.req(i).valid } @@ -213,8 +215,8 @@ class LLCWriteback(requestors: Int) extends Component } when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UFix(1) - when (count === UFix(REFILL_CYCLES-1)) { + count := count + UInt(1) + when (count === UInt(REFILL_CYCLES-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -230,51 +232,51 @@ class LLCWriteback(requestors: Int) extends Component io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Component +class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { - val req = (new FIFOIO) { new LLCDataReq(ways) }.flip - val req_data = (new FIFOIO) { new MemData }.flip - val writeback = (new FIFOIO) { UFix(width = MEM_ADDR_BITS) } - val writeback_data = (new FIFOIO) { new MemData } - val resp = (new FIFOIO) { new MemResp } - val mem_resp = (new PipeIO) { new MemResp }.flip - val mem_resp_set = UFix(INPUT, log2Up(sets)) - val mem_resp_way = UFix(INPUT, log2Up(ways)) + val req = Decoupled(new LLCDataReq(ways)).flip + val req_data = Decoupled(new MemData).flip + val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp) + val mem_resp = Valid(new MemResp).flip + val mem_resp_set = UInt(INPUT, log2Up(sets)) + val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS)) + val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] } - val q = (new Queue(latency+2)) { new QEntry } - val qReady = q.io.count <= UFix(q.entries-latency-1) - val valid = Reg(resetVal = Bool(false)) - val req = Reg() { io.req.bits.clone } - val count = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) + val q = Module(new Queue(new QEntry, latency+2)) + val qReady = q.io.count <= UInt(q.entries-latency-1) + val valid = RegReset(Bool(false)) + val req = Reg(io.req.bits.clone) + val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UFix(1) - when (valid && count === UFix(REFILL_CYCLES-1)) { valid := Bool(false) } + count := count + UInt(1) + when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UFix(1) } + when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := Fix(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUFix + data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := req.rw } when (io.mem_resp.valid) { data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUFix + data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt data.io.in.bits.rw := Bool(true) data.io.in.bits.wdata := io.mem_resp.bits.data } @@ -298,21 +300,21 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[Bits]) extends Compo io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Component // UNTESTED +class MemReqArb(n: Int) extends Module // UNTESTED { val io = new Bundle { - val cpu = Vec(n) { new ioMem().flip } + val cpu = Vec.fill(n){new ioMem().flip} val mem = new ioMem } - val lock = Reg(resetVal = Bool(false)) - val locker = Reg() { UFix() } + val lock = RegReset(Bool(false)) + val locker = Reg(UInt()) - val arb = new RRArbiter(n)(new MemReqCmd) + val arb = Module(new RRArbiter(new MemReqCmd, n)) val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UFix(log2Up(n)) + val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) for (i <- 0 until n) { - val me = UFix(i, log2Up(n)) + val me = UInt(i, log2Up(n)) arb.io.in(i).valid := io.cpu(i).req_cmd.valid arb.io.in(i).bits := io.cpu(i).req_cmd.bits arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) @@ -323,7 +325,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED val haveLock = lock && locker === me when (getLock) { lock := Bool(true) - locker := UFix(i) + locker := UInt(i) } when (getLock || haveLock) { io.cpu(i).req_data.ready := io.mem.req_data.ready @@ -341,7 +343,7 @@ class MemReqArb(n: Int) extends Component // UNTESTED when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], dataLeaf: Mem[Bits]) extends Component +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -351,48 +353,48 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da val tagWidth = MEM_ADDR_BITS - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty - val memCmdArb = (new Arbiter(2)) { new MemReqCmd } - val dataArb = (new Arbiter(2)) { new LLCDataReq(ways) } - val mshr = new LLCMSHRFile(sets, ways, outstanding) - val tags = new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways)) - val data = new LLCData(4, sets, ways, dataLeaf) - val writeback = new LLCWriteback(2) + val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) + val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) + val data = Module(new LLCData(4, sets, ways, dataLeaf)) + val writeback = Module(new LLCWriteback(2)) - val initCount = Reg(resetVal = UFix(0, log2Up(sets+1))) + val initCount = RegReset(UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UFix(1) } + when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = Reg(resetVal = Bool(false)) - val s2_valid = Reg(resetVal = Bool(false)) - val s2 = Reg() { new MemReqCmd } + val replay_s2 = RegReset(Bool(false)) + val s2_valid = RegReset(Bool(false)) + val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) - val s1 = Reg() { new MemReqCmd } + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } s2_valid := s1_valid replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec(ways) { Reg() { Bits(width = metaWidth) } } + val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} when (s1_valid) { s2 := s1 for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUFix(s2_hits) + val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUFix + val repl_tag = s2_tags(repl_way).toUInt val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) - val tag_wmask = Mux(initialize, Fix(-1, ways), UFixToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) + val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) tags.io.in.bits.rw := tag_we @@ -440,16 +442,16 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[Bits], da io.mem.req_data <> writeback.io.mem.req_data } -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) require(isPow2(entries) && entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = Reg(resetVal = Bool(false)) + val maybe_full = RegReset(Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -457,37 +459,37 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Component val ptr_match = enq_ptr === deq_ptr val empty = ptr_match && !maybe_full val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UFix(2) + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) do_flow := empty && io.deq.ready - val ram = Mem(entries, seqRead = true){Bits(width = data.getWidth)} - val ram_addr = Reg{Bits()} - val ram_out_valid = Reg{Bool()} + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UFix(1), deq_ptr) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, data.fromBits(ram(ram_addr))) + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) } -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Component +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module { - val io = new ioQueue(entries)(data) + val io = new QueueIO(data, entries) - val fq = new HellaFlowQueue(entries)(data) + val fq = Module(new HellaFlowQueue(entries)(data)) io.enq <> fq.io.enq io.deq <> Queue(fq.io.deq, 1, pipe = true) } object HellaQueue { - def apply[T <: Data](enq: FIFOIO[T], entries: Int) = { - val q = (new HellaQueue(entries)) { enq.bits.clone } + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) q.io.enq.valid := enq.valid // not using <> so that override is allowed q.io.enq.bits := enq.bits enq.ready := q.io.enq.ready @@ -495,7 +497,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module { val io = new Bundle { val cpu = new ioMem().flip @@ -507,17 +509,17 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component val inc = Bool() val dec = Bool() - val count = Reg(resetVal = UFix(numEntries, size)) - val watermark = count >= UFix(refillCycles) + val count = RegReset(UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) when (inc && !dec) { - count := count + UFix(1) + count := count + UInt(1) } when (!inc && dec) { - count := count - UFix(refillCycles) + count := count - UInt(refillCycles) } when (inc && dec) { - count := count - UFix(refillCycles-1) + count := count - UInt(refillCycles-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -528,7 +530,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Component io.mem.req_data <> io.cpu.req_data - val resp_dataq = (new HellaQueue(numEntries)) { new MemResp } + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) resp_dataq.io.enq <> io.mem.resp io.cpu.resp <> resp_dataq.io.deq diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..9badb5e9 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -2,40 +2,45 @@ package uncore import Chisel._ import scala.math._ -class MemData extends Bundle { +trait HasMemData extends Bundle { val data = Bits(width = MEM_DATA_BITS) } -class MemReqCmd extends Bundle { - val addr = UFix(width = MEM_ADDR_BITS) +trait HasMemAddr extends Bundle { + val addr = UInt(width = MEM_ADDR_BITS) +} + +trait HasMemTag extends Bundle { + val tag = UInt(width = MEM_TAG_BITS) +} + +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() - val tag = Bits(width = MEM_TAG_BITS) } -class MemResp extends Bundle { - val tag = Bits(width = MEM_TAG_BITS) - val data = Bits(width = MEM_DATA_BITS) -} +class MemResp extends HasMemData with HasMemTag + +class MemData extends HasMemData class ioMem extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new FIFOIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp).flip } class ioMemPipe extends Bundle { - val req_cmd = (new FIFOIO) { new MemReqCmd() } - val req_data = (new FIFOIO) { new MemData() } - val resp = (new PipeIO) { new MemResp() }.flip + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Valid(new MemResp).flip } class ioMemSerialized(w: Int) extends Bundle { - val req = (new FIFOIO) { Bits(width = w) } - val resp = (new PipeIO) { Bits(width = w) }.flip + val req = Decoupled(Bits(width = w)) + val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Component +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new ioMem().flip @@ -45,19 +50,19 @@ class MemSerdes(w: Int) extends Component val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val out_buf = Reg() { Bits() } - val in_buf = Reg() { Bits() } + val out_buf = Reg(Bits()) + val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_idle) - val send_cnt = Reg(resetVal = UFix(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.ready && send_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UFix((dbits-1)/w) + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val state = RegReset(s_idle) + val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := send_cnt + UFix(1) - out_buf := out_buf >> UFix(w) + send_cnt := send_cnt + UInt(1) + out_buf := out_buf >> UInt(w) } when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { out_buf := io.wide.req_cmd.bits.toBits @@ -76,31 +81,31 @@ class MemSerdes(w: Int) extends Component } when (state === s_read_addr && adone) { state := s_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_addr && adone) { state := s_write_idle - send_cnt := UFix(0) + send_cnt := UInt(0) } when (state === s_write_idle && io.wide.req_data.valid) { state := s_write_data } when (state === s_write_data && ddone) { - data_send_cnt := data_send_cnt + UFix(1) - state := Mux(data_send_cnt === UFix(REFILL_CYCLES-1), s_idle, s_write_idle) - send_cnt := UFix(0) + data_send_cnt := data_send_cnt + UInt(1) + state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + send_cnt := UInt(0) } - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val resp_val = Reg(resetVal = Bool(false)) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = RegReset(Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) - when (recv_cnt === UFix((rbits-1)/w)) { - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) + when (recv_cnt === UInt((rbits-1)/w)) { + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) resp_val := Bool(true) } in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) @@ -108,7 +113,7 @@ class MemSerdes(w: Int) extends Component io.wide.resp.valid := resp_val io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits.data := in_buf >> UInt(io.wide.resp.bits.tag.width) } class MemDesserIO(w: Int) extends Bundle { @@ -116,7 +121,7 @@ class MemDesserIO(w: Int) extends Bundle { val wide = new ioMem } -class MemDesser(w: Int) extends Component // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -124,59 +129,59 @@ class MemDesser(w: Int) extends Component // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(resetVal = UFix(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(resetVal = UFix(0, log2Up(REFILL_CYCLES))) - val adone = io.narrow.req.valid && recv_cnt === UFix((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UFix((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UFix((rbits-1)/w) + val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UFix() } - val state = Reg(resetVal = s_cmd_recv) + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val state = RegReset(s_cmd_recv) - val in_buf = Reg() { Bits() } + val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { - recv_cnt := recv_cnt + UFix(1) + recv_cnt := recv_cnt + UInt(1) in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) } io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv when (state === s_cmd_recv && adone) { state := s_cmd - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_cmd && io.wide.req_cmd.ready) { state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) } when (state === s_data_recv && ddone) { state := s_data - recv_cnt := UFix(0) + recv_cnt := UInt(0) } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - data_recv_cnt := data_recv_cnt + UFix(1) + data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UFix(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { state := s_cmd_recv } - recv_cnt := UFix(0) - data_recv_cnt := data_recv_cnt + UFix(1) + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) } - val req_cmd = in_buf >> UFix(((rbits+w-1)/w - (abits+w-1)/w)*w) + val req_cmd = in_buf >> UInt(((rbits+w-1)/w - (abits+w-1)/w)*w) io.wide.req_cmd.valid := state === s_cmd io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UFix(((rbits+w-1)/w - (dbits+w-1)/w)*w) + io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = (new Queue(REFILL_CYCLES)) { new MemResp } + val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UFix((rbits-1)/w) + dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UFix(w)) + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index cdaa8118..29c1dee8 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -3,24 +3,24 @@ import Chisel._ import scala.collection.mutable.Stack class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = new FIFOIO()(m) - val data = new FIFOIO()(d) + val meta = Decoupled(m) + val data = Decoupled(d) override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } } class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec(n) { new PairedDataIO()(m,d) }.flip + val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip val out = new PairedDataIO()(m,d) val meta_chosen = Bits(OUTPUT, log2Up(n)) val data_chosen = Bits(OUTPUT, log2Up(n)) override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Component { +class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) Reg(resetVal = Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(resetVal = UFix(n-1)) else UFix(n-1) + val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) + val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -28,7 +28,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data val grant_chosen_meta = !(locked && chosen_meta_has_data) (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UFix(i), grant(i) && valid_meta_has_data) && io.out.data.ready) + (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) io.out.meta.bits := io.in(meta_chosen).meta.bits @@ -37,11 +37,11 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = Reg(resetVal = UFix(0, width = log2Up(count))) - val cnt_next = cnt + UFix(1) + val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next - when(cnt_next === UFix(0)) { + when(cnt_next === UInt(0)) { locked := Bool(false) } } @@ -49,20 +49,20 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { when(!locked) { locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}){Bool()}.indexWhere{i: Bool => i} + lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} } } } } - val last_grant = Reg(resetVal = Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UFix(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UFix(i) > last_grant || ctrl(i + n)) + val last_grant = RegReset(Bits(0, log2Up(n))) + val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) + (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) var choose = Bits(n-1) for (i <- n-2 to 0 by -1) choose = Mux(io.in(i).meta.valid, Bits(i), choose) for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UFix(i) > last_grant, Bits(i), choose) + choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) meta_chosen := choose when (io.out.meta.fire()) { last_grant := meta_chosen } @@ -70,23 +70,23 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} } - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) + val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone) + val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UFix(i)) + arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UFix(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UFix(i)) + meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) + arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UFix(i)) + data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -99,8 +99,8 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { @@ -109,22 +109,22 @@ class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkC override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Component +abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}}.flip - val out = Vec(conf.nEndpoints){(new FIFOIO){(new PhysicalNetworkIO){data}}} + val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} } - val rdyVecs = List.fill(conf.nEndpoints)(Vec(conf.nEndpoints){Bool()}) + val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = (new LockingRRArbiter(conf.nEndpoints, count)){io.in(0).bits.clone} + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UFix(i)) + arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UFix(i)) + rdy := arb.ready && (in.bits.header.dst === UInt(i)) }} out <> rrarb.io.out }} @@ -135,30 +135,30 @@ class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: Physic case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { override val io: Vec[TileLinkType] val physicalNetworks: Seq[PhysicalNetwork] require(endpoints.length == conf.nEndpoints) } class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UFix(width = conf.idBits) - val dst = UFix(width = conf.idBits) + val src = UInt(width = conf.idBits) + val dst = UInt(width = conf.idBits) } object FIFOedLogicalNetworkIOWrapper { - def apply[T <: Data](in: FIFOIO[T], src: UFix = UFix(0), dst: UFix = UFix(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone } + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){ data }.flip - val out = (new FIFOIO){(new LogicalNetworkIO){ data }} + val in = Decoupled(data).flip + val out = Decoupled((new LogicalNetworkIO){data}) } io.out.valid := io.in.valid io.out.bits.payload := io.in.bits @@ -168,18 +168,18 @@ class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UFix, dst: UFix)(data: => T) } object FIFOedLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: FIFOIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = (new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone } + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) shim.io.in.valid := in.valid shim.io.in.bits := in.bits in.ready := shim.io.in.ready shim.io.out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Component { +class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { val io = new Bundle { - val in = (new FIFOIO){(new LogicalNetworkIO){ data }}.flip - val out = (new FIFOIO){ data } + val in = Decoupled((new LogicalNetworkIO){data}).flip + val out = Decoupled(data) } io.out.valid := io.in.valid io.out.bits := io.in.bits.payload diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index b56b1396..4d42fccf 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -1,71 +1,68 @@ package uncore import Chisel._ -class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Component +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module { val io = new Bundle { - val out_fast = new FIFOIO()(data).flip - val out_slow = new FIFOIO()(data) - - val in_fast = new FIFOIO()(data) - val in_slow = new FIFOIO()(data).flip - + val out_fast = Decoupled(data).flip + val out_slow = Decoupled(data) + val in_fast = Decoupled(data) + val in_slow = Decoupled(data).flip val clk_slow = Bool(OUTPUT) - - val set_divisor = new PipeIO()(Bits(width = 32)).flip + val set_divisor = Valid(Bits(width = 32)).flip val divisor = Bits(OUTPUT, 32) } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = Reg(resetVal = UFix(divisor_max-1)) - val d_shadow = Reg(resetVal = UFix(divisor_max-1)) - val hold = Reg(resetVal = UFix(divisor_max/4-1)) - val h_shadow = Reg(resetVal = UFix(divisor_max/4-1)) + val divisor = RegReset(UInt(divisor_max-1)) + val d_shadow = RegReset(UInt(divisor_max-1)) + val hold = RegReset(UInt(divisor_max/4-1)) + val h_shadow = RegReset(UInt(divisor_max/4-1)) when (io.set_divisor.valid) { - d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUFix - h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUFix + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt } - io.divisor := hold << UFix(16) | divisor + io.divisor := hold << UInt(16) | divisor - val count = Reg{UFix(width = log2Up(divisor_max))} - val clock = Reg{Bool()} - count := count + UFix(1) + val count = Reg{UInt(width = log2Up(divisor_max))} + val myclock = Reg{Bool()} + count := count + UInt(1) - val rising = count === (divisor >> UFix(1)) + val rising = count === (divisor >> UInt(1)) val falling = count === divisor - val held = count === (divisor >> UFix(1)) + hold + val held = count === (divisor >> UInt(1)) + hold when (falling) { divisor := d_shadow hold := h_shadow - count := UFix(0) - clock := Bool(false) + count := UInt(0) + myclock := Bool(false) } when (rising) { - clock := Bool(true) + myclock := Bool(true) } - val in_slow_rdy = Reg(resetVal = Bool(false)) - val out_slow_val = Reg(resetVal = Bool(false)) - val out_slow_bits = Reg() { data } + val in_slow_rdy = RegReset(Bool(false)) + val out_slow_val = RegReset(Bool(false)) + val out_slow_bits = Reg(data) - val fromhost_q = new Queue(1)(data) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) + val fromhost_q = Module(new Queue(data,1)) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast - val tohost_q = new Queue(1)(data) + val tohost_q = Module(new Queue(data,1)) tohost_q.io.enq <> io.out_fast tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy io.out_slow.valid := out_slow_val io.out_slow.bits := out_slow_bits - io.clk_slow := clock + io.clk_slow := myclock } diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index 6f7e6175..c16801ed 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,10 +5,11 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration + // TODO: override clone here, passing conf } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UFix(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = PADDR_BITS - OFFSET_BITS) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -29,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -39,7 +40,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -49,7 +50,7 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UFix, client_xact_id: UFix, subword_addr: UFix, atomic_opcode: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -61,7 +62,7 @@ object Acquire } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeBits) + val a_type = Bits(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -74,13 +75,13 @@ class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeBits) + val p_type = Bits(width = conf.co.probeTypeWidth) override def clone = { (new Probe).asInstanceOf[this.type] } } object Release { - def apply(r_type: Bits, addr: UFix, client_xact_id: UFix, master_xact_id: UFix)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -90,7 +91,7 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeBits) + val r_type = Bits(width = conf.co.releaseTypeWidth) override def clone = { (new Release).asInstanceOf[this.type] } } @@ -99,7 +100,7 @@ class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSource } class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeBits) + val g_type = Bits(width = conf.co.grantTypeWidth) override def clone = { (new Grant).asInstanceOf[this.type] } } @@ -111,13 +112,13 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with ClientSourcedIO { +class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } } class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends FIFOIO()(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { flip() override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } } @@ -142,25 +143,25 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI /* * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Component { +abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UFix + def arbIdx(in: Grant): UInt } */ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UFix(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UFix(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUFix + def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) + def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) + def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -170,14 +171,14 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -186,18 +187,18 @@ class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileL } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UFix = in.client_xact_id + def arbIdx(in: Grant): UInt = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -207,14 +208,14 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } @@ -223,18 +224,18 @@ class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfi } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Component { +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) - def acquireClientXactId(in: Acquire, id: Int) = UFix(id, log2Up(n)) - def grantClientXactId(in: Grant) = UFix(0) // DNC + def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) + def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id val io = new Bundle { - val in = Vec(n) { new UncachedTileLinkIO }.flip + val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData}) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -244,14 +245,14 @@ class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConf req.meta.ready := arb.meta.ready }} - val grant_ack_arb = (new RRArbiter(n)){ (new LogicalNetworkIO){new GrantAck} } + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } io.out.grant.ready := Bool(false) for (i <- 0 until n) { io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UFix(i)) { + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { io.in(i).grant.valid := io.out.grant.valid io.out.grant.ready := io.in(i).grant.ready } diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index 60e100e0..dde4190a 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -1,11 +1,11 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Component with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO - val incoherent = Vec(conf.ln.nClients) { Bool() }.asInput + val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput } } @@ -17,8 +17,8 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Create SHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(new VoluntaryReleaseTracker(_, bankId)) ++ - (conf.nReleaseTransactions until nTrackers).map(new AcquireTracker(_, bankId)) + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -28,7 +28,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = (new Arbiter(trackerList.size)) { Bool() } + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client alloc_arb.io.in(i).valid := t.acquire.meta.ready @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Probe }} + val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -52,21 +52,21 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val voluntary = co.isVoluntary(release.meta.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)){Bool()}.lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UFix(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UFix(0), release.meta.bits.payload.master_xact_id) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UFix(i)) && !block_releases + t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases t.release.data.bits := release.data.bits t.release.data.valid := release.data.valid } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)){Bool()}(release_idx) && !block_releases + release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = (new Arbiter(trackerList.size)){(new LogicalNetworkIO){ new Grant }} + val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } @@ -77,13 +77,13 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = new UncachedTileLinkIOArbiterThatPassesId(trackerList.size) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } io.master <> outer_arb.io.out } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Component with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -95,13 +95,13 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val mem_cmd_sent = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = RegReset(Bool(false)) + val mem_cmd_sent = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) @@ -110,23 +110,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.master.acquire.meta.valid := Bool(false) io.master.acquire.meta.bits.payload := cmd_to_write //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) + io.master.acquire.data.bits.payload.data := UInt(0) //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UFix(bankId) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.probe.valid := Bool(false) io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) // DNC io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UFix(0)) + io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) - io.client.grant.bits.payload.data := UFix(0) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) + io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) + io.client.grant.bits.header.src := UInt(bankId) io.client.grant_ack.valid := Bool(false) switch (state) { @@ -136,7 +136,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe xact := io.client.release.meta.bits.payload init_client_id_ := io.client.release.meta.bits.header.src release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UFix(0) + mem_cnt := UInt(0) mem_cmd_sent := Bool(false) state := s_mem } @@ -159,56 +159,54 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UFix() } - val state = Reg(resetVal = s_idle) + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val state = RegReset(s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UFix(0) else Reg(resetVal = UFix(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(resetVal = Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) + val probe_flags = RegReset(Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(resetVal = Bool(false)) - val acquire_data_needs_write = Reg(resetVal = Bool(false)) - val release_data_needs_write = Reg(resetVal = Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UFix(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - val a_w_mem_cmd_sent = Reg(resetVal = Bool(false)) - val r_w_mem_cmd_sent = Reg(resetVal = Bool(false)) + val x_needs_read = RegReset(Bool(false)) + val acquire_data_needs_write = RegReset(Bool(false)) + val release_data_needs_write = RegReset(Bool(false)) + val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val a_w_mem_cmd_sent = RegReset(Bool(false)) + val r_w_mem_cmd_sent = RegReset(Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UFixToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UFix(trackerId)) - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UFix(bankId) + io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + io.master.acquire.meta.bits.header.src := UInt(bankId) io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UFix(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header := UFix(bankId) + io.master.acquire.data.bits.payload.data := UInt(0) + io.master.acquire.data.bits.header.src := UInt(bankId) io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UFix(0)) - io.client.probe.bits.payload.master_xact_id := UFix(trackerId) + io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) + io.client.probe.bits.payload.master_xact_id := UInt(trackerId) io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UFix(0) - io.client.probe.bits.header.src := UFix(bankId) + io.client.probe.bits.header.dst := UInt(0) + io.client.probe.bits.header.src := UInt(bankId) io.client.grant.bits.payload.data := io.master.grant.bits.payload.data io.client.grant.bits.payload.g_type := grant_type io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UFix(trackerId) + io.client.grant.bits.payload.master_xact_id := UInt(trackerId) io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UFix(bankId) - io.client.grant.valid := (io.master.grant.valid && (UFix(trackerId) === io.master.grant.bits.payload.client_xact_id)) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) io.client.acquire.meta.ready := Bool(false) io.client.acquire.data.ready := Bool(false) io.client.release.meta.ready := Bool(false) @@ -222,11 +220,11 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when( io.client.acquire.meta.valid ) { xact := io.client.acquire.meta.bits.payload init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UFix(ln.nClients) // TODO: Broadcast only + init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UFix(0)) + x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) probe_flags := probe_initial_flags - mem_cnt := UFix(0) + mem_cnt := UInt(0) r_w_mem_cmd_sent := Bool(false) a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { @@ -241,12 +239,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.probe.bits.header.dst := curr_p_id } when(io.client.probe.ready) { - probe_flags := probe_flags & ~(UFixToOH(curr_p_id)) + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } io.client.release.meta.ready := Bool(true) when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UFix(1) - when(release_count === UFix(1)) { + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { state := s_mem } when( co.messageHasData(io.client.release.meta.bits.payload)) { @@ -281,7 +279,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UFix(0)), s_ack, + state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) } } @@ -290,7 +288,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UFix(trackerId)) { + when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } @@ -298,10 +296,10 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = Reg(resetVal = UFix(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UFix(1) + val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt_next = mem_cnt + UInt(1) - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: FIFOIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UFix) { + def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) master_acq.meta.bits.payload := cmd master_acq.data.bits.payload := client_data.bits.payload @@ -315,7 +313,7 @@ abstract trait OuterRequestGenerator { when(master_acq.data.ready) { client_data.ready:= Bool(true) mem_cnt := mem_cnt_next - when(mem_cnt === UFix(REFILL_CYCLES-1)) { + when(mem_cnt === UInt(REFILL_CYCLES-1)) { trigger := Bool(false) } } From d5c9eb0f54e7e27f0d07126c79dbc6c1e701a6c5 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Mon, 12 Aug 2013 20:52:18 -0700 Subject: [PATCH 187/688] reset -> resetVal, getReset -> reset --- uncore/src/llc.scala | 2 +- uncore/src/slowio.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..db49f027 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetVal = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..b8be1ed7 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(this.reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 9162fbc9b5dde864c85a7e0555eeccd05c84f866 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 12 Aug 2013 23:15:54 -0700 Subject: [PATCH 188/688] Clean up cloning in tilelink bundles --- uncore/src/tilelink.scala | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index c16801ed..fdd65094 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -5,7 +5,7 @@ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNet abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration - // TODO: override clone here, passing conf + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { @@ -30,7 +30,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -40,24 +40,15 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration) = { - val acq = new Acquire - acq.a_type := a_type - acq.addr := addr - acq.client_xact_id := client_xact_id + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode - acq.write_mask := Bits(0) acq } } @@ -66,17 +57,12 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) - override def clone = { (new Acquire).asInstanceOf[this.type] } } - -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new AcquireData).asInstanceOf[this.type] } -} +class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = Bits(width = conf.co.probeTypeWidth) - override def clone = { (new Probe).asInstanceOf[this.type] } } object Release @@ -92,22 +78,15 @@ object Release } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { val r_type = Bits(width = conf.co.releaseTypeWidth) - override def clone = { (new Release).asInstanceOf[this.type] } } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData { - override def clone = { (new ReleaseData).asInstanceOf[this.type] } -} +class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { val g_type = Bits(width = conf.co.grantTypeWidth) - override def clone = { (new Grant).asInstanceOf[this.type] } -} - -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId { - override def clone = { (new GrantAck).asInstanceOf[this.type] } } +class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId trait DirectionalIO trait ClientSourcedIO extends DirectionalIO From 7ff4126d049449aec3e4ff4e20fab8d2f08cd049 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 00:01:11 -0700 Subject: [PATCH 189/688] Abstracted UncachedTileLinkIOArbiters --- uncore/src/tilelink.scala | 136 ++++++++++---------------------------- 1 file changed, 35 insertions(+), 101 deletions(-) diff --git a/uncore/src/tilelink.scala b/uncore/src/tilelink.scala index fdd65094..6a3c8e20 100644 --- a/uncore/src/tilelink.scala +++ b/uncore/src/tilelink.scala @@ -120,123 +120,57 @@ class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkI override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -/* - * TODO: Merge the below classes into children of an abstract class in Chisel 2.0 -abstract class UncachedTileLinkIOArbiter(n: Int, co: CoherencePolicy)(implicit conf: LogicalNetworkConfiguration) extends Module { +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { def acquireClientXactId(in: Acquire, id: Int): Bits def grantClientXactId(in: Grant): Bits def arbIdx(in: Grant): UInt -} -*/ -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + val io = new Bundle { + val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val out = new UncachedTileLinkIO + } + def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) + val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + io.out.acquire <> acq_arb.io.out + io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + + val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + + io.out.grant.ready := Bool(false) + for (i <- 0 until n) { + io.in(i).grant.valid := Bool(false) + when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { + io.in(i).grant.valid := io.out.grant.valid + io.out.grant.ready := io.in(i).grant.ready + } + io.in(i).grant.bits := io.out.grant.bits + io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) + } +} + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id def grantClientXactId(in: Grant) = in.client_xact_id def arbIdx(in: Grant): UInt = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) +class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) def grantClientXactId(in: Grant) = UInt(0) // DNC def arbIdx(in: Grant) = in.client_xact_id - - val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip - val out = new UncachedTileLinkIO - } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} - - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } - - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) - } } - From 1308c08baa0eccdd98c3ec022a9de6217cc11cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Aug 2013 17:52:53 -0700 Subject: [PATCH 190/688] Reg standardization --- uncore/src/llc.scala | 2 +- uncore/src/network.scala | 42 ++++++++++++---------------------------- uncore/src/slowio.scala | 4 ++-- 3 files changed, 15 insertions(+), 33 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index 3e7c2cd5..cd4341bb 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -370,7 +370,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(update = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, reset = Bool(false)) + val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 29c1dee8..8733e2a6 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -148,43 +148,25 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOWrapper(src, dst)){ in.bits.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOWrapper[T <: Data](src: UInt, dst: UInt)(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled(data).flip - val out = Decoupled((new LogicalNetworkIO){data}) - } - io.out.valid := io.in.valid - io.out.bits.payload := io.in.bits - io.out.bits.header.dst := dst - io.out.bits.header.src := src - io.in.ready := io.out.ready -} object FIFOedLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { - val shim = Module((new FIFOedLogicalNetworkIOUnwrapper){ in.bits.payload.clone }) - shim.io.in.valid := in.valid - shim.io.in.bits := in.bits - in.ready := shim.io.in.ready - shim.io.out + val out = Decoupled(in.bits.payload.clone).asDirectionless + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out } } -class FIFOedLogicalNetworkIOUnwrapper[T <: Data]()(data: => T)(implicit lconf: LogicalNetworkConfiguration) extends Module { - val io = new Bundle { - val in = Decoupled((new LogicalNetworkIO){data}).flip - val out = Decoupled(data) - } - io.out.valid := io.in.valid - io.out.bits := io.in.bits.payload - io.in.ready := io.out.ready -} class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 4d42fccf..93c1324a 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -47,7 +47,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.getReset) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || reset) fromhost_q.io.enq.bits := io.in_slow.bits fromhost_q.io.deq <> io.in_fast @@ -58,7 +58,7 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module when (held) { in_slow_rdy := fromhost_q.io.enq.ready out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.getReset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + out_slow_bits := Mux(reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) } io.in_slow.ready := in_slow_rdy From 17d404b325fa71f715f03356baa060386840a41d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:27:38 -0700 Subject: [PATCH 191/688] final Reg changes --- uncore/src/llc.scala | 28 ++++++++++++++-------------- uncore/src/memserdes.scala | 18 +++++++++--------- uncore/src/network.scala | 8 ++++---- uncore/src/slowio.scala | 12 ++++++------ uncore/src/uncore.scala | 32 ++++++++++++++++---------------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/uncore/src/llc.scala b/uncore/src/llc.scala index cd4341bb..f9907934 100644 --- a/uncore/src/llc.scala +++ b/uncore/src/llc.scala @@ -29,7 +29,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val wdata = in.bits.wdata.toBits val wmask = in.bits.wmask.toBits val ren = in.valid && !in.bits.rw - val reg_ren = RegUpdate(ren) + val reg_ren = Reg(next=ren) val rdata = Vec.fill(nWide){Bits()} val r = Pipe(ren, in.bits.addr, postLatency) @@ -113,7 +113,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module override def clone = new MSHR().asInstanceOf[this.type] } - val valid = Vec.fill(outstanding){RegReset(Bool(false))} + val valid = Vec.fill(outstanding){Reg(init=Bool(false))} val validBits = valid.toBits val freeId = PriorityEncoder(~validBits) val mshr = Vec.fill(outstanding){Reg(new MSHR)} @@ -191,12 +191,12 @@ class LLCWriteback(requestors: Int) extends Module val mem = new ioMemPipe } - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val who = Reg(UInt()) val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -252,10 +252,10 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul } val q = Module(new Queue(new QEntry, latency+2)) val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = RegReset(Bool(false)) + val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) @@ -307,7 +307,7 @@ class MemReqArb(n: Int) extends Module // UNTESTED val mem = new ioMem } - val lock = RegReset(Bool(false)) + val lock = Reg(init=Bool(false)) val locker = Reg(UInt()) val arb = Module(new RRArbiter(new MemReqCmd, n)) @@ -360,17 +360,17 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], da val data = Module(new LLCData(4, sets, ways, dataLeaf)) val writeback = Module(new LLCWriteback(2)) - val initCount = RegReset(UInt(0, log2Up(sets+1))) + val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) when (initialize) { initCount := initCount + UInt(1) } - val replay_s2 = RegReset(Bool(false)) - val s2_valid = RegReset(Bool(false)) + val replay_s2 = Reg(init=Bool(false)) + val s2_valid = Reg(init=Bool(false)) val s2 = Reg(new MemReqCmd) val s3_rdy = Bool() val replay_s2_rdy = Bool() - val s1_valid = Reg(updateData = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, resetData = Bool(false)) + val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) val s1 = Reg(new MemReqCmd) when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } when (replay_s2 && replay_s2_rdy) { s1 := s2 } @@ -451,7 +451,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow - val maybe_full = RegReset(Bool(false)) + val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 val deq_ptr = Counter(do_deq, entries)._1 when (do_enq != do_deq) { maybe_full := do_enq } @@ -509,7 +509,7 @@ class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module val inc = Bool() val dec = Bool() - val count = RegReset(UInt(numEntries, size)) + val count = Reg(init=UInt(numEntries, size)) val watermark = count >= UInt(refillCycles) when (inc && !dec) { diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index 9badb5e9..7a021621 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -54,9 +54,9 @@ class MemSerdes(w: Int) extends Module val in_buf = Reg(Bits()) val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } - val state = RegReset(s_idle) - val send_cnt = RegReset(UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val state = Reg(init=s_idle) + val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -96,9 +96,9 @@ class MemSerdes(w: Int) extends Module send_cnt := UInt(0) } - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) - val resp_val = RegReset(Bool(false)) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) when (io.narrow.resp.valid) { @@ -129,14 +129,14 @@ class MemDesser(w: Int) extends Module // test rig side val rbits = io.wide.resp.bits.getWidth require(dbits >= abits && rbits >= dbits) - val recv_cnt = RegReset(UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = RegReset(UInt(0, log2Up(REFILL_CYCLES))) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } - val state = RegReset(s_cmd_recv) + val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { diff --git a/uncore/src/network.scala b/uncore/src/network.scala index 8733e2a6..9de890b1 100644 --- a/uncore/src/network.scala +++ b/uncore/src/network.scala @@ -19,8 +19,8 @@ class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bu class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { require(isPow2(count)) val io = new PairedArbiterIO(n)(meta,data) - val locked = if(count > 1) RegReset(Bool(false)) else Bool(false) - val lockIdx = if(count > 1) RegReset(UInt(n-1)) else UInt(n-1) + val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) + val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) val meta_chosen = Bits(width = log2Up(n)) @@ -37,7 +37,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock io.data_chosen := Mux(locked, lockIdx, meta_chosen) if(count > 1){ - val cnt = RegReset(UInt(0, width = log2Up(count))) + val cnt = Reg(init=UInt(0, width = log2Up(count))) val cnt_next = cnt + UInt(1) when(io.out.data.fire()){ cnt := cnt_next @@ -54,7 +54,7 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock } } } - val last_grant = RegReset(Bits(0, log2Up(n))) + val last_grant = Reg(init=Bits(0, log2Up(n))) val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) diff --git a/uncore/src/slowio.scala b/uncore/src/slowio.scala index 93c1324a..79c837d1 100644 --- a/uncore/src/slowio.scala +++ b/uncore/src/slowio.scala @@ -14,10 +14,10 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module } require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = RegReset(UInt(divisor_max-1)) - val d_shadow = RegReset(UInt(divisor_max-1)) - val hold = RegReset(UInt(divisor_max/4-1)) - val h_shadow = RegReset(UInt(divisor_max/4-1)) + val divisor = Reg(init=UInt(divisor_max-1)) + val d_shadow = Reg(init=UInt(divisor_max-1)) + val hold = Reg(init=UInt(divisor_max/4-1)) + val h_shadow = Reg(init=UInt(divisor_max/4-1)) when (io.set_divisor.valid) { d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt @@ -42,8 +42,8 @@ class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module myclock := Bool(true) } - val in_slow_rdy = RegReset(Bool(false)) - val out_slow_val = RegReset(Bool(false)) + val in_slow_rdy = Reg(init=Bool(false)) + val out_slow_val = Reg(init=Bool(false)) val out_slow_bits = Reg(data) val fromhost_q = Module(new Queue(data,1)) diff --git a/uncore/src/uncore.scala b/uncore/src/uncore.scala index dde4190a..bfc703fa 100644 --- a/uncore/src/uncore.scala +++ b/uncore/src/uncore.scala @@ -96,11 +96,11 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = RegReset(Bool(false)) - val mem_cmd_sent = RegReset(Bool(false)) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_needs_write = Reg(init=Bool(false)) + val mem_cmd_sent = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) io.has_acquire_conflict := Bool(false) @@ -160,23 +160,23 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } - val state = RegReset(s_idle) + val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = RegReset(UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = RegReset(UInt(0, width = log2Up(ln.nClients))) + val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) - val release_count = if (ln.nClients == 1) UInt(0) else RegReset(UInt(0, width = log2Up(ln.nClients))) - val probe_flags = RegReset(Bits(0, width = ln.nClients)) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = RegReset(Bool(false)) - val acquire_data_needs_write = RegReset(Bool(false)) - val release_data_needs_write = RegReset(Bool(false)) + val x_needs_read = Reg(init=Bool(false)) + val acquire_data_needs_write = Reg(init=Bool(false)) + val release_data_needs_write = Reg(init=Bool(false)) val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = RegReset(Bool(false)) - val r_w_mem_cmd_sent = RegReset(Bool(false)) + val a_w_mem_cmd_sent = Reg(init=Bool(false)) + val r_w_mem_cmd_sent = Reg(init=Bool(false)) val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { @@ -296,7 +296,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } abstract trait OuterRequestGenerator { - val mem_cnt = RegReset(UInt(0, width = log2Up(REFILL_CYCLES))) + val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) val mem_cnt_next = mem_cnt + UInt(1) def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { From 3763cd00049bfad25d1857223af32df504042f77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 15 Aug 2013 15:57:16 -0700 Subject: [PATCH 192/688] standardizing sbt build conventions --- uncore/README | 6 ++++++ uncore/build.sbt | 14 ++++++++++++++ uncore/src/{ => main/scala}/coherence.scala | 0 uncore/src/{ => main/scala}/consts.scala | 0 uncore/src/{ => main/scala}/llc.scala | 0 uncore/src/{ => main/scala}/memserdes.scala | 0 uncore/src/{ => main/scala}/network.scala | 0 uncore/src/{ => main/scala}/package.scala | 0 uncore/src/{ => main/scala}/slowio.scala | 0 uncore/src/{ => main/scala}/tilelink.scala | 0 uncore/src/{ => main/scala}/uncore.scala | 0 11 files changed, 20 insertions(+) create mode 100644 uncore/README create mode 100644 uncore/build.sbt rename uncore/src/{ => main/scala}/coherence.scala (100%) rename uncore/src/{ => main/scala}/consts.scala (100%) rename uncore/src/{ => main/scala}/llc.scala (100%) rename uncore/src/{ => main/scala}/memserdes.scala (100%) rename uncore/src/{ => main/scala}/network.scala (100%) rename uncore/src/{ => main/scala}/package.scala (100%) rename uncore/src/{ => main/scala}/slowio.scala (100%) rename uncore/src/{ => main/scala}/tilelink.scala (100%) rename uncore/src/{ => main/scala}/uncore.scala (100%) diff --git a/uncore/README b/uncore/README new file mode 100644 index 00000000..c920e19a --- /dev/null +++ b/uncore/README @@ -0,0 +1,6 @@ +This is the repository for uncore components assosciated with UCB_BAR chip +projects. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. +lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + diff --git a/uncore/build.sbt b/uncore/build.sbt new file mode 100644 index 00000000..d5f2e6ea --- /dev/null +++ b/uncore/build.sbt @@ -0,0 +1,14 @@ +organization := "edu.berkeley.cs" + +version := "2.0" + +name := "uncore" + +scalaVersion := "2.10.2" + +resolvers ++= Seq( + "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" +) + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" diff --git a/uncore/src/coherence.scala b/uncore/src/main/scala/coherence.scala similarity index 100% rename from uncore/src/coherence.scala rename to uncore/src/main/scala/coherence.scala diff --git a/uncore/src/consts.scala b/uncore/src/main/scala/consts.scala similarity index 100% rename from uncore/src/consts.scala rename to uncore/src/main/scala/consts.scala diff --git a/uncore/src/llc.scala b/uncore/src/main/scala/llc.scala similarity index 100% rename from uncore/src/llc.scala rename to uncore/src/main/scala/llc.scala diff --git a/uncore/src/memserdes.scala b/uncore/src/main/scala/memserdes.scala similarity index 100% rename from uncore/src/memserdes.scala rename to uncore/src/main/scala/memserdes.scala diff --git a/uncore/src/network.scala b/uncore/src/main/scala/network.scala similarity index 100% rename from uncore/src/network.scala rename to uncore/src/main/scala/network.scala diff --git a/uncore/src/package.scala b/uncore/src/main/scala/package.scala similarity index 100% rename from uncore/src/package.scala rename to uncore/src/main/scala/package.scala diff --git a/uncore/src/slowio.scala b/uncore/src/main/scala/slowio.scala similarity index 100% rename from uncore/src/slowio.scala rename to uncore/src/main/scala/slowio.scala diff --git a/uncore/src/tilelink.scala b/uncore/src/main/scala/tilelink.scala similarity index 100% rename from uncore/src/tilelink.scala rename to uncore/src/main/scala/tilelink.scala diff --git a/uncore/src/uncore.scala b/uncore/src/main/scala/uncore.scala similarity index 100% rename from uncore/src/uncore.scala rename to uncore/src/main/scala/uncore.scala From dc53529156d54eba953640e71f567c5bb9644cf4 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:00:51 -0700 Subject: [PATCH 193/688] added resolver, bumped chisel dependency --- uncore/build.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index d5f2e6ea..abe95365 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -8,7 +8,9 @@ scalaVersion := "2.10.2" resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" + "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", + "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" + ) -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.0" +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From 9aff60f340bb3d64ed7a1bf482cf80b93714cb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Aug 2013 16:16:42 -0700 Subject: [PATCH 194/688] whitespace error in build.sbt --- uncore/build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index abe95365..6ea793a4 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -10,7 +10,6 @@ resolvers ++= Seq( "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" - ) libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From b01fe4f6aa80e7b452910e22bdbbb996a4d26cc5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Sat, 24 Aug 2013 15:24:17 -0700 Subject: [PATCH 195/688] fix memserdes bit ordering --- uncore/src/memserdes.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/memserdes.scala b/uncore/src/memserdes.scala index b1272b28..2af04df8 100644 --- a/uncore/src/memserdes.scala +++ b/uncore/src/memserdes.scala @@ -107,8 +107,7 @@ class MemSerdes(w: Int) extends Component } io.wide.resp.valid := resp_val - io.wide.resp.bits.tag := in_buf(io.wide.resp.bits.tag.width-1,0) - io.wide.resp.bits.data := in_buf >> UFix(io.wide.resp.bits.tag.width) + io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } class MemDesserIO(w: Int) extends Bundle { From ee98cd8378c8b97e141f25390bb2f7cd1803e81d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 10:54:51 -0700 Subject: [PATCH 196/688] new enum syntax --- uncore/src/main/scala/coherence.scala | 66 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 4 +- uncore/src/main/scala/uncore.scala | 4 +- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7109be94..dcb80c8e 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -96,10 +96,10 @@ class ThreeStateIncoherence extends IncoherentPolicy { def nProbeTypes = 0 def nReleaseTypes = 2 def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(nClientStates){ UInt() } - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(nAcquireTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(nGrantTypes){ UInt() } + val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) + val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List() val hasDataAcquireTypeList = List(acquireWriteback) val hasDataReleaseTypeList = List(acquireWriteback) @@ -156,13 +156,13 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalValid :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -298,13 +298,13 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -456,13 +456,13 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -617,13 +617,13 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(nClientStates){ UInt() } - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(nMasterStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) @@ -782,12 +782,12 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(nClientStates){ UInt() } + val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(nAcquireTypes){ UInt() } - val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(nProbeTypes){ UInt() } - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(nReleaseTypes){ UInt() } - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(nGrantTypes){ UInt() } + val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) + val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 39c30e56..42611bc6 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -53,7 +53,7 @@ class MemSerdes(w: Int) extends Module val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(5) { UInt() } + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) @@ -134,7 +134,7 @@ class MemDesser(w: Int) extends Module // test rig side val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(5) { UInt() } + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(UInt(), 5) val state = Reg(init=s_cmd_recv) val in_buf = Reg(Bits()) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index bfc703fa..adf5f638 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -95,7 +95,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(4){ UInt() } + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) @@ -159,7 +159,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(5){ UInt() } + val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) From 1cac26fd76f7eea8668420d7d63903b42653ccb8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Sep 2013 16:15:41 -0700 Subject: [PATCH 197/688] NetworkIOs no longer use thunks --- uncore/src/main/scala/network.scala | 54 ++++++++++++++-------------- uncore/src/main/scala/tilelink.scala | 30 ++++++++-------- uncore/src/main/scala/uncore.scala | 4 +-- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 9de890b1..4905ffde 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -2,23 +2,23 @@ package uncore import Chisel._ import scala.collection.mutable.Stack -class PairedDataIO[M <: Data, D <: Data]()(m: => M, d: => D) extends Bundle { - val meta = Decoupled(m) - val data = Decoupled(d) - override def clone = { new PairedDataIO()(m,d).asInstanceOf[this.type] } +class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { + val meta = Decoupled(mType) + val data = Decoupled(dType) + override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } } -class PairedArbiterIO[M <: Data, D <: Data](n: Int)(m: => M, d: => D) extends Bundle { - val in = Vec.fill(n){new PairedDataIO()(m,d)}.flip - val out = new PairedDataIO()(m,d) - val meta_chosen = Bits(OUTPUT, log2Up(n)) - val data_chosen = Bits(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(n)(m,d).asInstanceOf[this.type] } +class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { + val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip + val out = new PairedDataIO(mType, dType) + val meta_chosen = UInt(OUTPUT, log2Up(n)) + val data_chosen = UInt(OUTPUT, log2Up(n)) + override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } } -class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock: Option[M => Bool] = None)(meta: => M, data: => D) extends Module { +class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { require(isPow2(count)) - val io = new PairedArbiterIO(n)(meta,data) + val io = new PairedArbiterIO(mType, dType, n) val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) val grant = List.fill(n)(Bool()) @@ -68,10 +68,10 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](n: Int, count: Int, needsLock when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(meta: => M, data: => D)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO()(new PhysicalNetworkIO()(meta),new PhysicalNetworkIO()(data))} + val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} } val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -79,7 +79,7 @@ class PairedCrossbar[M <: Data, D <: Data](count: Int, needsLock: Option[Physica val rdyVecs = metaRdyVecs zip dataRdyVecs io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(conf.nEndpoints, count, needsLock)(io.in(0).meta.bits.clone, io.in(0).data.bits.clone)) + val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) arb.meta.bits := in.meta.bits @@ -103,18 +103,18 @@ class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle val dst = UInt(width = conf.idBits) } -class PhysicalNetworkIO[T <: Data]()(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = (new PhysicalHeader) - val payload = data - override def clone = { new PhysicalNetworkIO()(data).asInstanceOf[this.type] } +class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { + val header = new PhysicalHeader + val payload = dType.clone + override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } } abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](count: Int = 1)(data: => T)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { val io = new Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled((new PhysicalNetworkIO){data})} + val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip + val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} } val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -148,7 +148,7 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { object FIFOedLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { - val out = Decoupled((new LogicalNetworkIO){in.bits.clone}).asDirectionless + val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits out.bits.header.dst := dst @@ -168,8 +168,8 @@ object FIFOedLogicalNetworkIOUnwrapper { } } -class LogicalNetworkIO[T <: Data]()(data: => T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { val header = new LogicalHeader - val payload = data - override def clone = { new LogicalNetworkIO()(data).asInstanceOf[this.type] } + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6a3c8e20..425cbccd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -91,32 +91,32 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe trait DirectionalIO trait ClientSourcedIO extends DirectionalIO trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO()(data).asInstanceOf[this.type] } +class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { + override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class ClientSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO()(meta,data).asInstanceOf[this.type] } +class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { + override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } } -class MasterSourcedFIFOIO[T <: Data]()(data: => T) extends DecoupledIO(data) with MasterSourcedIO { +class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedFIFOIO()(data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } } -class MasterSourcedDataIO[M <: Data, D <: Data]()(meta: => M, data: => D) extends PairedDataIO()(meta,data) with MasterSourcedIO { +class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { flip() - override def clone = { new MasterSourcedDataIO()(meta,data).asInstanceOf[this.type] } + override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } } class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Acquire), new LogicalNetworkIO()(new AcquireData)) - val grant = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Grant)) - val grant_ack = new ClientSourcedFIFOIO()(new LogicalNetworkIO()(new GrantAck)) + val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) + val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO()(new LogicalNetworkIO()(new Probe)) - val release = new ClientSourcedDataIO()(new LogicalNetworkIO()(new Release), new LogicalNetworkIO()(new ReleaseData)) + val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) + val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -131,7 +131,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur val out = new UncachedTileLinkIO } def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(n, REFILL_CYCLES, acqHasData _)((new LogicalNetworkIO){new Acquire},(new LogicalNetworkIO){new AcquireData})) + val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) io.out.acquire <> acq_arb.io.out io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { arb.data <> req.data @@ -141,7 +141,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur req.meta.ready := arb.meta.ready }} - val grant_ack_arb = Module(new RRArbiter((new LogicalNetworkIO){new GrantAck},n)) + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index adf5f638..a24f309d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -43,7 +43,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires // Handle probe request generation - val probe_arb = Module(new Arbiter((new LogicalNetworkIO){ new Probe }, trackerList.size)) + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.client.probe <> probe_arb.io.out probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } @@ -66,7 +66,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) // Reply to initial requestor - val grant_arb = Module(new Arbiter((new LogicalNetworkIO){ new Grant }, trackerList.size)) + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) io.client.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } From cc7783404df6dd87b7eaef3aa93b2b80906e8a13 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 12 Sep 2013 16:09:53 -0700 Subject: [PATCH 198/688] Add memory command M_XA_XOR --- uncore/src/main/scala/consts.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 15bfee4d..3a1a8482 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -21,13 +21,14 @@ trait MemoryOpConstants { val M_XWR = Bits("b00001"); // int store val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_XA_SWAP = Bits("b00100"); val M_FENCE = Bits("b00101"); // memory fence val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); - val M_XA_SWAP = Bits("b01001"); - val M_XA_AND = Bits("b01010"); - val M_XA_OR = Bits("b01011"); + val M_XA_XOR = Bits("b01001"); + val M_XA_OR = Bits("b01010"); + val M_XA_AND = Bits("b01011"); val M_XA_MIN = Bits("b01100"); val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); @@ -35,7 +36,7 @@ trait MemoryOpConstants { val M_INV = Bits("b10000"); // write back and invalidate line val M_CLN = Bits("b10001"); // write back line - def isAMO(cmd: Bits) = cmd(3) + def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) From 9bf10ae5d22ebcc927703d43842097867add7ea1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 19 Sep 2013 15:26:36 -0700 Subject: [PATCH 199/688] remove extraneous toBits (need new Chisel) --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From cc3dc1bd0fbed6aad7f8ba9ade655f80f5e12369 Mon Sep 17 00:00:00 2001 From: Huy Vo Date: Thu, 19 Sep 2013 20:10:56 -0700 Subject: [PATCH 200/688] bug fix --- uncore/src/main/scala/llc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f9907934..6627527a 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -466,7 +466,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val ram_addr = Reg(Bits()) val ram_out_valid = Reg(Bool()) ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits.toBits } + when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) From 20246b373ebfdb265c7bc095364384f1caee028b Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 24 Sep 2013 16:02:00 -0700 Subject: [PATCH 201/688] Properly ignore target files --- uncore/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 uncore/.gitignore diff --git a/uncore/.gitignore b/uncore/.gitignore new file mode 100644 index 00000000..555feb41 --- /dev/null +++ b/uncore/.gitignore @@ -0,0 +1,2 @@ +target/ +project/target/ From 42693d43adc3dbbb65b1b0a591d3a20db904a804 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Sep 2013 09:51:14 -0700 Subject: [PATCH 202/688] simplify build.sbt --- uncore/build.sbt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 6ea793a4..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,11 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -resolvers ++= Seq( - "Sonatype Snapshots" at "http://oss.sonatype.org/content/repositories/snapshots", - "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases", - "scct-github-repository" at "http://mtkopone.github.com/scct/maven-repo" -) - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "2.1-SNAPSHOT" From f440df5338d3c1e7ba56fcd57ca8fa2ae98f5f99 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 28 Oct 2013 22:37:41 -0700 Subject: [PATCH 203/688] rename M_FENCE to M_NOP --- uncore/src/main/scala/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 3a1a8482..e9d282ae 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -22,7 +22,7 @@ trait MemoryOpConstants { val M_PFR = Bits("b00010"); // prefetch with intent to read val M_PFW = Bits("b00011"); // prefetch with intent to write val M_XA_SWAP = Bits("b00100"); - val M_FENCE = Bits("b00101"); // memory fence + val M_NOP = Bits("b00101"); val M_XLR = Bits("b00110"); val M_XSC = Bits("b00111"); val M_XA_ADD = Bits("b01000"); From c350cbd6eae0153ed511ceac6b45a9476d52a965 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 13:19:04 -0800 Subject: [PATCH 204/688] move htif to uncore --- uncore/src/main/scala/htif.scala | 267 +++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 uncore/src/main/scala/htif.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala new file mode 100644 index 00000000..38fce75e --- /dev/null +++ b/uncore/src/main/scala/htif.scala @@ -0,0 +1,267 @@ +package rocket + +import Chisel._ +import Node._ +import uncore._ + +class HostIO(val w: Int) extends Bundle +{ + val clk = Bool(OUTPUT) + val clk_edge = Bool(OUTPUT) + val in = Decoupled(Bits(width = w)).flip + val out = Decoupled(Bits(width = w)) + val debug_stats_pcr = Bool(OUTPUT) +} + +class PCRReq extends Bundle +{ + val rw = Bool() + val addr = Bits(width = 5) + val data = Bits(width = 64) +} + +class HTIFIO(ntiles: Int) extends Bundle +{ + val reset = Bool(INPUT) + val id = UInt(INPUT, log2Up(ntiles)) + val pcr_req = Decoupled(new PCRReq).flip + val pcr_rep = Decoupled(Bits(width = 64)) + val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_rep = Decoupled(Bool()).flip + val debug_stats_pcr = Bool(OUTPUT) + // wired directly to stats register + // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work +} + +class SCRIO(n: Int) extends Bundle +{ + val rdata = Vec.fill(n){Bits(INPUT, 64)} + val wen = Bool(OUTPUT) + val waddr = UInt(OUTPUT, log2Up(n)) + val wdata = Bits(OUTPUT, 64) +} + +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +{ + implicit val (ln, co) = (conf.ln, conf.co) + val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val io = new Bundle { + val host = new HostIO(w) + val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val mem = new TileLinkIO + val scr = new SCRIO(nSCR) + } + + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) + // system is 'interesting' if any tile is 'interesting' + + val short_request_bits = 64 + val long_request_bits = 576 + require(short_request_bits % w == 0) + + val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits + val rx_count = Reg(init=UInt(0,rx_count_w)) + val rx_shifter = Reg(Bits(width = short_request_bits)) + val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) + val next_cmd = rx_shifter_in(3,0) + val cmd = Reg(Bits()) + val size = Reg(Bits()) + val pos = Reg(Bits()) + val seqno = Reg(Bits()) + val addr = Reg(Bits()) + when (io.host.in.valid && io.host.in.ready) { + rx_shifter := rx_shifter_in + rx_count := rx_count + UInt(1) + when (rx_count === UInt(short_request_bits/w-1)) { + cmd := next_cmd + size := rx_shifter_in(15,4) + pos := rx_shifter_in(15,4+OFFSET_BITS-3) + seqno := rx_shifter_in(23,16) + addr := rx_shifter_in(63,24) + } + } + + val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) + val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR + val packet_ram_depth = long_request_bits/short_request_bits-1 + val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + when (rx_word_done && io.host.in.ready) { + packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in + } + + val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) + + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_wdata = packet_ram(0) + + val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), + Bool(true))) + + val tx_count = Reg(init=UInt(0, rx_count_w)) + val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) + val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) + val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) + when (io.host.out.valid && io.host.out.ready) { + tx_count := tx_count + UInt(1) + } + + val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd != cmd_writemem && next_cmd != cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) + val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) + val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) + + val mem_acked = Reg(init=Bool(false)) + val mem_gxid = Reg(Bits()) + val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_needs_ack = Reg(Bool()) + when (io.mem.grant.valid) { + mem_acked := Bool(true) + mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gsrc := io.mem.grant.bits.header.src + mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + } + io.mem.grant.ready := Bool(true) + + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state = Reg(init=state_rx) + + val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) + when (state === state_rx && rx_done) { + state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, + state_tx)) + } + + val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val x_init = Module(new Queue(new Acquire, 1)) + when (state === state_mem_req && x_init.io.enq.ready) { + state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + } + when (state === state_mem_wdata && io.mem.acquire.data.ready) { + when (mem_cnt.andR) { + state := state_mem_wresp + } + mem_cnt := mem_cnt + UInt(1) + } + when (state === state_mem_wresp) { + when (mem_acked) { + state := state_mem_finish + mem_acked := Bool(false) + } + } + when (state === state_mem_rdata) { + when (io.mem.grant.valid) { + when (mem_cnt.andR) { + state := state_mem_finish + } + mem_cnt := mem_cnt + UInt(1) + } + mem_acked := Bool(false) + } + when (state === state_mem_finish && io.mem.grant_ack.ready) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << OFFSET_BITS-3) + } + when (state === state_tx && tx_done) { + when (tx_word_count === tx_size) { + rx_count := UInt(0) + tx_count := UInt(0) + } + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + } + + var mem_req_data: Bits = null + for (i <- 0 until MEM_DATA_BITS/short_request_bits) { + val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) + when (state === state_mem_rdata && io.mem.grant.valid) { + packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + } + mem_req_data = Cat(packet_ram(idx), mem_req_data) + } + x_init.io.enq.valid := state === state_mem_req + val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + x_init.io.enq.bits := Mux(cmd === cmd_writemem, + Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), + Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id + io.mem.acquire.data.valid := state === state_mem_wdata + io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid + io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.probe.ready := Bool(false) + io.mem.release.meta.valid := Bool(false) + io.mem.release.data.valid := Bool(false) + + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) + for (i <- 0 until nTiles) { + val my_reset = Reg(init=Bool(true)) + val my_ipi = Reg(init=Bool(false)) + + val cpu = io.cpu(i) + val me = pcr_coreid === UInt(i) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.bits.rw := cmd === cmd_writecr + cpu.pcr_req.bits.addr := pcr_addr + cpu.pcr_req.bits.data := pcr_wdata + cpu.reset := my_reset + + when (cpu.ipi_rep.ready) { + my_ipi := Bool(false) + } + cpu.ipi_rep.valid := my_ipi + cpu.ipi_req.ready := Bool(true) + for (j <- 0 until nTiles) { + when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { + my_ipi := Bool(true) + } + } + + when (cpu.pcr_req.valid && cpu.pcr_req.ready) { + state := state_pcr_resp + } + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (cmd === cmd_writecr) { + my_reset := pcr_wdata(0) + } + pcrReadData := my_reset.toBits + state := state_tx + } + + cpu.pcr_rep.ready := Bool(true) + when (cpu.pcr_rep.valid) { + pcrReadData := cpu.pcr_rep.bits + state := state_tx + } + } + + val scr_addr = addr(log2Up(nSCR)-1, 0) + val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} + for (i <- 0 until scr_rdata.size) + scr_rdata(i) := io.scr.rdata(i) + scr_rdata(0) := UInt(nTiles) + scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + + io.scr.wen := Bool(false) + io.scr.wdata := pcr_wdata + io.scr.waddr := scr_addr.toUInt + when (state === state_pcr_req && pcr_coreid === SInt(-1)) { + io.scr.wen := cmd === cmd_writecr + pcrReadData := scr_rdata(scr_addr) + state := state_tx + } + + val tx_cmd = Mux(nack, cmd_nack, cmd_ack) + val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) + val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) + val tx_data = Mux(tx_word_count === UInt(0), tx_header, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, pcrReadData, + packet_ram(packet_ram_raddr))) + + io.host.in.ready := state === state_rx + io.host.out.valid := state === state_tx + io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) +} From f13d76628b122d2dfc3950aa0761bcfd0f7394fc Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 7 Nov 2013 15:42:10 -0800 Subject: [PATCH 205/688] forgot to put htif into uncore package --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 38fce75e..ec905012 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,4 +1,4 @@ -package rocket +package uncore import Chisel._ import Node._ From 056bb156cab81319750c7de53a55af57dfee7599 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 20 Nov 2013 16:43:55 -0800 Subject: [PATCH 206/688] make CacheConstants an object --- uncore/src/main/scala/consts.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index e9d282ae..bbe6cbd5 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -54,6 +54,7 @@ trait AddressConstants { val PERM_BITS = 6; } +object CacheConstants extends CacheConstants trait CacheConstants { val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) From acc0d2b06c6007993552cf949fcd3af809b5ce54 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 25 Nov 2013 04:34:16 -0800 Subject: [PATCH 207/688] Only use LSBs for HTIF control regs For now, at least... --- uncore/src/main/scala/htif.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index ec905012..cd5c7534 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -196,6 +196,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati io.mem.release.meta.valid := Bool(false) io.mem.release.data.valid := Bool(false) + val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nTiles) { val my_reset = Reg(init=Bool(true)) @@ -203,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -223,7 +224,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (state === state_pcr_req && me && pcr_addr === pcr_reset) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From 4f1213cb8b638067702500c1d2c274595c690543 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 13 Jan 2014 21:45:14 -0800 Subject: [PATCH 208/688] Fix Scala integer overflow --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cd5c7534..fdd1162c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -244,7 +244,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt(((REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata From 3e634aef1d3afedacc4035248d81bc742377aff5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 22 Jan 2014 18:20:36 -0800 Subject: [PATCH 209/688] Fix HTIF for cache line sizes other than 64 B --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index fdd1162c..4e894af8 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = 576 + val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits From bbf801023039b406f5c6ad2ce952441fcf29d644 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Jan 2014 12:20:55 -0800 Subject: [PATCH 210/688] cleanups supporting uncore hierarchy --- uncore/src/main/scala/coherence.scala | 244 ++++++++++++++------------ uncore/src/main/scala/consts.scala | 17 +- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 25 +-- uncore/src/main/scala/tilelink.scala | 192 +++++++++++++------- uncore/src/main/scala/uncore.scala | 8 +- 6 files changed, 284 insertions(+), 206 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index dcb80c8e..e0db85f1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,10 +1,6 @@ package uncore import Chisel._ -abstract trait CoherenceAgentRole -abstract trait ClientCoherenceAgent extends CoherenceAgentRole -abstract trait MasterCoherenceAgent extends CoherenceAgentRole - abstract class CoherencePolicy { def nClientStates: Int def nMasterStates: Int @@ -31,41 +27,42 @@ abstract class CoherencePolicy { def newStateOnWriteback(): UInt def newStateOnFlush(): UInt def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): Bits + def newStateOnProbe(incoming: Probe, state: UInt): UInt def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt def getProbeType(a_type: UInt, global_state: UInt): UInt - def getReleaseTypeOnCacheControl(cmd: UInt): Bits - def getReleaseTypeOnVoluntaryWriteback(): Bits - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits - def getGrantType(a_type: UInt, count: UInt): Bits - def getGrantType(rel: Release, count: UInt): Bits + def getReleaseTypeOnCacheControl(cmd: UInt): UInt + def getReleaseTypeOnVoluntaryWriteback(): UInt + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getGrantType(a_type: UInt, count: UInt): UInt + def getGrantType(rel: Release, count: UInt): UInt def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def needsAckReply(a_type: UInt, global_state: UInt): Bool - def needsSelfProbe(acq: Acquire): Bool - def requiresAck(grant: Grant): Bool - def requiresAck(release: Release): Bool + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool + def requiresSelfProbe(a_type: UInt): Bool + def requiresAckForGrant(g_type: UInt): Bool + def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - def uSIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) + def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { - def getUncachedReadAcquireType: Bits - def getUncachedWriteAcquireType: Bits - def getUncachedReadWordAcquireType: Bits - def getUncachedWriteWordAcquireType: Bits - def getUncachedAtomicAcquireType: Bits + def getUncachedReadAcquireType: UInt + def getUncachedWriteAcquireType: UInt + def getUncachedReadWordAcquireType: UInt + def getUncachedWriteWordAcquireType: UInt + def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool } @@ -73,18 +70,19 @@ abstract class CoherencePolicyWithUncached extends CoherencePolicy with Uncached abstract class IncoherentPolicy extends CoherencePolicy { // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): Bits = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = Bits(0) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): Bits = Bits(0) - def getGrantType(rel: Release, count: UInt): Bits = Bits(0) + def newStateOnProbe(incoming: Probe, state: UInt): UInt = state + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) + def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) + def getGrantType(rel: Release, count: UInt): UInt = UInt(0) def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsAckReply(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) - def requiresAck(grant: Grant) = Bool(true) - def requiresAck(release: Release) = Bool(false) + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g_type: UInt) = Bool(true) + def requiresAckForRelease(r_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) } @@ -135,17 +133,17 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = releaseVoluntaryInvalidateData + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) } class MICoherence extends CoherencePolicyWithUncached { @@ -202,7 +200,7 @@ class MICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeCopy -> state @@ -220,9 +218,9 @@ class MICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -235,19 +233,19 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -258,7 +256,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -281,12 +279,15 @@ class MICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -350,7 +351,7 @@ class MEICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileExclusiveClean, @@ -373,9 +374,9 @@ class MEICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -390,19 +391,19 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, @@ -413,7 +414,7 @@ class MEICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -438,12 +439,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -514,7 +518,7 @@ class MSICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -537,9 +541,9 @@ class MSICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -554,19 +558,19 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -577,7 +581,7 @@ class MSICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -599,12 +603,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -675,7 +682,7 @@ class MESICoherence extends CoherencePolicyWithUncached { grantAtomicUncached -> tileInvalid )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeDowngrade -> tileShared, @@ -698,9 +705,9 @@ class MESICoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -715,19 +722,19 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, @@ -738,7 +745,7 @@ class MESICoherence extends CoherencePolicyWithUncached { acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -763,13 +770,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -795,7 +805,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uSIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -807,8 +817,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uSIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), + M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) )) } def needsWriteback (state: UInt): Bool = { @@ -846,7 +856,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadShared -> tileMigratoryClean)) )) } - def newStateOnProbe(incoming: Probe, state: UInt): Bits = { + def newStateOnProbe(incoming: Probe, state: UInt): UInt = { MuxLookup(incoming.p_type, state, Array( probeInvalidate -> tileInvalid, probeInvalidateOthers -> tileInvalid, @@ -875,11 +885,11 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } - def getReleaseTypeOnCacheControl(cmd: UInt): Bits = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): Bits = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): Bits = { + def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO + def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) + def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uSIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -894,19 +904,19 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uSIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uSIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uSIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) + case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { - uSIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) + uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) } - def messageIsUncached(acq: Acquire): Bool = uSIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) - def isCoherenceConflict(addr1: Bits, addr2: Bits): Bool = (addr1 === addr2) + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): Bits = { + def getGrantType(a_type: UInt, count: UInt): UInt = { MuxLookup(a_type, grantReadUncached, Array( acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, @@ -918,7 +928,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): Bits = { + def getGrantType(rel: Release, count: UInt): UInt = { MuxLookup(rel.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) @@ -944,12 +954,16 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def needsAckReply(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { + needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) + } + + def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) } - def requiresAck(grant: Grant) = grant.g_type != grantVoluntaryAck - def requiresAck(release: Release) = Bool(false) - def needsSelfProbe(acq: Acquire) = Bool(false) + def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck + def requiresAckForRelease(r_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index bbe6cbd5..d4f7130c 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -6,14 +6,15 @@ import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { - val MT_X = Bits("b???", 3); - val MT_B = Bits("b000", 3); - val MT_H = Bits("b001", 3); - val MT_W = Bits("b010", 3); - val MT_D = Bits("b011", 3); - val MT_BU = Bits("b100", 3); - val MT_HU = Bits("b101", 3); - val MT_WU = Bits("b110", 3); + val MT_SZ = 3 + val MT_X = Bits("b???") + val MT_B = Bits("b000") + val MT_H = Bits("b001") + val MT_W = Bits("b010") + val MT_D = Bits("b011") + val MT_BU = Bits("b100") + val MT_HU = Bits("b101") + val MT_WU = Bits("b110") val M_SZ = 5 val M_X = Bits("b?????"); diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4e894af8..608ac051 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module with ClientCoherenceAgent +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -120,7 +120,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAck(io.mem.grant.bits.payload) + mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 4905ffde..613a3b65 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -68,11 +68,13 @@ class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, c when (io.out.meta.fire()) { last_grant := meta_chosen } } -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} - } +} + +class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new PairedCrossbarIO(mType, dType) val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) @@ -111,11 +113,12 @@ class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfi abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new Bundle { +class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} - } +} +class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { + val io = new BasicCrossbarIO(dType) val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) @@ -133,14 +136,12 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical } } -case class LogicalNetworkConfiguration(nEndpoints: Int, idBits: Int, nMasters: Int, nClients: Int) - -abstract class LogicalNetwork[TileLinkType <: Bundle](endpoints: Seq[CoherenceAgentRole])(implicit conf: LogicalNetworkConfiguration) extends Module { - override val io: Vec[TileLinkType] - val physicalNetworks: Seq[PhysicalNetwork] - require(endpoints.length == conf.nEndpoints) +case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { + val nEndpoints = nMasters + nClients } +abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module + class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val src = UInt(width = conf.idBits) val dst = UInt(width = conf.idBits) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 425cbccd..cd50213f 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -40,20 +40,25 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.write_mask := write_mask acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } + def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = new Acquire + acq := a + acq + } } class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { - val a_type = Bits(width = conf.co.acquireTypeWidth) + val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) @@ -61,13 +66,29 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Probe +{ + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val prb = new Probe + prb.p_type := p_type + prb.addr := addr + prb.master_xact_id := master_xact_id + prb + } +} class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = Bits(width = conf.co.probeTypeWidth) + val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: Bits, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + val rel = new Release + rel.r_type := r_type + rel.addr := addr + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -77,100 +98,141 @@ object Release } } class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { - val r_type = Bits(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = conf.co.releaseTypeWidth) } class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = Bits(width = conf.co.grantTypeWidth) + val g_type = UInt(width = conf.co.grantTypeWidth) } class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId -trait DirectionalIO -trait ClientSourcedIO extends DirectionalIO -trait MasterSourcedIO extends DirectionalIO -class ClientSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with ClientSourcedIO { - override def clone = { new ClientSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class ClientSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with ClientSourcedIO { - override def clone = { new ClientSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} -class MasterSourcedFIFOIO[T <: Data](dType: T) extends DecoupledIO(dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedFIFOIO(dType).asInstanceOf[this.type] } -} -class MasterSourcedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends PairedDataIO(mType, dType) with MasterSourcedIO { - flip() - override def clone = { new MasterSourcedDataIO(mType, dType).asInstanceOf[this.type] } -} class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new ClientSourcedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) - val grant = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Grant)) - val grant_ack = new ClientSourcedFIFOIO(new LogicalNetworkIO(new GrantAck)) + val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { - val probe = new MasterSourcedFIFOIO(new LogicalNetworkIO(new Probe)) - val release = new ClientSourcedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends Module { - def acquireClientXactId(in: Acquire, id: Int): Bits - def grantClientXactId(in: Grant): Bits - def arbIdx(in: Grant): UInt - +abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) + + type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId + type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData + + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def masterSourcedClientXactId(in: MasterSourcedWithId): Bits + def arbIdx(in: MasterSourcedWithId): UInt + + def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] + (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], + out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, + arbN, REFILL_CYCLES, hasData _)) + out <> arb.io.out + ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + arb.data <> req.data + arb.meta.valid := req.meta.valid + arb.meta.bits := req.meta.bits + arb.meta.bits.payload := req.meta.bits.payload + arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) + req.meta.ready := arb.meta.ready + }} + } + + def hookupMasterSource[M <: MasterSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { + out.ready := Bool(false) + for (i <- 0 until arbN) { + ins(i).valid := Bool(false) + when (arbIdx(out.bits.payload) === UInt(i)) { + ins(i).valid := out.valid + out.ready := ins(i).ready + } + ins(i).bits := out.bits + ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + } + } +} + +abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - def acqHasData(acq: LogicalNetworkIO[Acquire]) = co.messageHasData(acq.payload) - val acq_arb = Module(new PairedLockingRRArbiter(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData), n, REFILL_CYCLES, acqHasData _)) - io.out.acquire <> acq_arb.io.out - io.in.map(_.acquire).zipWithIndex.zip(acq_arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload.client_xact_id := acquireClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready - }} + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupMasterSource(io.in.map(_.grant), io.out.grant) val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) io.out.grant_ack <> grant_ack_arb.io.out grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } +} - io.out.grant.ready := Bool(false) - for (i <- 0 until n) { - io.in(i).grant.valid := Bool(false) - when (arbIdx(io.out.grant.bits.payload) === UInt(i)) { - io.in(i).grant.valid := io.out.grant.valid - io.out.grant.ready := io.in(i).grant.ready - } - io.in(i).grant.bits := io.out.grant.bits - io.in(i).grant.bits.payload.client_xact_id := grantClientXactId(io.out.grant.bits.payload) +abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { + val io = new Bundle { + val in = Vec.fill(n){new TileLinkIO}.flip + val out = new TileLinkIO } + + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupMasterSource(io.in.map(_.grant), io.out.grant) + + io.in.map{ _.probe.valid := io.out.probe.valid } + io.in.map{ _.probe.bits := io.out.probe.bits } + io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) + + val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) + io.out.grant_ack <> grant_ack_arb.io.out + grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(n))) - def grantClientXactId(in: Grant) = in.client_xact_id >> UInt(log2Up(n)) - def arbIdx(in: Grant) = in.client_xact_id(log2Up(n)-1,0).toUInt +abstract trait AppendsArbiterId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id >> UInt(log2Up(arbN)) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id(log2Up(arbN)-1,0).toUInt } -class UncachedTileLinkIOArbiterThatPassesId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = in.client_xact_id - def grantClientXactId(in: Grant) = in.client_xact_id - def arbIdx(in: Grant): UInt = in.client_xact_id +abstract trait PassesId { + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + in.client_xact_id + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } -class UncachedTileLinkIOArbiterThatUsesNewId(n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) { - def acquireClientXactId(in: Acquire, id: Int) = UInt(id, log2Up(n)) - def grantClientXactId(in: Grant) = UInt(0) // DNC - def arbIdx(in: Grant) = in.client_xact_id +abstract trait UsesNewId { + val arbN: Int + def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + UInt(id, log2Up(arbN)) + def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + UInt(0) + def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + in.client_xact_id } + +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index a24f309d..8b410639 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module with MasterCoherenceAgent { +abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { val io = new Bundle { val client = (new TileLinkIO).flip val master = new UncachedTileLinkIO @@ -279,13 +279,13 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen } . elsewhen (x_needs_read) { doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) } . otherwise { - state := Mux(co.needsAckReply(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle)) + state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, + Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) } } is(s_ack) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAck(io.client.grant.bits.payload), s_busy, s_idle) } + when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { From 02dbd6b0aaaf0967ea26afbc2e4d32d12882d1ac Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 12 Feb 2014 18:39:40 -0800 Subject: [PATCH 211/688] Don't assign to your own inputs --- uncore/src/main/scala/uncore.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8b410639..67dfde85 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -127,7 +127,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe io.client.grant.bits.payload.data := UInt(0) io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { @@ -212,7 +211,6 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen io.client.release.meta.ready := Bool(false) io.client.release.data.ready := Bool(false) io.master.grant.ready := io.client.grant.ready - io.client.grant_ack.valid := Bool(false) switch (state) { is(s_idle) { From dec98eb047db3d5184961e7297b3728fe7be80a9 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Tue, 18 Mar 2014 11:37:08 -0700 Subject: [PATCH 212/688] Update library dependencies (jenkins builds) Add chisel as an explicit library dependency --- uncore/build.sbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/build.sbt b/uncore/build.sbt index 8794797d..565e5fc7 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,3 +5,5 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" + +libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From a228dbfa0d76237258026ff08f76375edef6524b Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 20 Mar 2014 10:40:05 -0700 Subject: [PATCH 213/688] Revert "Update library dependencies (jenkins builds)" This reverts commit e7a12143d0bfe8b3b4a4dcc78d119a89553ade8c. --- uncore/build.sbt | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 565e5fc7..8794797d 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -5,5 +5,3 @@ version := "2.0" name := "uncore" scalaVersion := "2.10.2" - -libraryDependencies += "edu.berkeley.cs" %% "chisel" % "latest.release" From fbca7c6bb35914ac0c3f043090d4e8d407306ae2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 29 Mar 2014 10:53:49 -0700 Subject: [PATCH 214/688] refactor ioMem and associcated constants. merge Aqcuire and AcquireData --- uncore/src/main/scala/coherence.scala | 17 -- uncore/src/main/scala/consts.scala | 6 - uncore/src/main/scala/htif.scala | 56 +++-- uncore/src/main/scala/llc.scala | 72 +++--- uncore/src/main/scala/memserdes.scala | 141 +++++++++--- uncore/src/main/scala/network.scala | 113 +-------- uncore/src/main/scala/tilelink.scala | 89 ++++--- uncore/src/main/scala/uncore.scala | 320 +++++++++++--------------- 8 files changed, 384 insertions(+), 430 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index e0db85f1..c9d0f4eb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -48,7 +48,6 @@ abstract class CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool def needsOuterWrite(a_type: UInt, global_state: UInt): Bool def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -79,7 +78,6 @@ abstract class IncoherentPolicy extends CoherencePolicy { def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def requiresAckForGrant(g_type: UInt) = Bool(true) def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -282,9 +280,6 @@ class MICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -442,9 +437,6 @@ class MEICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -606,9 +598,6 @@ class MSICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -773,9 +762,6 @@ class MESICoherence extends CoherencePolicyWithUncached { def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -958,9 +944,6 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) } - def requiresDatalessGrant(a_type: UInt, global_state: UInt): Bool = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached ||a_type === acquireInvalidateOthers) - } def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index d4f7130c..b0cdcdd6 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -70,9 +70,3 @@ trait TileLinkSizeConstants { trait MemoryInterfaceConstants extends CacheConstants with AddressConstants -{ - val MEM_TAG_BITS = 5 - val MEM_DATA_BITS = 128 - val REFILL_CYCLES = CACHE_DATA_SIZE_IN_BYTES*8/MEM_DATA_BITS - val MEM_ADDR_BITS = PADDR_BITS - OFFSET_BITS -} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 608ac051..c4d0ffb0 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -56,7 +56,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + MEM_DATA_BITS*REFILL_CYCLES + val long_request_bits = short_request_bits + conf.dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -124,26 +124,23 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati } io.mem.grant.ready := Bool(true) - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_req :: state_mem_wdata :: state_mem_wresp :: state_mem_rdata :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) val state = Reg(init=state_rx) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { - state := Mux(rx_cmd === cmd_readmem || rx_cmd === cmd_writemem, state_mem_req, + state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, + Mux(rx_cmd === cmd_writemem, state_mem_wreq, Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, - state_tx)) + state_tx))) } - val mem_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val x_init = Module(new Queue(new Acquire, 1)) - when (state === state_mem_req && x_init.io.enq.ready) { - state := Mux(cmd === cmd_writemem, state_mem_wdata, state_mem_rdata) + val acq_q = Module(new Queue(new Acquire, 1)) + when (state === state_mem_wreq && acq_q.io.enq.ready) { + state := state_mem_wresp } - when (state === state_mem_wdata && io.mem.acquire.data.ready) { - when (mem_cnt.andR) { - state := state_mem_wresp - } - mem_cnt := mem_cnt + UInt(1) + when (state === state_mem_rreq && acq_q.io.enq.ready) { + state := state_mem_rresp } when (state === state_mem_wresp) { when (mem_acked) { @@ -151,12 +148,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_acked := Bool(false) } } - when (state === state_mem_rdata) { + when (state === state_mem_rresp) { when (io.mem.grant.valid) { - when (mem_cnt.andR) { - state := state_mem_finish - } - mem_cnt := mem_cnt + UInt(1) + state := state_mem_finish } mem_acked := Bool(false) } @@ -170,31 +164,33 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati rx_count := UInt(0) tx_count := UInt(0) } - state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_req, state_rx) + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } var mem_req_data: Bits = null - for (i <- 0 until MEM_DATA_BITS/short_request_bits) { - val idx = Cat(mem_cnt, UInt(i, log2Up(MEM_DATA_BITS/short_request_bits))) - when (state === state_mem_rdata && io.mem.grant.valid) { + for (i <- 0 until conf.dataBits/short_request_bits) { + val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - x_init.io.enq.valid := state === state_mem_req + acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) - x_init.io.enq.bits := Mux(cmd === cmd_writemem, + acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) - io.mem.acquire.meta <> FIFOedLogicalNetworkIOWrapper(x_init.io.deq, UInt(conf.ln.nClients), UInt(0)) // By convention HTIF is the client with the largest id - io.mem.acquire.data.valid := state === state_mem_wdata - io.mem.acquire.data.bits.payload.data := mem_req_data + io.mem.acquire.valid := acq_q.io.deq.valid + acq_q.io.deq.ready := io.mem.acquire.ready + io.mem.acquire.bits.payload := acq_q.io.deq.bits + io.mem.acquire.bits.payload.data := mem_req_data + io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid io.mem.grant_ack.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) - io.mem.release.meta.valid := Bool(false) - io.mem.release.data.valid := Bool(false) + io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) @@ -244,7 +240,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(REFILL_CYCLES*MEM_DATA_BITS/8) << x_init.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 6627527a..1bfa4da7 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,42 +72,42 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int) extends MemReqCmd +class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] } -class LLCTagReq(ways: Int) extends HasMemAddr +class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, MEM_ADDR_BITS - log2Up(sets)) + val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) - val mem = new ioMemPipe + val mem = new MemPipeIO val mem_resp_set = UInt(OUTPUT, log2Up(sets)) val mem_resp_way = UInt(OUTPUT, log2Up(ways)) } class MSHR extends Bundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() - val refillCount = UInt(width = log2Up(REFILL_CYCLES)) + val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = MEM_ADDR_BITS - log2Up(sets)) + val old_tag = UInt(width = conf.addrBits - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -139,7 +139,7 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module val refillCount = mshr(refillId).refillCount when (io.mem.resp.valid) { mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(REFILL_CYCLES-1)) { mshr(refillId).refilled := Bool(true) } + when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } } val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) @@ -183,12 +183,12 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int) extends Module io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = MEM_ADDR_BITS)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new ioMemPipe + val mem = new MemPipeIO } val valid = Reg(init=Bool(false)) @@ -196,7 +196,7 @@ class LLCWriteback(requestors: Int) extends Module val addr = Reg(UInt()) val cmd_sent = Reg(Bool()) val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) var anyReq = Bool(false) for (i <- 0 until requestors) { @@ -216,7 +216,7 @@ class LLCWriteback(requestors: Int) extends Module when (io.mem.req_data.valid && io.mem.req_data.ready) { count := count + UInt(1) - when (count === UInt(REFILL_CYCLES-1)) { + when (count === UInt(refill_cycles-1)) { data_sent := Bool(true) when (cmd_sent) { valid := Bool(false) } } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int) extends Module io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = MEM_ADDR_BITS)) + val writeback = Decoupled(UInt(width = conf.addrBits)) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*REFILL_CYCLES, 1, latency-1, leaf)(Bits(width = MEM_DATA_BITS))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -254,12 +254,12 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul val qReady = q.io.count <= UInt(q.entries-latency-1) val valid = Reg(init=Bool(false)) val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) - val refillCount = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val count = Reg(init=UInt(0, log2Up(refill_cycles))) + val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) when (data.io.in.valid && !io.mem_resp.valid) { count := count + UInt(1) - when (valid && count === UInt(REFILL_CYCLES-1)) { valid := Bool(false) } + when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } } when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } @@ -300,11 +300,11 @@ class LLCData(latency: Int, sets: Int, ways: Int, leaf: Mem[UInt]) extends Modul io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int) extends Module // UNTESTED +class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = Vec.fill(n){new ioMem().flip} - val mem = new ioMem + val cpu = Vec.fill(n){new MemIO().flip} + val mem = new MemIO } val lock = Reg(init=Bool(false)) @@ -339,26 +339,26 @@ class MemReqArb(n: Int) extends Module // UNTESTED } io.mem.resp.ready := io.cpu(respWho).resp.ready - val unlock = Counter(io.mem.req_data.fire(), REFILL_CYCLES)._2 + val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends Module +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } - val tagWidth = MEM_ADDR_BITS - log2Up(sets) + val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding)) + val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, dataLeaf)) - val writeback = Module(new LLCWriteback(2)) + val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) + val writeback = Module(new LLCWriteback(2, refill_cycles)) val initCount = Reg(init=UInt(0, log2Up(sets+1))) val initialize = !initCount(log2Up(sets)) @@ -497,11 +497,11 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends Module +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val cpu = new ioMem().flip - val mem = new ioMemPipe + val cpu = new MemIO().flip + val mem = new MemPipeIO } val numEntries = numRequests * refillCycles diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 42611bc6..21a987fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,49 +2,56 @@ package uncore import Chisel._ import scala.math._ -trait HasMemData extends Bundle { - val data = Bits(width = MEM_DATA_BITS) +case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) + +abstract trait MemoryIFSubBundle extends Bundle { + implicit val conf: MemoryIFConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] } -trait HasMemAddr extends Bundle { - val addr = UInt(width = MEM_ADDR_BITS) +trait HasMemData extends MemoryIFSubBundle { + val data = Bits(width = conf.dataBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = MEM_TAG_BITS) +trait HasMemAddr extends MemoryIFSubBundle { + val addr = UInt(width = conf.addrBits) } -class MemReqCmd extends HasMemAddr with HasMemTag { +trait HasMemTag extends MemoryIFSubBundle { + val tag = UInt(width = conf.tagBits) +} + +class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag +class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag -class MemData extends HasMemData +class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData -class ioMem extends Bundle { +class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class ioMemPipe extends Bundle { +class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class ioMemSerialized(w: Int) extends Bundle +class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module { val io = new Bundle { - val wide = new ioMem().flip - val narrow = new ioMemSerialized(w) + val wide = new MemIO().flip + val narrow = new MemSerializedIO(w) } val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth @@ -56,7 +63,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -92,12 +99,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(REFILL_CYCLES-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -115,12 +122,12 @@ class MemSerdes(w: Int) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int) extends Bundle { - val narrow = new ioMemSerialized(w).flip - val wide = new ioMem +class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { + val narrow = new MemSerializedIO(w).flip + val wide = new MemIO } -class MemDesser(w: Int) extends Module // test rig side +class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth @@ -129,7 +136,7 @@ class MemDesser(w: Int) extends Module // test rig side require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(REFILL_CYCLES))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -157,13 +164,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(REFILL_CYCLES-1)) { + when (data_recv_cnt === UInt(conf.dataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -177,10 +184,92 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, REFILL_CYCLES)) + val dataq = Module(new Queue(new MemResp, conf.dataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) io.narrow.resp.valid := dataq.io.deq.valid io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemIO + } + + require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) + //require(tlconf.clientXactIdBits <= mifconf.tagBits) + + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + val cnt_max = mifconf.dataBeats + + val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val buf_out = Reg(Bits()) + val tag_out = Reg(Bits()) + val addr_out = Reg(Bits()) + val has_data = Reg(init=Bool(false)) + + val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) + val active_in = Reg(init=Bool(false)) + val buf_in = Reg(Bits()) + val tag_in = Reg(UInt(width = mifconf.tagBits)) + + // Decompose outgoing TL Acquires into MemIO cmd and data + when(!active_out && io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + cnt_out := UInt(0) + buf_out := io.uncached.acquire.bits.payload.data + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + } + when(active_out) { + when(mem_cmd_q.io.enq.fire()) { + cmd_sent_out := Bool(true) + } + when(mem_data_q.io.enq.fire()) { + cnt_out := cnt_out + UInt(1) + buf_out := buf_out >> UInt(mifconf.dataBits) + } + when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + active_out := Bool(false) + } + } + + io.uncached.acquire.ready := !active_out + mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) + mem_data_q.io.enq.bits.data := buf_out + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + + // Aggregate incoming MemIO responses into TL Grants + io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) + io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) + io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + when(!active_in && io.mem.resp.valid) { + active_in := Bool(true) + cnt_in := UInt(1) + buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + tag_in := io.mem.resp.bits.tag + } + when(active_in) { + when(io.uncached.grant.fire()) { + active_in := Bool(false) + } + when(io.mem.resp.fire()) { + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + cnt_in := cnt_in + UInt(1) + } + } +} + diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 613a3b65..6889bd7f 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,102 +1,5 @@ package uncore import Chisel._ -import scala.collection.mutable.Stack - -class PairedDataIO[M <: Data, D <: Data](mType: M, dType: D) extends Bundle { - val meta = Decoupled(mType) - val data = Decoupled(dType) - override def clone = { new PairedDataIO(mType, dType).asInstanceOf[this.type] } -} - -class PairedArbiterIO[M <: Data, D <: Data](mType: M, dType: D, n: Int) extends Bundle { - val in = Vec.fill(n){new PairedDataIO(mType, dType)}.flip - val out = new PairedDataIO(mType, dType) - val meta_chosen = UInt(OUTPUT, log2Up(n)) - val data_chosen = UInt(OUTPUT, log2Up(n)) - override def clone = { new PairedArbiterIO(mType, dType, n).asInstanceOf[this.type] } -} - -class PairedLockingRRArbiter[M <: Data, D <: Data](mType: M, dType: D, n: Int, count: Int, needsLock: Option[M => Bool] = None) extends Module { - require(isPow2(count)) - val io = new PairedArbiterIO(mType, dType, n) - val locked = if(count > 1) Reg(init=Bool(false)) else Bool(false) - val lockIdx = if(count > 1) Reg(init=UInt(n-1)) else UInt(n-1) - val grant = List.fill(n)(Bool()) - val meta_chosen = Bits(width = log2Up(n)) - - val chosen_meta_has_data = needsLock.map(_(io.in(meta_chosen).meta.bits)).getOrElse(Bool(true)) - val valid_meta_has_data = io.in(meta_chosen).meta.valid && chosen_meta_has_data - val grant_chosen_meta = !(locked && chosen_meta_has_data) - (0 until n).map(i => io.in(i).meta.ready := grant(i) && grant_chosen_meta && io.out.meta.ready) - (0 until n).map(i => io.in(i).data.ready := Mux(locked, lockIdx === UInt(i), grant(i) && valid_meta_has_data) && io.out.data.ready) - io.out.meta.valid := io.in(meta_chosen).meta.valid && grant_chosen_meta - io.out.data.valid := Mux(locked, io.in(lockIdx).data.valid, io.in(meta_chosen).data.valid && valid_meta_has_data) - io.out.meta.bits := io.in(meta_chosen).meta.bits - io.out.data.bits := Mux(locked, io.in(lockIdx).data.bits, io.in(meta_chosen).data.bits) - io.meta_chosen := meta_chosen - io.data_chosen := Mux(locked, lockIdx, meta_chosen) - - if(count > 1){ - val cnt = Reg(init=UInt(0, width = log2Up(count))) - val cnt_next = cnt + UInt(1) - when(io.out.data.fire()){ - cnt := cnt_next - when(cnt_next === UInt(0)) { - locked := Bool(false) - } - } - when(io.out.meta.fire()) { - when(needsLock.map(_(io.out.meta.bits)).getOrElse(Bool(true))) { - when(!locked) { - locked := Bool(true) - lockIdx := Vec(io.in.map{in => in.meta.fire()}).indexWhere{i: Bool => i} - } - } - } - } - val last_grant = Reg(init=Bits(0, log2Up(n))) - val ctrl = ArbiterCtrl((0 until n).map(i => io.in(i).meta.valid && UInt(i) > last_grant) ++ io.in.map(_.meta.valid)) - (0 until n).map(i => grant(i) := ctrl(i) && UInt(i) > last_grant || ctrl(i + n)) - - var choose = Bits(n-1) - for (i <- n-2 to 0 by -1) - choose = Mux(io.in(i).meta.valid, Bits(i), choose) - for (i <- n-1 to 1 by -1) - choose = Mux(io.in(i).meta.valid && UInt(i) > last_grant, Bits(i), choose) - meta_chosen := choose - - when (io.out.meta.fire()) { last_grant := meta_chosen } -} - -class PairedCrossbarIO[M <: Data, D <: Data](mType: M, dType: D)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){new PairedDataIO(new PhysicalNetworkIO(mType),new PhysicalNetworkIO(dType))} -} - -class PairedCrossbar[M <: Data, D <: Data](mType: M, dType: D, count: Int, needsLock: Option[PhysicalNetworkIO[M] => Bool] = None)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new PairedCrossbarIO(mType, dType) - - val metaRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val dataRdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints){Bool()}) - val rdyVecs = metaRdyVecs zip dataRdyVecs - - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new PairedLockingRRArbiter(io.in(0).meta.bits.clone, io.in(0).data.bits.clone, conf.nEndpoints, count, needsLock)) - rrarb.io.in zip io.in zip rdys._1 zip rdys._2 map { case (((arb, in), meta_rdy), data_rdy) => { - arb.meta.valid := in.meta.valid && (in.meta.bits.header.dst === UInt(i)) - arb.meta.bits := in.meta.bits - meta_rdy := arb.meta.ready && (in.meta.bits.header.dst === UInt(i)) - arb.data.valid := in.data.valid && (in.data.bits.header.dst === UInt(i)) - arb.data.bits := in.data.bits - data_rdy := arb.data.ready && (in.data.bits.header.dst === UInt(i)) - }} - out <> rrarb.io.out - }} - for(i <- 0 until conf.nEndpoints) { - io.in(i).meta.ready := rdyVecs.map(r => r._1(i)).reduceLeft(_||_) - io.in(i).data.ready := rdyVecs.map(r => r._2(i)).reduceLeft(_||_) - } -} case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) @@ -147,7 +50,13 @@ class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { val dst = UInt(width = conf.idBits) } -object FIFOedLogicalNetworkIOWrapper { +class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { + val header = new LogicalHeader + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } +} + +object DecoupledLogicalNetworkIOWrapper { def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid @@ -159,7 +68,7 @@ object FIFOedLogicalNetworkIOWrapper { } } -object FIFOedLogicalNetworkIOUnwrapper { +object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid @@ -168,9 +77,3 @@ object FIFOedLogicalNetworkIOUnwrapper { out } } - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val header = new LogicalHeader - val payload = dType.clone - override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } -} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index cd50213f..6f877856 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -35,18 +35,24 @@ object Acquire acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id + acq.data := Bits(0) acq.write_mask := Bits(0) acq.subword_addr := Bits(0) acq.atomic_opcode := Bits(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { val acq = apply(a_type, addr, client_xact_id) + acq.data := data + acq + } + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt)(implicit conf: TileLinkConfiguration): Acquire = { - val acq = apply(a_type, addr, client_xact_id) + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq @@ -57,15 +63,17 @@ object Acquire acq } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId { + +class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) } -class AcquireData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData - object Probe { def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -76,16 +84,20 @@ object Probe prb } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { + +class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasPhysicalAddress + with HasMasterTransactionId { val p_type = UInt(width = conf.co.probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { val rel = new Release rel.r_type := r_type rel.addr := addr + rel.data := data rel } def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { @@ -94,16 +106,45 @@ object Release rel.addr := addr rel.client_xact_id := client_xact_id rel.master_xact_id := master_xact_id + rel.data := UInt(0) + rel + } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + val rel = apply(r_type, addr, client_xact_id, master_xact_id) + rel.data := data rel } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId { + +class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasMasterTransactionId + with HasTileLinkData { val r_type = UInt(width = conf.co.releaseTypeWidth) } -class ReleaseData(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasTileLinkData +object Grant +{ + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + val gnt = new Grant + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := UInt(0) + gnt + } + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + val gnt = apply(g_type, client_xact_id, master_xact_id) + gnt.data := data + gnt + } +} -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { +class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage + with HasTileLinkData + with HasClientTransactionId + with HasMasterTransactionId { val g_type = UInt(width = conf.co.grantTypeWidth) } @@ -112,7 +153,7 @@ class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMe class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln - val acquire = new PairedDataIO(new LogicalNetworkIO(new Acquire), new LogicalNetworkIO(new AcquireData)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } @@ -120,7 +161,7 @@ class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new PairedDataIO(new LogicalNetworkIO(new Release), new LogicalNetworkIO(new ReleaseData)) + val release = new DecoupledIO(new LogicalNetworkIO(new Release)) override def clone = { new TileLinkIO().asInstanceOf[this.type] } } @@ -129,32 +170,28 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId - type ClientSourcedWithData = ClientSourcedMessage with HasTileLinkData def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def masterSourcedClientXactId(in: MasterSourcedWithId): Bits def arbIdx(in: MasterSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId, D <: ClientSourcedWithData] - (ins: Seq[PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]], - out: PairedDataIO[LogicalNetworkIO[M],LogicalNetworkIO[D]]) { + def hookupClientSource[M <: ClientSourcedWithId] + (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], + out: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) - val arb = Module(new PairedLockingRRArbiter(out.meta.bits.clone, out.data.bits.clone, - arbN, REFILL_CYCLES, hasData _)) + val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { - arb.data <> req.data - arb.meta.valid := req.meta.valid - arb.meta.bits := req.meta.bits - arb.meta.bits.payload := req.meta.bits.payload - arb.meta.bits.payload.client_xact_id := clientSourcedClientXactId(req.meta.bits.payload, id) - req.meta.ready := arb.meta.ready + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready }} } def hookupMasterSource[M <: MasterSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) for (i <- 0 until arbN) { ins(i).valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 67dfde85..3e162583 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -31,16 +31,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - alloc_arb.io.in(i).valid := t.acquire.meta.ready - t.acquire.meta.bits := acquire.meta.bits - t.acquire.meta.valid := alloc_arb.io.in(i).ready - - t.acquire.data.bits := acquire.data.bits - t.acquire.data.valid := acquire.data.valid + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.meta.ready := trackerList.map(_.io.client.acquire.meta.ready).reduce(_||_) && !block_acquires - acquire.data.ready := trackerList.map(_.io.client.acquire.data.ready).reduce(_||_) - alloc_arb.io.out.ready := acquire.meta.valid && !block_acquires + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -49,21 +45,18 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration // Handle releases, which might be voluntary and might have data val release = io.client.release - val voluntary = co.isVoluntary(release.meta.bits.payload) + val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.meta.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.client - t.release.meta.bits := release.meta.bits - t.release.meta.valid := release.meta.valid && (release_idx === UInt(i)) && !block_releases - t.release.data.bits := release.data.bits - t.release.data.valid := release.data.valid + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.meta.ready := Vec(trackerList.map(_.io.client.release.meta.ready)).read(release_idx) && !block_releases - release.data.ready := trackerList.map(_.io.client.release.data.ready).reduce(_||_) + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) @@ -83,7 +76,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module with OuterRequestGenerator { +abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -92,63 +85,56 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_needs_write = Reg(init=Bool(false)) - val mem_cmd_sent = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) io.master.grant.ready := Bool(false) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := cmd_to_write - //TODO io.master.acquire.bits.header.dst - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - //TODO io.master.acquire_data.bits.header.dst - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) io.client.probe.valid := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) // DNC + io.client.release.ready := Bool(false) io.client.grant.valid := Bool(false) - io.client.grant.bits.payload.g_type := co.getGrantType(xact, UInt(0)) - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.payload.data := UInt(0) - io.client.grant.bits.header.dst := init_client_id_ io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.meta.ready := Bool(true) - when( io.client.release.meta.valid ) { - xact := io.client.release.meta.bits.payload - init_client_id_ := io.client.release.meta.bits.header.src - release_data_needs_write := co.messageHasData(io.client.release.meta.bits.payload) - mem_cnt := UInt(0) - mem_cmd_sent := Bool(false) - state := s_mem + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - mem_cmd_sent, - init_client_id_) - } . otherwise { state := s_ack } + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } } is(s_ack) { io.client.grant.valid := Bool(true) @@ -158,172 +144,138 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { - val s_idle :: s_ack :: s_mem :: s_probe :: s_busy :: Nil = Enum(UInt(), 5) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_data_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt_ = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt_) + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) val probe_flags = Reg(init=Bits(0, width = ln.nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val x_needs_read = Reg(init=Bool(false)) - val acquire_data_needs_write = Reg(init=Bool(false)) - val release_data_needs_write = Reg(init=Bool(false)) - val cmd_to_write = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId)) - val cmd_to_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val a_w_mem_cmd_sent = Reg(init=Bool(false)) - val r_w_mem_cmd_sent = Reg(init=Bool(false)) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val probe_initial_flags = Bits(width = ln.nClients) probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(io.client.acquire.meta.bits.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, io.client.acquire.meta.bits.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, io.client.release.meta.bits.payload.addr) && (state != s_idle) - io.master.acquire.meta.valid := Bool(false) - io.master.acquire.meta.bits.payload := Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - io.master.acquire.meta.bits.header.src := UInt(bankId) - io.master.acquire.data.valid := Bool(false) - io.master.acquire.data.bits.payload.data := UInt(0) - io.master.acquire.data.bits.header.src := UInt(bankId) - io.client.probe.valid := Bool(false) - io.client.probe.bits.payload.p_type := co.getProbeType(xact.a_type, UInt(0)) - io.client.probe.bits.payload.master_xact_id := UInt(trackerId) - io.client.probe.bits.payload.addr := xact.addr - io.client.probe.bits.header.dst := UInt(0) - io.client.probe.bits.header.src := UInt(bankId) - io.client.grant.bits.payload.data := io.master.grant.bits.payload.data - io.client.grant.bits.payload.g_type := grant_type - io.client.grant.bits.payload.client_xact_id := xact.client_xact_id - io.client.grant.bits.payload.master_xact_id := UInt(trackerId) - io.client.grant.bits.header.dst := init_client_id_ - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.valid := (io.master.grant.valid && (UInt(trackerId) === io.master.grant.bits.payload.client_xact_id)) - io.client.acquire.meta.ready := Bool(false) - io.client.acquire.data.ready := Bool(false) - io.client.release.meta.ready := Bool(false) - io.client.release.data.ready := Bool(false) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read io.master.grant.ready := io.client.grant.ready + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + switch (state) { is(s_idle) { - io.client.acquire.meta.ready := Bool(true) - when( io.client.acquire.meta.valid ) { - xact := io.client.acquire.meta.bits.payload - init_client_id_ := io.client.acquire.meta.bits.header.src - init_sharer_cnt_ := UInt(ln.nClients) // TODO: Broadcast only - acquire_data_needs_write := co.messageHasData(io.client.acquire.meta.bits.payload) - x_needs_read := co.needsOuterRead(io.client.acquire.meta.bits.payload.a_type, UInt(0)) + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - mem_cnt := UInt(0) - r_w_mem_cmd_sent := Bool(false) - a_w_mem_cmd_sent := Bool(false) if(ln.nClients > 1) { release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, s_mem) - } else state := s_mem + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) } } is(s_probe) { - when(probe_flags.orR) { - io.client.probe.valid := Bool(true) - io.client.probe.bits.header.dst := curr_p_id - } + // Generate probes + io.client.probe.valid := probe_flags.orR when(io.client.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } - io.client.release.meta.ready := Bool(true) - when(io.client.release.meta.valid) { - if(ln.nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := s_mem - } - when( co.messageHasData(io.client.release.meta.bits.payload)) { - release_data_needs_write := Bool(true) - release_data_client_id := io.client.release.meta.bits.header.src + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } - is(s_mem) { - when (release_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.release.data, - cmd_to_write, - release_data_needs_write, - r_w_mem_cmd_sent, - release_data_client_id) - } . elsewhen(acquire_data_needs_write) { - doOuterReqWrite(io.master.acquire, - io.client.acquire.data, - cmd_to_write, - acquire_data_needs_write, - a_w_mem_cmd_sent, - init_client_id_) - } . elsewhen (x_needs_read) { - doOuterReqRead(io.master.acquire, cmd_to_read, x_needs_read) - } . otherwise { - state := Mux(co.requiresDatalessGrant(xact.a_type, UInt(0)), s_ack, - Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle)) + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_ack) { + is(s_make_grant) { io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := Mux(co.requiresAckForGrant(io.client.grant.bits.payload.g_type), s_busy, s_idle) } + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when (io.client.grant_ack.valid && io.client.grant_ack.bits.payload.master_xact_id === UInt(trackerId)) { + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } } } - -abstract trait OuterRequestGenerator { - val mem_cnt = Reg(init=UInt(0, width = log2Up(REFILL_CYCLES))) - val mem_cnt_next = mem_cnt + UInt(1) - - def doOuterReqWrite[T <: HasTileLinkData](master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], client_data: DecoupledIO[LogicalNetworkIO[T]], cmd: Acquire, trigger: Bool, cmd_sent: Bool, desired_client_data_src_id: UInt) { - val do_write = client_data.valid && (client_data.bits.header.src === desired_client_data_src_id) - master_acq.meta.bits.payload := cmd - master_acq.data.bits.payload := client_data.bits.payload - when(master_acq.meta.fire()) { - cmd_sent := Bool(true) - } - when (do_write) { - master_acq.meta.valid := !cmd_sent - when (master_acq.meta.ready || cmd_sent) { - master_acq.data.valid := client_data.valid - when(master_acq.data.ready) { - client_data.ready:= Bool(true) - mem_cnt := mem_cnt_next - when(mem_cnt === UInt(REFILL_CYCLES-1)) { - trigger := Bool(false) - } - } - } - } - } - - def doOuterReqRead(master_acq: PairedDataIO[LogicalNetworkIO[Acquire],LogicalNetworkIO[AcquireData]], cmd: Acquire, trigger: Bool) { - master_acq.meta.valid := Bool(true) - master_acq.meta.bits.payload := cmd - when(master_acq.meta.ready) { - trigger := Bool(false) - } - } -} From b1df49ba30a9aed51527bba52afc6e151707ce2d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 1 Apr 2014 17:14:45 -0700 Subject: [PATCH 215/688] removed AddressSpaceConstants, CacheConstants, and TileLinkSizeConstants --- uncore/src/main/scala/consts.scala | 28 --------------------------- uncore/src/main/scala/htif.scala | 10 +++++----- uncore/src/main/scala/memserdes.scala | 5 +++++ uncore/src/main/scala/package.scala | 5 +---- uncore/src/main/scala/tilelink.scala | 10 +++++----- 5 files changed, 16 insertions(+), 42 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index b0cdcdd6..64d6c3a7 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -2,7 +2,6 @@ package uncore package constants import Chisel._ -import scala.math.max object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { @@ -43,30 +42,3 @@ trait MemoryOpConstants { def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } - -object AddressConstants extends AddressConstants -trait AddressConstants { - val PADDR_BITS = 32 - val VADDR_BITS = 43; - val PGIDX_BITS = 13; - val PPN_BITS = PADDR_BITS-PGIDX_BITS; - val VPN_BITS = VADDR_BITS-PGIDX_BITS; - val ASID_BITS = 7; - val PERM_BITS = 6; -} - -object CacheConstants extends CacheConstants -trait CacheConstants { - val CACHE_DATA_SIZE_IN_BYTES = 1 << 6 - val OFFSET_BITS = log2Up(CACHE_DATA_SIZE_IN_BYTES) -} - -trait TileLinkSizeConstants { - val ACQUIRE_WRITE_MASK_BITS = 6 - val ACQUIRE_SUBWORD_ADDR_BITS = 3 - val ACQUIRE_ATOMIC_OP_BITS = 4 -} - -trait MemoryInterfaceConstants extends - CacheConstants with - AddressConstants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index c4d0ffb0..9ea26f86 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,7 +41,7 @@ class SCRIO(n: Int) extends Bundle val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module { implicit val (ln, co) = (conf.ln, conf.co) val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client @@ -75,7 +75,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (rx_count === UInt(short_request_bits/w-1)) { cmd := next_cmd size := rx_shifter_in(15,4) - pos := rx_shifter_in(15,4+OFFSET_BITS-3) + pos := rx_shifter_in(15,4+offsetBits-3) seqno := rx_shifter_in(23,16) addr := rx_shifter_in(63,24) } @@ -95,7 +95,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) - val bad_mem_packet = size(OFFSET_BITS-1-3,0).orR || addr(OFFSET_BITS-1-3,0).orR + val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), Bool(true))) @@ -157,7 +157,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati when (state === state_mem_finish && io.mem.grant_ack.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) - addr := addr + UInt(1 << OFFSET_BITS-3) + addr := addr + UInt(1 << offsetBits-3) } when (state === state_tx && tx_done) { when (tx_word_count === tx_size) { @@ -176,7 +176,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int)(implicit conf: TileLinkConfigurati mem_req_data = Cat(packet_ram(idx), mem_req_data) } acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq - val init_addr = addr.toUInt >> UInt(OFFSET_BITS-3) + val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 21a987fd..dea3e801 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,6 +2,11 @@ package uncore import Chisel._ import scala.math._ +case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { + val ppnBits = paddrBits - pgIdxBits + val vpnBits = vaddrBits - pgIdxBits +} + case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) abstract trait MemoryIFSubBundle extends Bundle { diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 4ac83d0b..501d511d 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,7 +1,4 @@ -package object uncore extends - uncore.constants.MemoryOpConstants with - uncore.constants.TileLinkSizeConstants with - uncore.constants.MemoryInterfaceConstants +package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6f877856..451c5844 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int) +case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { implicit val conf: TileLinkConfiguration @@ -9,7 +9,7 @@ abstract trait TileLinkSubBundle extends Bundle { } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = PADDR_BITS - OFFSET_BITS) + val addr = UInt(width = conf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { @@ -69,9 +69,9 @@ class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMes with HasClientTransactionId with HasTileLinkData { val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = ACQUIRE_WRITE_MASK_BITS) - val subword_addr = Bits(width = ACQUIRE_SUBWORD_ADDR_BITS) - val atomic_opcode = Bits(width = ACQUIRE_ATOMIC_OP_BITS) + val write_mask = Bits(width = conf.writeMaskBits) + val subword_addr = Bits(width = conf.wordAddrBits) + val atomic_opcode = Bits(width = conf.atomicOpBits) } object Probe From bf2ff7804eda45a6a690a32a11c39cafe9481539 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 17 Apr 2014 17:01:40 -0700 Subject: [PATCH 216/688] Add chisel-dependent.sbt for -DchiselVersion="latest.release" If -DchiselVersion is specified on the command line, add the appropriate chisel library to libraryDependencies. --- uncore/chisel-dependent.sbt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uncore/chisel-dependent.sbt diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt new file mode 100644 index 00000000..fa8d5490 --- /dev/null +++ b/uncore/chisel-dependent.sbt @@ -0,0 +1,6 @@ +// Provide a managed dependency on chisel if -DchiselVersion="" is +// supplied on the command line. + +val chiselVersion = System.getProperty("chiselVersion", "None") + +libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From 5613dc7d1ba5278cc9ee00e59242f2624c9fea5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 16 Apr 2014 18:14:12 -0700 Subject: [PATCH 217/688] replaced Lists with Vecs --- uncore/src/main/scala/coherence.scala | 107 ++++++++++++-------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c9d0f4eb..327e31e5 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -52,8 +52,6 @@ abstract class CoherencePolicy { def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool - - def uIntListContains(list: List[UInt], elem: UInt): Bool = list.map(elem === _).reduceLeft(_||_) } trait UncachedTransactions { @@ -96,10 +94,9 @@ class ThreeStateIncoherence extends IncoherentPolicy { val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List() - val hasDataAcquireTypeList = List(acquireWriteback) - val hasDataReleaseTypeList = List(acquireWriteback) - val hasDataGrantTypeList = List(grantData) + val hasDataAcquireTypeVec = Vec(acquireWriteback) + val hasDataReleaseTypeVec = Vec(acquireWriteback) + val hasDataGrantTypeVec = Vec(grantData) def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) def isValid (state: UInt): Bool = state != tileInvalid @@ -135,13 +132,13 @@ class ThreeStateIncoherence extends IncoherentPolicy { def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) case rel: Release => Bool(false) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = Bool(false) } class MICoherence extends CoherencePolicyWithUncached { @@ -160,10 +157,10 @@ class MICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -231,15 +228,15 @@ class MICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -302,10 +299,10 @@ class MEICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid def isValid (state: UInt): Bool = state != tileInvalid @@ -386,15 +383,15 @@ class MEICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -460,10 +457,10 @@ class MSICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), @@ -550,15 +547,15 @@ class MSICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -621,10 +618,10 @@ class MESICoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) def isHit (cmd: UInt, state: UInt): Bool = { Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), @@ -711,15 +708,15 @@ class MESICoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } def messageUpdatesDataArray (reply: Grant): Bool = { (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) @@ -785,13 +782,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeList = List(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeList = List(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeList = List(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) - val hasDataReleaseTypeList = List(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), uIntListContains(List(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty), state), (state != tileInvalid)) + Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) } def isValid (state: UInt): Bool = { state != tileInvalid @@ -803,8 +800,8 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state), - M_CLN -> uIntListContains(List(tileExclusiveDirty,tileMigratoryDirty),state) + M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), + M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) )) } def needsWriteback (state: UInt): Bool = { @@ -875,7 +872,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(uIntListContains(List(tileExclusiveDirty, tileMigratoryDirty), state), + probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData @@ -890,15 +887,13 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => uIntListContains(hasDataAcquireTypeList, acq.a_type) - case grant: Grant => uIntListContains(hasDataGrantTypeList, grant.g_type) - case rel: Release => uIntListContains(hasDataReleaseTypeList, rel.r_type) + case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) + case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - uIntListContains(List(grantReadShared, grantReadExclusive, grantReadMigratory), reply.g_type) - } - def messageIsUncached(acq: Acquire): Bool = uIntListContains(uncachedAcquireTypeList, acq.a_type) + def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) + def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) From 39681303b83e275c6bfcf22146e6a98720d1920a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 22 Apr 2014 16:55:35 -0700 Subject: [PATCH 218/688] beginning of l2 cache --- uncore/src/main/scala/cache.scala | 431 ++++++++++++++++++++++++++++++ 1 file changed, 431 insertions(+) create mode 100644 uncore/src/main/scala/cache.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala new file mode 100644 index 00000000..a4c2755c --- /dev/null +++ b/uncore/src/main/scala/cache.scala @@ -0,0 +1,431 @@ +package uncore +import Chisel._ + +abstract class CacheConfig { + def sets: Int + def ways: Int + def tl: TileLinkConfiguration + def as: AddressSpaceConfiguration + def dm: Boolean + def states: Int + def lines: Int + def tagbits: Int + def idxbits: Int + def offbits: Int + def untagbits: Int + def rowbits: Int + def metabits: Int + def statebits: Int +} + +case class L2CacheConfig(val sets: Int, val ways: Int, + nReleaseTransactions: Int, nAcquireTransactions: Int, + nrpq: Int, nsdq: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) extends CacheConfig { + def databits = tl.dataBits + def states = tl.co.nMasterStates + def lines = sets*ways + def dm = ways == 1 + def offbits = 0 + def lineaddrbits = tl.addrBits + def idxbits = log2Up(sets) + def waybits = log2Up(ways) + def untagbits = offbits + idxbits + def tagbits = lineaddrbits - idxbits + def databytes = databits/8 + def wordoffbits = log2Up(databytes) + def rowbits = tl.dataBits + def rowbytes = rowbits/8 + def rowoffbits = log2Up(rowbytes) + def refillcycles = tl.dataBits/(rowbits) + def statebits = log2Up(states) + def metabits = statebits + tagbits + + require(states > 0) + require(isPow2(sets)) + require(isPow2(ways)) // TODO: relax this + require(rowbits == tl.dataBits) //TODO: relax this? +} + +abstract trait CacheBundle extends Bundle { + implicit val conf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract trait L2CacheBundle extends Bundle { + implicit val conf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +} + +abstract class ReplacementPolicy { + def way: UInt + def miss: Unit + def hit: Unit +} + +class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { + private val replace = Bool() + replace := Bool(false) + val lfsr = LFSR16(replace) + + def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def miss = replace := Bool(true) + def hit = {} +} + +object L2MetaData { + def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + val meta = new L2MetaData + meta.state := state + meta.tag := tag + meta + } +} +class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { + val state = UInt(width = conf.statebits) + val tag = Bits(width = conf.tagbits) +} + +class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { + val idx = Bits(width = conf.idxbits) +} + +class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { + val way_en = Bits(width = conf.ways) + val data = new L2MetaData() +} + +class L2MetaDataArray(implicit conf: CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + } + + val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) + val rst = rst_cnt < UInt(conf.sets) + when (rst) { rst_cnt := rst_cnt+UInt(1) } + + val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + + when (rst || io.write.valid) { + val addr = Mux(rst, rst_cnt, io.write.bits.idx) + val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val mask = Mux(rst, SInt(-1), io.write.bits.way_en) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + } + val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + + for (w <- 0 until conf.ways) { + val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) + io.resp(w).state := m >> UInt(conf.tagbits) + io.resp(w).tag := m + } + + io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM + io.write.ready := !rst +} + +class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = conf.ways) + val addr = Bits(width = conf.untagbits) +} + +class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { + val wmask = Bits(width = conf.tl.writeMaskBits) + val data = Bits(width = conf.rowbits) +} + +class L2DataArray(implicit conf: L2CacheConfig) extends Module { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + } + + val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) + val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) + + val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + for (w <- 0 until conf.ways) { + val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) +{ + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + + // Create SHRs for outstanding transactions + val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions + val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ + (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.client.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe request generation + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.client.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + + // Handle releases, which might be voluntary and might have data + val release = io.client.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.client + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.client.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + + // Free finished transactions + val ack = io.client.grant_ack + trackerList.map(_.io.client.grant_ack.valid := ack.valid) + trackerList.map(_.io.client.grant_ack.bits := ack.bits) + ack.ready := Bool(true) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + io.master <> outer_arb.io.out +} + + +abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new TileLinkIO).flip + val master = new UncachedTileLinkIO + val tile_incoherent = Bits(INPUT, ln.nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + } + + val c_acq = io.client.acquire.bits + val c_rel = io.client.release.bits + val c_gnt = io.client.grant.bits + val c_ack = io.client.grant_ack.bits + val m_gnt = io.master.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val incoming_rel = io.client.release.bits + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + (state != s_idle) + + io.master.grant.ready := Bool(false) + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data) + io.client.acquire.ready := Bool(false) + io.client.probe.valid := Bool(false) + io.client.release.ready := Bool(false) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + xact.client_xact_id, + UInt(trackerId)) + + switch (state) { + is(s_idle) { + io.client.release.ready := Bool(true) + when( io.client.release.valid ) { + xact := incoming_rel.payload + init_client_id := incoming_rel.header.src + state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + } + } + is(s_mem) { + io.master.acquire.valid := Bool(true) + when(io.master.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + //TODO: Will need id reg for merged release xacts + + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) + val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data) + val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data) + val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + + val probe_initial_flags = Bits(width = ln.nClients) + probe_initial_flags := Bits(0) + if (ln.nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.master.acquire.valid := Bool(false) + io.master.acquire.bits.header.src := UInt(bankId) + //io.master.acquire.bits.header.dst TODO + io.master.acquire.bits.payload := outer_read + io.master.grant.ready := io.client.grant.ready + + io.client.probe.valid := Bool(false) + io.client.probe.bits.header.src := UInt(bankId) + io.client.probe.bits.header.dst := curr_p_id + io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + io.client.grant.valid := Bool(false) + io.client.grant.bits.header.src := UInt(bankId) + io.client.grant.bits.header.dst := init_client_id + io.client.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.client.acquire.ready := Bool(false) + io.client.release.ready := Bool(false) + + switch (state) { + is(s_idle) { + io.client.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + when( io.client.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + probe_flags := probe_initial_flags + if(ln.nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + is(s_probe) { + // Generate probes + io.client.probe.valid := probe_flags.orR + when(io.client.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.client.release.valid) { + when(co.messageHasData(c_rel.payload)) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_rel + when(io.master.acquire.ready) { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.client.release.ready := Bool(true) + if(ln.nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_read + when(io.master.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.master.acquire.valid := Bool(true) + io.master.acquire.bits.payload := outer_write_acq + when(io.master.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.client.grant.valid := Bool(true) + when(io.client.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.client.grant.valid := Bool(true) + } + when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From f8f29c69b8ad5c619cbe849c69f0eda690ba70dc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 23 Apr 2014 16:24:20 -0700 Subject: [PATCH 219/688] MetaData & friends moved to uncore/ --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a4c2755c..bd677c08 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -74,34 +74,34 @@ class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolic def hit = {} } -object L2MetaData { +object MetaData { def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { - val meta = new L2MetaData + val meta = new MetaData meta.state := state meta.tag := tag meta } } -class L2MetaData(implicit val conf: CacheConfig) extends CacheBundle { +class MetaData(implicit val conf: CacheConfig) extends CacheBundle { val state = UInt(width = conf.statebits) val tag = Bits(width = conf.tagbits) } -class L2MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { +class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { val idx = Bits(width = conf.idxbits) } -class L2MetaWriteReq(implicit conf: CacheConfig) extends L2MetaReadReq()(conf) { +class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { val way_en = Bits(width = conf.ways) - val data = new L2MetaData() + val data = new MetaData() } -class L2MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new L2MetaData).asOutput} + val read = Decoupled(new MetaReadReq).flip + val write = Decoupled(new MetaWriteReq).flip + val resp = Vec.fill(conf.ways){(new MetaData).asOutput} } val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) From 3f53d532c25c2039735a3ee2f86f263ce54b4eb1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 14:58:38 -0700 Subject: [PATCH 220/688] uniquify tilelink conf val name for easier subtyping --- uncore/src/main/scala/tilelink.scala | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 451c5844..22941d29 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,24 +4,24 @@ import Chisel._ case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) abstract trait TileLinkSubBundle extends Bundle { - implicit val conf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val tlconf: TileLinkConfiguration + override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] } trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = tlconf.addrBits) } trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = conf.clientXactIdBits) + val client_xact_id = Bits(width = tlconf.clientXactIdBits) } trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = conf.masterXactIdBits) + val master_xact_id = Bits(width = tlconf.masterXactIdBits) } trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = conf.dataBits) + val data = Bits(width = tlconf.dataBits) } trait SourcedMessage extends Bundle @@ -64,14 +64,14 @@ object Acquire } } -class Acquire(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = conf.co.acquireTypeWidth) - val write_mask = Bits(width = conf.writeMaskBits) - val subword_addr = Bits(width = conf.wordAddrBits) - val atomic_opcode = Bits(width = conf.atomicOpBits) + val a_type = UInt(width = tlconf.co.acquireTypeWidth) + val write_mask = Bits(width = tlconf.writeMaskBits) + val subword_addr = Bits(width = tlconf.wordAddrBits) + val atomic_opcode = Bits(width = tlconf.atomicOpBits) } object Probe @@ -85,10 +85,10 @@ object Probe } } -class Probe(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = conf.co.probeTypeWidth) + val p_type = UInt(width = tlconf.co.probeTypeWidth) } object Release @@ -116,12 +116,12 @@ object Release } } -class Release(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage +class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = conf.co.releaseTypeWidth) + val r_type = UInt(width = tlconf.co.releaseTypeWidth) } object Grant @@ -141,14 +141,14 @@ object Grant } } -class Grant(implicit val conf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = conf.co.grantTypeWidth) + val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val conf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { From 1163131d1e44804e065a8edea3065111745f535f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 15:17:05 -0700 Subject: [PATCH 221/688] TileLinkIO.GrantAck -> TileLinkIO.Finish --- uncore/src/main/scala/cache.scala | 10 +++++----- uncore/src/main/scala/htif.scala | 8 ++++---- uncore/src/main/scala/tilelink.scala | 16 ++++++++-------- uncore/src/main/scala/uncore.scala | 10 +++++----- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bd677c08..3af6f29f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,9 +214,9 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -239,7 +239,7 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -423,7 +423,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 9ea26f86..41d6df28 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -154,7 +154,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } mem_acked := Bool(false) } - when (state === state_mem_finish && io.mem.grant_ack.ready) { + when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -186,9 +186,9 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module - io.mem.grant_ack.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.grant_ack.bits.payload.master_xact_id := mem_gxid - io.mem.grant_ack.bits.header.dst := mem_gsrc + io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack + io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 22941d29..68da8213 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -148,14 +148,14 @@ class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMes val g_type = UInt(width = tlconf.co.grantTypeWidth) } -class GrantAck(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { implicit val ln = conf.ln val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val grant_ack = new DecoupledIO(new LogicalNetworkIO(new GrantAck)) + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } @@ -214,9 +214,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupMasterSource(io.in.map(_.grant), io.out.grant) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { @@ -233,9 +233,9 @@ abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) e io.in.map{ _.probe.bits := io.out.probe.bits } io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - val grant_ack_arb = Module(new RRArbiter(new LogicalNetworkIO(new GrantAck), n)) - io.out.grant_ack <> grant_ack_arb.io.out - grant_ack_arb.io.in zip io.in map { case (arb, req) => arb <> req.grant_ack } + val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) + io.out.finish <> finish_arb.io.out + finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } abstract trait AppendsArbiterId { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3e162583..0a3b7d6e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -64,9 +64,9 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } // Free finished transactions - val ack = io.client.grant_ack - trackerList.map(_.io.client.grant_ack.valid := ack.valid) - trackerList.map(_.io.client.grant_ack.bits := ack.bits) + val ack = io.client.finish + trackerList.map(_.io.client.finish.valid := ack.valid) + trackerList.map(_.io.client.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port @@ -89,7 +89,7 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten val c_acq = io.client.acquire.bits val c_rel = io.client.release.bits val c_gnt = io.client.grant.bits - val c_ack = io.client.grant_ack.bits + val c_ack = io.client.finish.bits val m_gnt = io.master.grant.bits } @@ -273,7 +273,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgen when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.client.grant.valid := Bool(true) } - when(io.client.grant_ack.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 52c6de56414b9da67957deb258d7bc0bcb3f41c5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 26 Apr 2014 19:11:36 -0700 Subject: [PATCH 222/688] DRAMSideLLCLike trait. TSHRFile. New L2 config objects. --- uncore/src/main/scala/cache.scala | 114 ++++++++++++++++++++--------- uncore/src/main/scala/llc.scala | 14 ++-- uncore/src/main/scala/uncore.scala | 31 +++++--- 3 files changed, 106 insertions(+), 53 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3af6f29f..3d42e97d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -abstract class CacheConfig { +trait CacheConfig { def sets: Int def ways: Int def tl: TileLinkConfiguration @@ -18,12 +18,16 @@ abstract class CacheConfig { def statebits: Int } -case class L2CacheConfig(val sets: Int, val ways: Int, - nReleaseTransactions: Int, nAcquireTransactions: Int, - nrpq: Int, nsdq: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) extends CacheConfig { - def databits = tl.dataBits +case class L2CacheConfig( + val sets: Int, val ways: Int, + val nrpq: Int, val nsdq: Int, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int, + val tl: TileLinkConfiguration, + val as: AddressSpaceConfiguration) + extends CoherenceAgentConfiguration + with CacheConfig +{ def states = tl.co.nMasterStates def lines = sets*ways def dm = ways == 1 @@ -33,8 +37,9 @@ case class L2CacheConfig(val sets: Int, val ways: Int, def waybits = log2Up(ways) def untagbits = offbits + idxbits def tagbits = lineaddrbits - idxbits - def databytes = databits/8 - def wordoffbits = log2Up(databytes) + def wordbits = 64 + def wordbytes = wordbits/8 + def wordoffbits = log2Up(wordbytes) def rowbits = tl.dataBits def rowbytes = rowbits/8 def rowoffbits = log2Up(rowbytes) @@ -45,17 +50,17 @@ case class L2CacheConfig(val sets: Int, val ways: Int, require(states > 0) require(isPow2(sets)) require(isPow2(ways)) // TODO: relax this - require(rowbits == tl.dataBits) //TODO: relax this? + require(refillcycles == 1) //TODO: relax this? } abstract trait CacheBundle extends Bundle { - implicit val conf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val cacheconf: CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] } abstract trait L2CacheBundle extends Bundle { - implicit val conf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] + implicit val l2cacheconf: L2CacheConfig + override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } abstract class ReplacementPolicy { @@ -64,31 +69,31 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val conf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(conf.dm) UInt(0) else lfsr(log2Up(conf.ways)-1,0) + def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) def miss = replace := Bool(true) def hit = {} } object MetaData { - def apply(tag: Bits, state: UInt)(implicit conf: CacheConfig) = { + def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { val meta = new MetaData meta.state := state meta.tag := tag meta } } -class MetaData(implicit val conf: CacheConfig) extends CacheBundle { - val state = UInt(width = conf.statebits) - val tag = Bits(width = conf.tagbits) +class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { + val state = UInt(width = cacheconf.statebits) + val tag = Bits(width = cacheconf.tagbits) } -class MetaReadReq(implicit val conf: CacheConfig) extends CacheBundle { - val idx = Bits(width = conf.idxbits) +class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { + val idx = Bits(width = cacheconf.idxbits) } class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { @@ -128,27 +133,26 @@ class MetaDataArray(implicit conf: CacheConfig) extends Module { io.write.ready := !rst } -class L2DataReadReq(implicit val conf: L2CacheConfig) extends L2CacheBundle { - val way_en = Bits(width = conf.ways) - val addr = Bits(width = conf.untagbits) +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + val way_en = Bits(width = l2cacheconf.ways) + val addr = Bits(width = l2cacheconf.tl.addrBits) } class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.rowbits) + val data = Bits(width = conf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.rowbits)} + val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} } - val waddr = io.write.bits.addr >> UInt(conf.rowoffbits) - val raddr = io.read.bits.addr >> UInt(conf.rowoffbits) - - val wmask = FillInterleaved(conf.databits, io.write.bits.wmask) + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) for (w <- 0 until conf.ways) { val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { @@ -161,11 +165,51 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent()(conf.tl) -{ +trait InternalRequestState extends CacheBundle { + val tag_match = Bool() + val old_meta = new MetaData + val way_en = Bits(width = cacheconf.ways) +} + +class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) + with InternalRequestState + +class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) + with InternalRequestState + +class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { + implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) + val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) - // Create SHRs for outstanding transactions + val tshrfile = Module(new TSHRFile(bankId)) + + io.client <> tshrfile.io.client + io.master <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { + implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + val io = new Bundle { + val client = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val meta_read_req = Decoupled(new MetaReadReq) + val meta_write_req = Decoupled(new MetaWriteReq) + val data_read_req = Decoupled(new MetaReadReq) + val data_write_req = Decoupled(new MetaWriteReq) + } + + // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) @@ -222,7 +266,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + io.outer <> outer_arb.io.out } diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1bfa4da7..382a5f96 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -343,13 +343,15 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module -{ +abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } +} +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +{ val tagWidth = conf.addrBits - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -497,12 +499,8 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) extends Module -{ - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) + extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0a3b7d6e..cbcd39b6 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,18 +1,27 @@ package uncore import Chisel._ -abstract class CoherenceAgent(implicit conf: TileLinkConfiguration) extends Module { +trait CoherenceAgentConfiguration { + def tl: TileLinkConfiguration + def nReleaseTransactions: Int + def nAcquireTransactions: Int +} + +abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO - val incoherent = Vec.fill(conf.ln.nClients){Bool()}.asInput + val client = (new TileLinkIO()(conf.tl)).flip + val master = new UncachedTileLinkIO()(conf.tl) + val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration(tl: TileLinkConfiguration, nReleaseTransactions: Int, nAcquireTransactions: Int) +case class L2CoherenceAgentConfiguration( + val tl: TileLinkConfiguration, + val nReleaseTransactions: Int, + val nAcquireTransactions: Int) extends CoherenceAgentConfiguration -class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends CoherenceAgent()(conf.tl) -{ +class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends CoherenceAgent { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) // Create SHRs for outstanding transactions @@ -76,7 +85,7 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: L2CoherenceAgentConfiguration } -abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) extends Module { +abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { val client = (new TileLinkIO).flip @@ -94,7 +103,8 @@ abstract class XactTracker()(implicit conf: L2CoherenceAgentConfiguration) exten } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -143,7 +153,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Cohe } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CoherenceAgentConfiguration) extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) + extends XactTracker()(conf) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } From 023722992162c123fee90d2395c14d7c866a2dd6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 29 Apr 2014 16:49:18 -0700 Subject: [PATCH 223/688] client/master -> inner/outer --- uncore/src/main/scala/cache.scala | 160 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 160 ++++++++++++++--------------- 2 files changed, 160 insertions(+), 160 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3d42e97d..4eb28f97 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -191,8 +191,8 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) - io.client <> tshrfile.io.client - io.master <> tshrfile.io.outer + tshrfile.io.inner <> io.inner + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -200,7 +200,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new InternalTileLinkIO).flip + val inner = (new InternalTileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) @@ -218,27 +218,27 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -246,26 +246,26 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } @@ -273,18 +273,18 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new InternalTileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -293,46 +293,46 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -361,7 +361,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -369,37 +369,37 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -415,18 +415,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -434,7 +434,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -444,30 +444,30 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cbcd39b6..c8e33216 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -9,8 +9,8 @@ trait CoherenceAgentConfiguration { abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { val io = new Bundle { - val client = (new TileLinkIO()(conf.tl)).flip - val master = new UncachedTileLinkIO()(conf.tl) + val inner = (new TileLinkIO()(conf.tl)).flip + val outer = new UncachedTileLinkIO()(conf.tl) val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput } } @@ -33,27 +33,27 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.client.acquire + val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.client.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.client.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.probe } + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data - val release = io.client.release + val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) @@ -61,45 +61,45 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.client + val t = trackerList(i).io.inner t.release.bits := release.bits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } - release.ready := Vec(trackerList.map(_.io.client.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.client.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.client.grant } + io.inner.grant <> grant_arb.io.out + grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions - val ack = io.client.finish - trackerList.map(_.io.client.finish.valid := ack.valid) - trackerList.map(_.io.client.finish.bits := ack.bits) + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.master } - io.master <> outer_arb.io.out + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out } abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val client = (new TileLinkIO).flip - val master = new UncachedTileLinkIO + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } - val c_acq = io.client.acquire.bits - val c_rel = io.client.release.bits - val c_gnt = io.client.grant.bits - val c_ack = io.client.finish.bits - val m_gnt = io.master.grant.bits + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits } @@ -109,46 +109,46 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val incoming_rel = io.client.release.bits + val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && (state != s_idle) - io.master.grant.ready := Bool(false) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) - io.client.acquire.ready := Bool(false) - io.client.probe.valid := Bool(false) - io.client.release.ready := Bool(false) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), xact.client_xact_id, UInt(trackerId)) switch (state) { is(s_idle) { - io.client.release.ready := Bool(true) - when( io.client.release.valid ) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) } } is(s_mem) { - io.master.acquire.valid := Bool(true) - when(io.master.acquire.ready) { state := s_ack } + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { state := s_idle } + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } } } } @@ -178,7 +178,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC probe_initial_flags := Bits(0) if (ln.nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.client.acquire.bits.payload) + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -186,37 +186,37 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - io.master.acquire.valid := Bool(false) - io.master.acquire.bits.header.src := UInt(bankId) - //io.master.acquire.bits.header.dst TODO - io.master.acquire.bits.payload := outer_read - io.master.grant.ready := io.client.grant.ready + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + //io.outer.acquire.bits.header.dst TODO + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready - io.client.probe.valid := Bool(false) - io.client.probe.bits.header.src := UInt(bankId) - io.client.probe.bits.header.dst := curr_p_id - io.client.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), xact.addr, UInt(trackerId)) val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) - io.client.grant.valid := Bool(false) - io.client.grant.bits.header.src := UInt(bankId) - io.client.grant.bits.header.dst := init_client_id - io.client.grant.bits.payload := Grant(grant_type, + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) - io.client.acquire.ready := Bool(false) - io.client.release.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) switch (state) { is(s_idle) { - io.client.acquire.ready := Bool(true) + io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) - when( io.client.acquire.valid ) { + when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only @@ -232,18 +232,18 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } is(s_probe) { // Generate probes - io.client.probe.valid := probe_flags.orR - when(io.client.probe.ready) { + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) } // Handle releases, which may have data to be written back - when(io.client.release.valid) { + when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_rel - when(io.master.acquire.ready) { - io.client.release.ready := Bool(true) + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -251,7 +251,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } } .otherwise { - io.client.release.ready := Bool(true) + io.inner.release.ready := Bool(true) if(ln.nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -261,30 +261,30 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } is(s_mem_read) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_read - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_mem_write) { - io.master.acquire.valid := Bool(true) - io.master.acquire.bits.payload := outer_write_acq - when(io.master.acquire.ready) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } is(s_make_grant) { - io.client.grant.valid := Bool(true) - when(io.client.grant.ready) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.master.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.client.grant.valid := Bool(true) + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) } - when(io.client.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { state := s_idle } } From 45172f1f37eb5e0633bf9eb2cd700d3cd15979f5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 1 May 2014 01:44:59 -0700 Subject: [PATCH 224/688] parameterize metadataarray --- uncore/src/main/scala/cache.scala | 91 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4eb28f97..610317d2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -14,8 +14,6 @@ trait CacheConfig { def offbits: Int def untagbits: Int def rowbits: Int - def metabits: Int - def statebits: Int } case class L2CacheConfig( @@ -45,7 +43,6 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) - def metabits = statebits + tagbits require(states > 0) require(isPow2(sets)) @@ -79,54 +76,61 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -object MetaData { - def apply(tag: Bits, state: UInt)(implicit cacheconf: CacheConfig) = { - val meta = new MetaData - meta.state := state - meta.tag := tag - meta - } -} -class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { - val state = UInt(width = cacheconf.statebits) +abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) } +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq(implicit conf: CacheConfig) extends MetaReadReq()(conf) { +class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) - val data = new MetaData() + val data = gen.clone + override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl + def gen = resetMeta.clone val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq).flip - val resp = Vec.fill(conf.ways){(new MetaData).asOutput} + val write = Decoupled(new MetaWriteReq(gen)).flip + val resp = Vec.fill(conf.ways){gen.asOutput} } + val metabits = resetMeta.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = conf.metabits*conf.ways), conf.sets, seqRead = true) + val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Cat(Mux(rst, tl.co.newStateOnFlush, io.write.bits.data.state), io.write.bits.data.tag) + val data = Mux(rst, resetMeta, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(conf.metabits, mask)) + tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(conf.metabits*(w+1)-1, conf.metabits*w) - io.resp(w).state := m >> UInt(conf.tagbits) - io.resp(w).tag := m + val m = tag(metabits*(w+1)-1, metabits*w) + io.resp(w) := gen.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -165,23 +169,23 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -trait InternalRequestState extends CacheBundle { +trait L2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new MetaData - val way_en = Bits(width = cacheconf.ways) + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) } -class InternalAcquire(implicit val cacheconf: CacheConfig) extends Acquire()(cacheconf.tl) - with InternalRequestState +class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) + with L2InternalRequestState -class InternalRelease(implicit val cacheconf: CacheConfig) extends Release()(cacheconf.tl) - with InternalRequestState +class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) + with L2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new InternalAcquire)) + val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new InternalRelease)) + val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } @@ -191,6 +195,21 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) + // tags + val meta = Module(new MetaDataArray(new L2MetaData)) + + // data + val data = Module(new L2DataArray) + + // replacement policy + val replacer = new RandomReplacement +/* + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) +*/ + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -204,9 +223,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq) - val data_read_req = Decoupled(new MetaReadReq) - val data_write_req = Decoupled(new MetaWriteReq) + val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) + val data_read_req = Decoupled(new L2DataReadReq) + val data_write_req = Decoupled(new L2DataWriteReq) } // Create TSHRs for outstanding transactions From bc3ef1011eb7182d48f739bfff823772e36d2f62 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 May 2014 12:59:45 -0700 Subject: [PATCH 225/688] correct use of function value to initialize MetaDataArray --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 610317d2..e41e1cfb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -104,16 +104,16 @@ class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends Me override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) extends Module { +class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - def gen = resetMeta.clone + val rstVal = gen() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq(gen)).flip - val resp = Vec.fill(conf.ways){gen.asOutput} + val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip + val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = resetMeta.getWidth + val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) when (rst) { rst_cnt := rst_cnt+UInt(1) } @@ -122,7 +122,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext when (rst || io.write.valid) { val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, resetMeta, io.write.bits.data).toBits + val data = Mux(rst, rstVal, io.write.bits.data).toBits val mask = Mux(rst, SInt(-1), io.write.bits.way_en) tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) } @@ -130,7 +130,7 @@ class MetaDataArray[T <: MetaData](resetMeta: T)(implicit conf: CacheConfig) ext for (w <- 0 until conf.ways) { val m = tag(metabits*(w+1)-1, metabits*w) - io.resp(w) := gen.fromBits(m) + io.resp(w) := rstVal.clone.fromBits(m) } io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM @@ -196,7 +196,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA val tshrfile = Module(new TSHRFile(bankId)) // tags - val meta = Module(new MetaDataArray(new L2MetaData)) + val meta = Module(new MetaDataArray(() => new L2MetaData)) // data val data = Module(new L2DataArray) From 0e39346a126778862361b8adf43834f14e0fd3fd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 7 May 2014 01:51:46 -0700 Subject: [PATCH 226/688] L2-specific metadataarray wrapper, hookups to tshrfile --- uncore/src/main/scala/cache.scala | 212 +++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e41e1cfb..e5851a6b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -43,6 +43,7 @@ case class L2CacheConfig( def rowoffbits = log2Up(rowbytes) def refillcycles = tl.dataBits/(rowbits) def statebits = log2Up(states) + def nTSHRs = nReleaseTransactions + nAcquireTransactions require(states > 0) require(isPow2(sets)) @@ -60,6 +61,16 @@ abstract trait L2CacheBundle extends Bundle { override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] } +trait HasL2Id extends L2CacheBundle { + val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) +} + +trait HasL2InternalRequestState extends L2CacheBundle { + val tag_match = Bool() + val old_meta = new L2MetaData + val way_en = Bits(width = l2cacheconf.ways) +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -80,20 +91,6 @@ abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle val tag = Bits(width = cacheconf.tagbits) } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData - with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) -} - -/* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) -} -*/ - class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } @@ -137,49 +134,125 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { +object L2MetaData { + def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { + val meta = new L2MetaData + meta.tag := tag + meta.state := state + meta.sharers := sharers + meta + } +} +class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData + with L2CacheBundle { + val state = UInt(width = l2cacheconf.statebits) + val sharers = Bits(width = l2cacheconf.tl.ln.nClients) +} + +/* +class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { + val cstate = UInt(width = cacheconf.cstatebits) + val mstate = UInt(width = cacheconf.mstatebits) + val sharers = Bits(width = cacheconf.tl.ln.nClients) +} +*/ + +class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq + with HasL2Id { + val tag = Bits(width = l2cacheconf.tagbits) +} + +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) + with HasL2Id + +class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { + implicit val tl = conf.tl + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) + val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + + val replacer = new RandomReplacement + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.old_meta := Mux(s2_tag_match, + L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { val way_en = Bits(width = l2cacheconf.ways) val addr = Bits(width = l2cacheconf.tl.addrBits) } -class L2DataWriteReq(implicit conf: L2CacheConfig) extends L2DataReadReq()(conf) { - val wmask = Bits(width = conf.tl.writeMaskBits) - val data = Bits(width = conf.tl.dataBits) +class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { + val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) + val data = Bits(width = l2cacheconf.tl.dataBits) +} + +class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle + with HasL2Id { + val data = Bits(width = l2cacheconf.tl.dataBits) } class L2DataArray(implicit conf: L2CacheConfig) extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip - val resp = Vec.fill(conf.ways){Bits(OUTPUT, conf.tl.dataBits)} + val resp = Valid(new L2DataResp) } val waddr = io.write.bits.addr val raddr = io.read.bits.addr val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - for (w <- 0 until conf.ways) { + val resp = (0 until conf.ways).map { w => val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } - io.resp(w) := array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) io.read.ready := Bool(true) io.write.ready := Bool(true) } -trait L2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2MetaData - val way_en = Bits(width = l2cacheconf.ways) -} - class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with L2InternalRequestState + with HasL2InternalRequestState class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) @@ -194,23 +267,16 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - - // tags - val meta = Module(new MetaDataArray(() => new L2MetaData)) - - // data + val meta = Module(new L2MetaDataArray) val data = Module(new L2DataArray) - // replacement policy - val replacer = new RandomReplacement -/* - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) -*/ - tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -219,19 +285,31 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val incoherent = Vec.fill(ln.nClients){Bool()}.asInput - val meta_read_req = Decoupled(new MetaReadReq) - val meta_write_req = Decoupled(new MetaWriteReq(new L2MetaData)) - val data_read_req = Decoupled(new L2DataReadReq) - val data_write_req = Decoupled(new L2DataWriteReq) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, i) => a <> i } } // Create TSHRs for outstanding transactions val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new L2VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new L2AcquireTracker(id, bankId))) + val trackerList = (0 until conf.nReleaseTransactions).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId)) + } ++ (conf.nReleaseTransactions until nTrackers).map { id => + Module(new L2AcquireTracker(id, bankId)) + } // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -251,10 +329,11 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + //io.inner.probe <> probe_arb.io.out + //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -272,9 +351,10 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + //io.inner.grant <> grant_arb.io.out + //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -282,21 +362,35 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { trackerList.map(_.io.inner.finish.bits := ack.bits) ack.ready := Bool(true) - // Create an arbiter for the one memory port + // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + + } abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val io = new Bundle { - val inner = (new InternalTileLinkIO).flip + val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO val tile_incoherent = Bits(INPUT, ln.nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip } val c_acq = io.inner.acquire.bits From 364a6de2145dd3e81cd432cf31fa56bb177eef3f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 18 May 2014 19:26:35 -0700 Subject: [PATCH 227/688] Use Mem instead of Vec[Reg] --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 41d6df28..cc1c1c62 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -84,7 +84,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR val packet_ram_depth = long_request_bits/short_request_bits-1 - val packet_ram = Vec.fill(packet_ram_depth){Reg(Bits(width = short_request_bits))} + val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) when (rx_word_done && io.host.in.ready) { packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in } From 3c329df7e761295b4d8e1b8b7bd3a41b32630271 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 28 May 2014 13:35:08 -0700 Subject: [PATCH 228/688] refactor Metadata, clean and expand coherence API --- uncore/src/main/scala/cache.scala | 102 ++-- uncore/src/main/scala/coherence.scala | 807 ++++++++++++++------------ uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 6 +- 4 files changed, 502 insertions(+), 415 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e5851a6b..9a5afee3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -67,7 +67,7 @@ trait HasL2Id extends L2CacheBundle { trait HasL2InternalRequestState extends L2CacheBundle { val tag_match = Bool() - val old_meta = new L2MetaData + val old_meta = new L2Metadata val way_en = Bits(width = l2cacheconf.ways) } @@ -87,46 +87,45 @@ class RandomReplacement(implicit val cacheconf: CacheConfig) extends Replacement def hit = {} } -abstract class MetaData(implicit val cacheconf: CacheConfig) extends CacheBundle { +abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { val tag = Bits(width = cacheconf.tagbits) + val coh: CoherenceMetadata } class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { val idx = Bits(width = cacheconf.idxbits) } -class MetaWriteReq[T <: MetaData](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { +class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { val way_en = Bits(width = conf.ways) val data = gen.clone override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] } -class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { implicit val tl = conf.tl - val rstVal = gen() + val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) val rst = rst_cnt < UInt(conf.sets) + val waddr = Mux(rst, rst_cnt, io.write.bits.idx) + val wdata = Mux(rst, rstVal, io.write.bits.data).toBits + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tags = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) - + val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) when (rst || io.write.valid) { - val addr = Mux(rst, rst_cnt, io.write.bits.idx) - val data = Mux(rst, rstVal, io.write.bits.data).toBits - val mask = Mux(rst, SInt(-1), io.write.bits.way_en) - tags.write(addr, Fill(conf.ways, data), FillInterleaved(metabits, mask)) + tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) } - val tag = tags(RegEnable(io.read.bits.idx, io.read.valid)) + val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) for (w <- 0 until conf.ways) { - val m = tag(metabits*(w+1)-1, metabits*w) + val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -134,26 +133,23 @@ class MetaDataArray[T <: MetaData](gen: () => T)(implicit conf: CacheConfig) ext io.write.ready := !rst } -object L2MetaData { - def apply(tag: Bits, state: UInt, sharers: UInt)(implicit conf: L2CacheConfig) = { - val meta = new L2MetaData +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + val meta = new L2Metadata meta.tag := tag - meta.state := state - meta.sharers := sharers + meta.coh := coh meta } } -class L2MetaData(implicit val l2cacheconf: L2CacheConfig) extends MetaData +class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata with L2CacheBundle { - val state = UInt(width = l2cacheconf.statebits) - val sharers = Bits(width = l2cacheconf.tl.ln.nClients) + val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone } /* -class L3MetaData(implicit conf: L3CacheConfig) extends MetaData()(conf) { - val cstate = UInt(width = cacheconf.cstatebits) - val mstate = UInt(width = cacheconf.mstatebits) - val sharers = Bits(width = cacheconf.tl.ln.nClients) +class L3Metadata(implicit conf: L3CacheConfig) extends Metadata + with L3CacheBundle { + val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? } */ @@ -162,14 +158,14 @@ class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq val tag = Bits(width = l2cacheconf.tagbits) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2MetaData](new L2MetaData) +class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { +class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { implicit val tl = conf.tl val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip @@ -177,7 +173,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val resp = Valid(new L2MetaResp) } - val meta = Module(new MetaDataArray(() => L2MetaData(UInt(0), tl.co.newStateOnFlush, UInt(0)))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -186,11 +182,11 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).state)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR - val s2_hit_state = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).state, s1_clk_en))) - val s2_hit_sharers = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).sharers, s1_clk_en))) + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = new RandomReplacement val s1_replaced_way_en = UIntToOH(replacer.way) @@ -202,7 +198,7 @@ class L2MetaDataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match io.resp.bits.old_meta := Mux(s2_tag_match, - L2MetaData(s2_repl_meta.tag, s2_hit_state, s2_hit_sharers), + L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } @@ -267,7 +263,7 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetaDataArray) + val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner @@ -300,7 +296,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, i) => a <> i } + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } } // Create TSHRs for outstanding transactions @@ -331,9 +332,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Handle probe requests doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - //val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - //io.inner.probe <> probe_arb.io.out - //probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Handle releases, which might be voluntary and might have data val release = io.inner.release @@ -352,9 +350,6 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - //val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) - //io.inner.grant <> grant_arb.io.out - //grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } // Free finished transactions val ack = io.inner.finish @@ -372,7 +367,8 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) } @@ -406,6 +402,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -430,15 +427,28 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2Ca xact.client_xact_id, UInt(trackerId)) + switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { xact := incoming_rel.payload init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + state := s_mem } } +/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits + old_meta := io.meta.resp.bits.old_meta + state := Mux(s_meta_write +Mux(co.messageHasData(xact), s_mem, s_ack) + } + */ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -463,7 +473,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -491,7 +501,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -511,7 +521,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 327e31e5..45a1970c 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,7 +1,86 @@ package uncore import Chisel._ -abstract class CoherencePolicy { +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + var res = default; + for ((t, v) <- mapping.reverse){ + res = Mux(t, v, res); + } + res + } +} + +abstract class CoherenceMetadata extends Bundle + +object ClientMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy) = { + val m = new ClientMetadata + m.state := state + m + } +} +class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata()(c).asInstanceOf[this.type] +} + +object MasterMetadata { + def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + val m = new MasterMetadata + m.state := state + m.sharers := m.sharers.flush() + m + } + def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + val m = apply(state) + m.sharers := sharers + m + } +} +class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { + val state = UInt(width = c.masterStateWidth) + val sharers = c.dir.clone + override def clone = new MasterMetadata()(c).asInstanceOf[this.type] +} +/* +class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { + val cstate = UInt(width = outer.clientStateWidth) + val mstate = UInt(width = inner.masterStateWidth) + val sharers = inner.dir.sharers.clone +} +*/ + +abstract class DirectoryRepresentation extends Bundle { + val sharers: UInt + def pop(id: UInt): DirectoryRepresentation + def push(id: UInt): DirectoryRepresentation + def flush(dummy: Int = 0): DirectoryRepresentation + def none(dummy: Int = 0): Bool + def one(dummy: Int = 0): Bool +} + +class NullRepresentation extends DirectoryRepresentation { + val sharers = UInt(0) + def pop(id: UInt) = this + def push(id: UInt) = this + def flush(dummy: Int = 0) = this + def none(dummy: Int = 0) = Bool(false) + def one(dummy: Int = 0) = Bool(false) +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation { + val sharers = UInt(width = nClients) + def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } + def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } + def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = sharers === UInt(0) + def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +} + +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -15,26 +94,29 @@ abstract class CoherencePolicy { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, state: UInt): Bool - def isValid (state: UInt): Bool + def isHit (cmd: UInt, m: ClientMetadata): Bool + def isValid (m: ClientMetadata): Bool + def isHit (incoming: Acquire, m: MasterMetadata): Bool + def isValid (m: MasterMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool - def needsWriteback (state: UInt): Bool + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool + def needsWriteback(m: ClientMetadata): Bool - def newStateOnHit(cmd: UInt, state: UInt): UInt - def newStateOnCacheControl(cmd: UInt): UInt - def newStateOnWriteback(): UInt - def newStateOnFlush(): UInt - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt - def newStateOnProbe(incoming: Probe, state: UInt): UInt + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata + def clientMetadataOnFlush: ClientMetadata + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata + def masterMetadataOnFlush: MasterMetadata + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, global_state: UInt): UInt + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt + def getProbeType(a_type: UInt, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt def getGrantType(a_type: UInt, count: UInt): UInt def getGrantType(rel: Release, count: UInt): UInt @@ -45,9 +127,10 @@ abstract class CoherencePolicy { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def needsOuterRead(a_type: UInt, global_state: UInt): Bool - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool + def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool + def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool @@ -63,85 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached extends CoherencePolicy with UncachedTransactions +abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) + with UncachedTransactions -abstract class IncoherentPolicy extends CoherencePolicy { - // UNIMPLEMENTED - def newStateOnProbe(incoming: Probe, state: UInt): UInt = state - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = UInt(0) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = Bool(false) - def getGrantType(a_type: UInt, count: UInt): UInt = UInt(0) - def getGrantType(rel: Release, count: UInt): UInt = UInt(0) - def getProbeType(a_type: UInt, global_state: UInt): UInt = UInt(0) - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = Bool(false) - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) - def requiresAckForGrant(g_type: UInt) = Bool(true) - def requiresAckForRelease(r_type: UInt) = Bool(false) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = Bool(false) - -} - -class ThreeStateIncoherence extends IncoherentPolicy { - def nClientStates = 3 - def nMasterStates = 0 - def nAcquireTypes = 3 - def nProbeTypes = 0 - def nReleaseTypes = 2 - def nGrantTypes = 3 - val tileInvalid :: tileClean :: tileDirty :: Nil = Enum(UInt(), nClientStates) - val acquireReadClean :: acquireReadDirty :: acquireWriteback :: Nil = Enum(UInt(), nAcquireTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantData :: grantAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataAcquireTypeVec = Vec(acquireWriteback) - val hasDataReleaseTypeVec = Vec(acquireWriteback) - val hasDataGrantTypeVec = Vec(grantData) - - def isHit ( cmd: UInt, state: UInt): Bool = (state === tileClean || state === tileDirty) - def isValid (state: UInt): Bool = state != tileInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire) = Bool(false) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = state === tileDirty - def needsWriteback (state: UInt): Bool = state === tileDirty - - def newState(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileDirty, Mux(isRead(cmd), Mux(state === tileDirty, tileDirty, tileClean), state)) - } - def newStateOnHit(cmd: UInt, state: UInt): UInt = newState(cmd, state) - def newStateOnCacheControl(cmd: UInt) = tileInvalid //TODO - def newStateOnWriteback() = tileInvalid - def newStateOnFlush() = tileInvalid - def newStateOnGrant(incoming: Grant, outstanding: Acquire) = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantData -> Mux(outstanding.a_type === acquireReadDirty, tileDirty, tileClean), - grantAck -> tileInvalid - )) - } - - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, acquireReadClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = releaseVoluntaryInvalidateData - - def messageHasData( msg: SourcedMessage ) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) - case rel: Release => Bool(false) - case _ => Bool(false) - } - def messageUpdatesDataArray (reply: Grant) = (reply.g_type === grantData) - def messageIsUncached(acq: Acquire): Bool = Bool(false) -} - -class MICoherence extends CoherencePolicyWithUncached { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -149,8 +157,8 @@ class MICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 5 def nGrantTypes = 7 - val tileInvalid :: tileValid :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalValid :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -162,43 +170,54 @@ class MICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) + def isValid (m: ClientMetadata): Bool = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) + def isValid (m: MasterMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileValid), Array( - M_INV -> (state === tileValid), - M_CLN -> (state === tileValid) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientValid), Array( + M_INV -> (m.state === clientValid), + M_CLN -> (m.state === clientValid) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = state - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileValid - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> tileValid, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientValid + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> clientValid, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseCopyAck) -> m )) } @@ -211,11 +230,11 @@ class MICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = acquireReadExclusive + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeCopy -> releaseCopyData @@ -224,7 +243,7 @@ class MICoherence extends CoherencePolicyWithUncached { probeInvalidate -> releaseInvalidateAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -257,7 +276,7 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -268,22 +287,21 @@ class MICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence extends CoherencePolicyWithUncached { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -291,8 +309,8 @@ class MEICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 8 - val tileInvalid :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -304,50 +322,64 @@ class MEICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = state != tileInvalid - def isValid (state: UInt): Bool = state != tileInvalid + def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) + def isValid (m: ClientMetadata) = m.state != clientInvalid + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileExclusiveClean - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileExclusiveClean, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientExclusiveClean + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -360,15 +392,15 @@ class MEICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -379,7 +411,7 @@ class MEICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -413,7 +445,7 @@ class MEICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, @@ -425,15 +457,14 @@ class MEICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -441,7 +472,7 @@ class MEICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence extends CoherencePolicyWithUncached { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -449,8 +480,8 @@ class MSICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusive :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -462,56 +493,70 @@ class MSICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> tileExclusiveDirty, - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> clientExclusiveDirty, + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -524,15 +569,15 @@ class MSICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -543,7 +588,7 @@ class MSICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -576,8 +621,7 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -586,15 +630,14 @@ class MSICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = Bool(false) @@ -602,7 +645,7 @@ class MSICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence extends CoherencePolicyWithUncached { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -610,8 +653,8 @@ class MESICoherence extends CoherencePolicyWithUncached { def nReleaseTypes = 7 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val globalInvalid :: globalShared :: globalExclusiveClean :: Nil = Enum(UInt(), nMasterStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -623,56 +666,71 @@ class MESICoherence extends CoherencePolicyWithUncached { val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), (state === tileExclusiveClean || state === tileExclusiveDirty), - (state === tileShared || state === tileExclusiveClean || state === tileExclusiveDirty)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), + (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> (state === tileExclusiveDirty), - M_CLN -> (state === tileExclusiveDirty) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> (m.state === clientExclusiveDirty), + M_CLN -> (m.state === clientExclusiveDirty) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), tileExclusiveDirty, state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, tileExclusiveDirty, tileExclusiveClean), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeDowngrade -> tileShared, - probeCopy -> state + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> m.state + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m )) } @@ -685,15 +743,15 @@ class MESICoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> releaseInvalidateData, probeDowngrade -> releaseDowngradeData, @@ -704,7 +762,7 @@ class MESICoherence extends CoherencePolicyWithUncached { probeDowngrade -> releaseDowngradeAck, probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -738,7 +796,7 @@ class MESICoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -750,15 +808,14 @@ class MESICoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) @@ -767,15 +824,16 @@ class MESICoherence extends CoherencePolicyWithUncached { def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence extends CoherencePolicyWithUncached { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 0 + def nMasterStates = 3 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 - val tileInvalid :: tileShared :: tileExclusiveClean :: tileExclusiveDirty :: tileSharedByTwo :: tileMigratoryClean :: tileMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -787,72 +845,92 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def isHit (cmd: UInt, state: UInt): Bool = { - Mux(isWriteIntent(cmd), Vec(tileExclusiveClean, tileExclusiveDirty, tileMigratoryClean, tileMigratoryDirty).contains(state), (state != tileInvalid)) + def isHit (cmd: UInt, m: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) } - def isValid (state: UInt): Bool = { - state != tileInvalid + def isValid (m: ClientMetadata): Bool = { + m.state != clientInvalid } + def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) + def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && messageIsUncached(outstanding)) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } - def needsTransactionOnCacheControl(cmd: UInt, state: UInt): Bool = { - MuxLookup(cmd, (state === tileExclusiveDirty), Array( - M_INV -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state), - M_CLN -> Vec(tileExclusiveDirty,tileMigratoryDirty).contains(state) + def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { + MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( + M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), + M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) )) } - def needsWriteback (state: UInt): Bool = { - needsTransactionOnCacheControl(M_INV, state) + def needsWriteback (m: ClientMetadata): Bool = { + needsTransactionOnCacheControl(M_INV, m) } - def newStateOnHit(cmd: UInt, state: UInt): UInt = { - Mux(isWrite(cmd), MuxLookup(state, tileExclusiveDirty, Array( - tileExclusiveClean -> tileExclusiveDirty, - tileMigratoryClean -> tileMigratoryDirty)), state) - } - def newStateOnCacheControl(cmd: UInt) = { - MuxLookup(cmd, tileInvalid, Array( - M_INV -> tileInvalid, - M_CLN -> tileShared - )) - } - def newStateOnWriteback() = newStateOnCacheControl(M_INV) - def newStateOnFlush() = newStateOnCacheControl(M_INV) - def newStateOnGrant(incoming: Grant, outstanding: Acquire): UInt = { - MuxLookup(incoming.g_type, tileInvalid, Array( - grantReadShared -> tileShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, tileExclusiveDirty, Array( - acquireReadExclusive -> tileExclusiveDirty, - acquireReadShared -> tileExclusiveClean)), - grantReadExclusiveAck -> tileExclusiveDirty, - grantReadUncached -> tileInvalid, - grantWriteUncached -> tileInvalid, - grantReadWordUncached -> tileInvalid, - grantWriteWordUncached -> tileInvalid, - grantAtomicUncached -> tileInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, tileMigratoryDirty, Array( - acquireInvalidateOthers -> tileMigratoryDirty, - acquireReadExclusive -> tileMigratoryDirty, - acquireReadShared -> tileMigratoryClean)) - )) - } - def newStateOnProbe(incoming: Probe, state: UInt): UInt = { - MuxLookup(incoming.p_type, state, Array( - probeInvalidate -> tileInvalid, - probeInvalidateOthers -> tileInvalid, - probeCopy -> state, - probeDowngrade -> MuxLookup(state, tileShared, Array( - tileExclusiveClean -> tileSharedByTwo, - tileExclusiveDirty -> tileSharedByTwo, - tileSharedByTwo -> tileShared, - tileMigratoryClean -> tileSharedByTwo, - tileMigratoryDirty -> tileInvalid)) + def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( + Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( + MuxLookup(cmd, clientInvalid, Array( + M_INV -> clientInvalid, + M_CLN -> clientShared + )))(this) + def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadUncached -> clientInvalid, + grantWriteUncached -> clientInvalid, + grantReadWordUncached -> clientInvalid, + grantWriteWordUncached -> clientInvalid, + grantAtomicUncached -> clientInvalid, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + )))(this) + def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( + MuxLookup(incoming.p_type, m.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> m.state, + probeDowngrade -> MuxLookup(m.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid)) + )))(this) + def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { + val popped = m.sharers.pop(src) + val next = MasterMetadata( + Mux(popped.none(), masterInvalid, + Mux(popped.one(), masterExclusive, masterShared)), + popped)(this) + def is(r: UInt) = incoming.r_type === r + MuxBundle(m, Array( + is(releaseVoluntaryInvalidateData) -> next, + is(releaseInvalidateData) -> next, + is(releaseDowngradeData) -> m, + is(releaseCopyData) -> m, + is(releaseInvalidateAck) -> next, + is(releaseDowngradeAck) -> m, + is(releaseCopyAck) -> m, + is(releaseDowngradeDataMigratory) -> m, + is(releaseDowngradeAckHasCopy) -> m, + is(releaseInvalidateDataMigratory) -> next, + is(releaseInvalidateAckMigratory) -> next )) } + def getUncachedReadAcquireType = acquireReadUncached def getUncachedWriteAcquireType = acquireWriteUncached def getUncachedReadWordAcquireType = acquireReadWordUncached @@ -862,28 +940,28 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck - def getAcquireTypeOnPrimaryMiss(cmd: UInt, state: UInt): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, state: UInt, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(state === tileInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) + def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { + Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) } def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, state: UInt): UInt = { + def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(tileExclusiveDirty, tileMigratoryDirty).contains(state), + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(state === tileMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), + probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), probeCopy -> releaseCopyData )) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(tileExclusiveClean === state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(state === tileSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(state != tileInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck )) - Mux(needsWriteback(state), with_data, without_data) + Mux(needsWriteback(m), with_data, without_data) } def messageHasData(msg: SourcedMessage) = msg match { @@ -906,7 +984,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { acquireReadWordUncached -> grantReadWordUncached, acquireWriteWordUncached -> grantWriteWordUncached, acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } def getGrantType(rel: Release, count: UInt): UInt = { @@ -916,7 +994,7 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { } - def getProbeType(a_type: UInt, global_state: UInt): UInt = { + def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { MuxLookup(a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, @@ -929,15 +1007,14 @@ class MigratoryCoherence extends CoherencePolicyWithUncached { )) } - def needsOuterRead(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) } - def needsOuterWrite(a_type: UInt, global_state: UInt): Bool = { + def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterAcquire(a_type: UInt, global_state: UInt): Bool = { - needsOuterRead(a_type, global_state) || needsOuterWrite(a_type, global_state) - } + def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) + def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index dea3e801..22fa933c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -232,7 +232,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.needsOuterWrite(io.uncached.acquire.bits.payload.a_type, UInt(0)) + has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c8e33216..09bb0714 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -167,7 +167,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.needsOuterRead(xact.a_type, UInt(0)) + val pending_outer_read = co.requiresOuterRead(xact.a_type) val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data) val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, @@ -195,7 +195,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, UInt(0)), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -215,7 +215,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.needsOuterRead(c_acq.payload.a_type, UInt(0)) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src From 1ae7a9376cb2a0ec631eb1d634171f767327a20c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 13 Jun 2014 03:25:52 -0700 Subject: [PATCH 229/688] Fix unhandled LLC writeback hazard --- uncore/src/main/scala/llc.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..8da87042 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -438,10 +438,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From a04ef4f5f42e8f5ed758d47c89656266b9ac12ad Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 11:44:43 -0700 Subject: [PATCH 230/688] Quick change to work with new Width class. Replace .width with .needWidth() --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..28cb5217 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 8da87042..1c8671d0 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 + val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From 0020ded36745bad05fa7376d08b828311f84e6cb Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Fri, 13 Jun 2014 14:53:48 -0700 Subject: [PATCH 231/688] Replace needWidth() with getWidth. --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 28cb5217..29eba5da 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -91,7 +91,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.needWidth()-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1c8671d0..3a21cdca 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.needWidth() <= leaf.data.needWidth() && n > leaf.n) 1 << math.floor(math.log(leaf.data.needWidth()/data.needWidth())/math.log(2)).toInt else 1 - val nWide = if (data.needWidth() > leaf.data.needWidth()) 1+(data.needWidth()-1)/leaf.data.needWidth() else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,10 +39,10 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.needWidth(), UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.needWidth()*(j+1))-1, leaf.data.needWidth()*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } .otherwise { if (postLatency > 0) ridx := idx } @@ -61,7 +61,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.needWidth()*(k+1)-1, gen.needWidth()*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -268,7 +268,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.needWidth()) + data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +385,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.needWidth()-1, s2.addr.needWidth()-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +395,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.needWidth()-1, mshr.io.tag.bits.addr.needWidth()-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) From f411fdcce39c354061348bd74968e86eaaeee100 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 8 Aug 2014 12:21:57 -0700 Subject: [PATCH 232/688] Full conversion to params. Compiles but does not elaborate. --- uncore/src/main/scala/cache.scala | 252 +++++++++----------------- uncore/src/main/scala/htif.scala | 54 +++--- uncore/src/main/scala/llc.scala | 37 ++-- uncore/src/main/scala/memserdes.scala | 94 +++++----- uncore/src/main/scala/network.scala | 57 +++--- uncore/src/main/scala/tilelink.scala | 106 ++++++----- uncore/src/main/scala/uncore.scala | 71 ++++---- 7 files changed, 301 insertions(+), 370 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9a5afee3..247110a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,75 +1,18 @@ package uncore import Chisel._ -trait CacheConfig { - def sets: Int - def ways: Int - def tl: TileLinkConfiguration - def as: AddressSpaceConfiguration - def dm: Boolean - def states: Int - def lines: Int - def tagbits: Int - def idxbits: Int - def offbits: Int - def untagbits: Int - def rowbits: Int -} - -case class L2CacheConfig( - val sets: Int, val ways: Int, - val nrpq: Int, val nsdq: Int, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int, - val tl: TileLinkConfiguration, - val as: AddressSpaceConfiguration) - extends CoherenceAgentConfiguration - with CacheConfig -{ - def states = tl.co.nMasterStates - def lines = sets*ways - def dm = ways == 1 - def offbits = 0 - def lineaddrbits = tl.addrBits - def idxbits = log2Up(sets) - def waybits = log2Up(ways) - def untagbits = offbits + idxbits - def tagbits = lineaddrbits - idxbits - def wordbits = 64 - def wordbytes = wordbits/8 - def wordoffbits = log2Up(wordbytes) - def rowbits = tl.dataBits - def rowbytes = rowbits/8 - def rowoffbits = log2Up(rowbytes) - def refillcycles = tl.dataBits/(rowbits) - def statebits = log2Up(states) - def nTSHRs = nReleaseTransactions + nAcquireTransactions - - require(states > 0) - require(isPow2(sets)) - require(isPow2(ways)) // TODO: relax this - require(refillcycles == 1) //TODO: relax this? -} - -abstract trait CacheBundle extends Bundle { - implicit val cacheconf: CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(cacheconf).asInstanceOf[this.type] -} - -abstract trait L2CacheBundle extends Bundle { - implicit val l2cacheconf: L2CacheConfig - override def clone = this.getClass.getConstructors.head.newInstance(l2cacheconf).asInstanceOf[this.type] -} - -trait HasL2Id extends L2CacheBundle { - val id = UInt(width = log2Up(l2cacheconf.nTSHRs)) -} - -trait HasL2InternalRequestState extends L2CacheBundle { - val tag_match = Bool() - val old_meta = new L2Metadata - val way_en = Bits(width = l2cacheconf.ways) -} +case object NSets extends Field[Int] +case object NWays extends Field[Int] +case object IsDM extends Field[Boolean] +case object TagBits extends Field[Int] +case object IdxBits extends Field[Int] +case object OffBits extends Field[Int] +case object UntagBits extends Field[Int] +case object WayBits extends Field[Int] +case object RowBits extends Field[Int] +case object WordBits extends Field[Int] +case object RefillCycles extends Field[Int] +case object Replacer extends Field[() => ReplacementPolicy] abstract class ReplacementPolicy { def way: UInt @@ -77,54 +20,53 @@ abstract class ReplacementPolicy { def hit: Unit } -class RandomReplacement(implicit val cacheconf: CacheConfig) extends ReplacementPolicy { +class RandomReplacement(ways: Int) extends ReplacementPolicy { private val replace = Bool() replace := Bool(false) val lfsr = LFSR16(replace) - def way = if(cacheconf.dm) UInt(0) else lfsr(log2Up(cacheconf.ways)-1,0) + def way = if(ways == 1) UInt(0) else lfsr(log2Up(ways)-1,0) def miss = replace := Bool(true) def hit = {} } -abstract class Metadata(implicit val cacheconf: CacheConfig) extends CacheBundle { - val tag = Bits(width = cacheconf.tagbits) +abstract class Metadata extends Bundle { + val tag = Bits(width = params(TagBits)) val coh: CoherenceMetadata } -class MetaReadReq(implicit val cacheconf: CacheConfig) extends CacheBundle { - val idx = Bits(width = cacheconf.idxbits) +class MetaReadReq extends Bundle { + val idx = Bits(width = params(IdxBits)) } -class MetaWriteReq[T <: Metadata](gen: T)(implicit conf: CacheConfig) extends MetaReadReq { - val way_en = Bits(width = conf.ways) +class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { + val way_en = Bits(width = params(WayBits)) val data = gen.clone - override def clone = new MetaWriteReq(gen)(conf).asInstanceOf[this.type] + override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConfig) extends Module { - implicit val tl = conf.tl +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(conf.ways){rstVal.clone.asOutput} + val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(conf.sets+1))) - val rst = rst_cnt < UInt(conf.sets) + val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) + val rst = rst_cnt < UInt(params(NSets)) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*conf.ways), conf.sets, seqRead = true) + val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(conf.ways, wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until conf.ways) { + for (w <- 0 until params(NWays)) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -133,62 +75,63 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T)(implicit conf: CacheConf io.write.ready := !rst } +trait HasL2Id extends Bundle { + val id = UInt(width = log2Up(params(NTransactors))) +} + +trait HasL2InternalRequestState extends Bundle { + val tag_match = Bool() + val old_meta = new L2Metadata + val way_en = Bits(width = params(NWays)) +} + object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata)(implicit conf: L2CacheConfig) = { + def apply(tag: Bits, coh: MasterMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata(implicit val l2cacheconf: L2CacheConfig) extends Metadata - with L2CacheBundle { - val coh = l2cacheconf.tl.co.masterMetadataOnFlush.clone +class L2Metadata extends Metadata { + val coh = params(TLCoherence).masterMetadataOnFlush.clone } -/* -class L3Metadata(implicit conf: L3CacheConfig) extends Metadata - with L3CacheBundle { - val coh = MixedMetaData(conf.innerTL.co, conf.outerTL.co) ? -} -*/ - -class L2MetaReadReq(implicit val l2cacheconf: L2CacheConfig) extends MetaReadReq - with HasL2Id { - val tag = Bits(width = l2cacheconf.tagbits) +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = params(TagBits)) } -class L2MetaWriteReq(implicit val l2cacheconf: L2CacheConfig) extends MetaWriteReq[L2Metadata](new L2Metadata) +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle +class L2MetaResp extends Bundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { - implicit val tl = conf.tl +class L2MetadataArray extends Module { + val (co, ways) = (params(TLCoherence), params(NWays)) val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip val resp = Valid(new L2MetaResp) } - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), tl.co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until conf.ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && tl.co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - val replacer = new RandomReplacement + val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => @@ -203,23 +146,21 @@ class L2MetadataArray(implicit conf: L2CacheConfig) extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val way_en = Bits(width = l2cacheconf.ways) - val addr = Bits(width = l2cacheconf.tl.addrBits) +class L2DataReadReq extends Bundle with HasL2Id { + val way_en = Bits(width = params(NWays)) + val addr = Bits(width = params(TLAddrBits)) } -class L2DataWriteReq(implicit l2cacheconf: L2CacheConfig) extends L2DataReadReq { - val wmask = Bits(width = l2cacheconf.tl.writeMaskBits) - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) } -class L2DataResp(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle - with HasL2Id { - val data = Bits(width = l2cacheconf.tl.dataBits) +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) } -class L2DataArray(implicit conf: L2CacheConfig) extends Module { +class L2DataArray extends Module { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -228,9 +169,9 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(conf.wordbits, io.write.bits.wmask) - val resp = (0 until conf.ways).map { w => - val array = Mem(Bits(width=conf.rowbits), conf.sets*conf.refillcycles, seqRead = true) + val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) + val resp = (0 until params(NWays)).map { w => + val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -244,23 +185,11 @@ class L2DataArray(implicit conf: L2CacheConfig) extends Module { io.write.ready := Bool(true) } -class L2InternalAcquire(implicit val l2cacheconf: L2CacheConfig) extends Acquire()(l2cacheconf.tl) - with HasL2InternalRequestState +class L2HellaCache(bankId: Int) extends CoherenceAgent { -class L2InternalRelease(implicit val l2cacheconf: L2CacheConfig) extends Release()(l2cacheconf.tl) - with HasL2InternalRequestState - -class InternalTileLinkIO(implicit val l2cacheconf: L2CacheConfig) extends L2CacheBundle { - implicit val (tl, ln) = (l2cacheconf.tl, l2cacheconf.tl.ln) - val acquire = new DecoupledIO(new LogicalNetworkIO(new L2InternalAcquire)) - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip - val release = new DecoupledIO(new LogicalNetworkIO(new L2InternalRelease)) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) -} - -class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) + require(isPow2(params(NSets))) + require(isPow2(params(NWays))) + require(params(RefillCycles) == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -278,12 +207,12 @@ class L2HellaCache(bankId: Int)(implicit conf: L2CacheConfig) extends CoherenceA } -class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class TSHRFile(bankId: Int) extends Module { + val (co, nClients) = (params(TLCoherence), params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(ln.nClients){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) val meta_resp = Valid(new L2MetaResp).flip @@ -305,10 +234,9 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } // Create TSHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map { id => + val trackerList = (0 until params(NReleaseTransactors)).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (conf.nReleaseTransactions until nTrackers).map { id => + } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -373,12 +301,12 @@ class TSHRFile(bankId: Int)(implicit conf: L2CacheConfig) extends Module { } -abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class L2XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) val meta_read = Decoupled(new L2MetaReadReq) @@ -397,11 +325,11 @@ abstract class L2XactTracker()(implicit conf: L2CacheConfig) extends Module { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val new_meta = Reg(new L2Metadata) val incoming_rel = io.inner.release.bits @@ -460,16 +388,16 @@ Mux(co.messageHasData(xact), s_mem, s_ack) } } -class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig) extends L2XactTracker()(conf) { +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -480,12 +408,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -501,7 +429,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, conf.tl.co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) @@ -525,9 +453,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -550,7 +478,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -558,7 +486,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: L2CacheConfig } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cc1c1c62..153bad3a 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,6 +4,11 @@ import Chisel._ import Node._ import uncore._ +case object HTIFWidth extends Field[Int] +case object HTIFNSCR extends Field[Int] +case object HTIFOffsetBits extends Field[Int] +case object HTIFNCores extends Field[Int] + class HostIO(val w: Int) extends Bundle { val clk = Bool(OUTPUT) @@ -20,43 +25,48 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO(ntiles: Int) extends Bundle +class HTIFIO extends Bundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(ntiles)) + val id = UInt(INPUT, log2Up(params(HTIFNCores))) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(ntiles))) + val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO(n: Int) extends Bundle +class SCRIO extends Bundle { - val rdata = Vec.fill(n){Bits(INPUT, 64)} + val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(n)) + val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) val wdata = Bits(OUTPUT, 64) } -class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: TileLinkConfiguration) extends Module +class HTIF(pcr_RESET: Int) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - val nTiles = ln.nClients-1 // This HTIF is itself a TileLink client + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) val io = new Bundle { val host = new HostIO(w) - val cpu = Vec.fill(nTiles){new HTIFIO(nTiles).flip} + val cpu = Vec.fill(nCores){new HTIFIO().flip} val mem = new TileLinkIO - val scr = new SCRIO(nSCR) + val scr = new SCRIO } + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + conf.dataBits + val long_request_bits = short_request_bits + dataBits require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -92,7 +102,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) - val pcr_coreid = addr(log2Up(nTiles)-1+20+1,20) + val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR @@ -114,13 +124,13 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val mem_acked = Reg(init=Bool(false)) val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt(width = conf.ln.idBits)) + val mem_gsrc = Reg(UInt()) val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := conf.co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) } io.mem.grant.ready := Bool(true) @@ -168,8 +178,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } var mem_req_data: Bits = null - for (i <- 0 until conf.dataBits/short_request_bits) { - val idx = UInt(i, log2Up(conf.dataBits/short_request_bits)) + for (i <- 0 until dataBits/short_request_bits) { + val idx = UInt(i, log2Up(dataBits/short_request_bits)) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } @@ -184,7 +194,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(conf.ln.nClients) // By convention HTIF is the client with the largest id + io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.master_xact_id := mem_gxid @@ -194,7 +204,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) - for (i <- 0 until nTiles) { + for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) val my_ipi = Reg(init=Bool(false)) @@ -211,7 +221,7 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti } cpu.ipi_rep.valid := my_ipi cpu.ipi_req.ready := Bool(true) - for (j <- 0 until nTiles) { + for (j <- 0 until nCores) { when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { my_ipi := Bool(true) } @@ -239,8 +249,8 @@ class HTIF(w: Int, pcr_RESET: Int, nSCR: Int, offsetBits: Int)(implicit conf: Ti val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nTiles) - scr_rdata(1) := UInt((BigInt(conf.dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(0) := UInt(nCores) + scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 382a5f96..5bc89e38 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -72,26 +72,26 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn io.rdata := Mux1H(rdataSel, rdataDeep) } -class LLCDataReq(ways: Int)(implicit conf: MemoryIFConfiguration) extends MemReqCmd +class LLCDataReq(ways: Int) extends MemReqCmd { val way = UInt(width = log2Up(ways)) val isWriteback = Bool() - override def clone = new LLCDataReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCDataReq(ways).asInstanceOf[this.type] } -class LLCTagReq(ways: Int)(implicit val conf: MemoryIFConfiguration) extends HasMemAddr +class LLCTagReq(ways: Int) extends HasMemAddr { val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways)(conf).asInstanceOf[this.type] + override def clone = new LLCTagReq(ways).asInstanceOf[this.type] } -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Decoupled(new MemReqCmd).flip val repl_way = UInt(INPUT, log2Up(ways)) val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, conf.addrBits - log2Up(sets)) + val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) val data = Decoupled(new LLCDataReq(ways)) val tag = Decoupled(new LLCTagReq(ways)) val mem = new MemPipeIO @@ -100,14 +100,14 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im } class MSHR extends Bundle { - val addr = UInt(width = conf.addrBits) + val addr = UInt(width = params(MIFAddrBits)) val way = UInt(width = log2Up(ways)) val tag = io.cpu.bits.tag.clone val refilled = Bool() val refillCount = UInt(width = log2Up(refill_cycles)) val requested = Bool() val old_dirty = Bool() - val old_tag = UInt(width = conf.addrBits - log2Up(sets)) + val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) val wb_busy = Bool() override def clone = new MSHR().asInstanceOf[this.type] @@ -183,10 +183,10 @@ class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int)(im io.mem_resp_way := mshr(refillId).way } -class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module { val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = conf.addrBits)).flip } + val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } val data = Vec.fill(requestors){Decoupled(new MemData).flip } val mem = new MemPipeIO } @@ -232,12 +232,12 @@ class LLCWriteback(requestors: Int, refill_cycles: Int)(implicit conf: MemoryIFC io.mem.req_data.bits := io.data(who).bits } -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends Module +class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module { val io = new Bundle { val req = Decoupled(new LLCDataReq(ways)).flip val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = conf.addrBits)) + val writeback = Decoupled(UInt(width = params(MIFAddrBits))) val writeback_data = Decoupled(new MemData) val resp = Decoupled(new MemResp) val mem_resp = Valid(new MemResp).flip @@ -245,7 +245,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -300,7 +300,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ io.writeback_data.bits := q.io.deq.bits } -class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration) extends Module +class MemReqArb(n: Int, refill_cycles: Int) extends Module { val io = new Bundle { val cpu = Vec.fill(n){new MemIO().flip} @@ -343,16 +343,16 @@ class MemReqArb(n: Int, refill_cycles: Int)(implicit conf: MemoryIFConfiguration when (unlock) { lock := Bool(false) } } -abstract class DRAMSideLLCLike(implicit conf: MemoryIFConfiguration) extends Module { +abstract class DRAMSideLLCLike extends Module { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt])(implicit conf: MemoryIFConfiguration) extends DRAMSideLLCLike +class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { - val tagWidth = conf.addrBits - log2Up(sets) + val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) @@ -499,8 +499,7 @@ object HellaQueue } } -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int)(implicit conf: MemoryIFConfiguration) - extends DRAMSideLLCLike { +class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 22fa933c..eba971fd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -2,57 +2,58 @@ package uncore import Chisel._ import scala.math._ -case class AddressSpaceConfiguration(paddrBits: Int, vaddrBits: Int, pgIdxBits: Int, asidBits: Int, permBits:Int) { - val ppnBits = paddrBits - pgIdxBits - val vpnBits = vaddrBits - pgIdxBits +case object PAddrBits extends Field[Int] +case object VAddrBits extends Field[Int] +case object PgIdxBits extends Field[Int] +case object ASIdBits extends Field[Int] +case object PermBits extends Field[Int] +case object PPNBits extends Field[Int] +case object VPNBits extends Field[Int] + +case object MIFAddrBits extends Field[Int] +case object MIFDataBits extends Field[Int] +case object MIFTagBits extends Field[Int] +case object MIFDataBeats extends Field[Int] + +trait HasMemData extends Bundle { + val data = Bits(width = params(MIFDataBits)) } -case class MemoryIFConfiguration(addrBits: Int, dataBits: Int, tagBits: Int, dataBeats: Int) - -abstract trait MemoryIFSubBundle extends Bundle { - implicit val conf: MemoryIFConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(conf).asInstanceOf[this.type] +trait HasMemAddr extends Bundle { + val addr = UInt(width = params(MIFAddrBits)) } -trait HasMemData extends MemoryIFSubBundle { - val data = Bits(width = conf.dataBits) +trait HasMemTag extends Bundle { + val tag = UInt(width = params(MIFTagBits)) } -trait HasMemAddr extends MemoryIFSubBundle { - val addr = UInt(width = conf.addrBits) -} - -trait HasMemTag extends MemoryIFSubBundle { - val tag = UInt(width = conf.tagBits) -} - -class MemReqCmd(implicit val conf: MemoryIFConfiguration) extends HasMemAddr with HasMemTag { +class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp(implicit val conf: MemoryIFConfiguration) extends HasMemData with HasMemTag +class MemResp extends HasMemData with HasMemTag -class MemData(implicit val conf: MemoryIFConfiguration) extends HasMemData +class MemData extends HasMemData -class MemIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Decoupled(new MemResp).flip } -class MemPipeIO(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemPipeIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) val req_data = Decoupled(new MemData) val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module +class MemSerdes(w: Int) extends Module { val io = new Bundle { val wide = new MemIO().flip @@ -61,6 +62,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -68,7 +70,7 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -104,12 +106,12 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(conf.dataBeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -127,21 +129,22 @@ class MemSerdes(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) } -class MemDesserIO(w: Int)(implicit val conf: MemoryIFConfiguration) extends Bundle { +class MemDesserIO(w: Int) extends Bundle { val narrow = new MemSerializedIO(w).flip val wide = new MemIO } -class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module // test rig side +class MemDesser(w: Int) extends Module // test rig side { val io = new MemDesserIO(w) val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth + val dbeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(conf.dataBeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -169,13 +172,13 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(conf.dataBeats-1)) { + when (data_recv_cnt === UInt(dbeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -189,7 +192,7 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, conf.dataBeats)) + val dataq = Module(new Queue(new MemResp, dbeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -198,18 +201,21 @@ class MemDesser(w: Int)(implicit val conf: MemoryIFConfiguration) extends Module } //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkConfiguration, mifconf: MemoryIFConfiguration) extends Module { +class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { val uncached = new UncachedTileLinkIO().flip val mem = new MemIO } - - require(tlconf.dataBits == mifconf.dataBits*mifconf.dataBeats) - //require(tlconf.clientXactIdBits <= mifconf.tagBits) + val co = params(TLCoherence) + val tbits = params(MIFTagBits) + val dbits = params(MIFDataBits) + val dbeats = params(MIFDataBits) + require(params(TLDataBits) == dbits*dbeats) + //require(params(TLClientXactIdBits) <= params(MIFTagBits)) val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = mifconf.dataBeats + val cnt_max = dbeats val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) val active_out = Reg(init=Bool(false)) @@ -222,7 +228,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) val active_in = Reg(init=Bool(false)) val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = mifconf.tagBits)) + val tag_in = Reg(UInt(width = tbits)) // Decompose outgoing TL Acquires into MemIO cmd and data when(!active_out && io.uncached.acquire.valid) { @@ -232,7 +238,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon buf_out := io.uncached.acquire.bits.payload.data tag_out := io.uncached.acquire.bits.payload.client_xact_id addr_out := io.uncached.acquire.bits.payload.addr - has_data := tlconf.co.messageHasData(io.uncached.acquire.bits.payload) + has_data := co.messageHasData(io.uncached.acquire.bits.payload) } when(active_out) { when(mem_cmd_q.io.enq.fire()) { @@ -240,7 +246,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon } when(mem_data_q.io.enq.fire()) { cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(mifconf.dataBits) + buf_out := buf_out >> UInt(dbits) } when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { active_out := Bool(false) @@ -264,7 +270,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(mifconf.dataBits*(cnt_max-1)) + buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) tag_in := io.mem.resp.bits.tag } when(active_in) { @@ -272,7 +278,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int)(implicit tlconf: TileLinkCon active_in := Bool(false) } when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*mifconf.dataBits-1,mifconf.dataBits)) + buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) cnt_in := cnt_in + UInt(1) } } diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 6889bd7f..72002d45 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,32 +1,35 @@ package uncore import Chisel._ -case class PhysicalNetworkConfiguration(nEndpoints: Int, idBits: Int) +case object LNMasters extends Field[Int] +case object LNClients extends Field[Int] +case object LNEndpoints extends Field[Int] -class PhysicalHeader(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) } -class PhysicalNetworkIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val header = new PhysicalHeader +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) val payload = dType.clone - override def clone = { new PhysicalNetworkIO(dType).asInstanceOf[this.type] } + override def clone = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] } -abstract class PhysicalNetwork(conf: PhysicalNetworkConfiguration) extends Module - -class BasicCrossbarIO[T <: Data](dType: T)(implicit conf: PhysicalNetworkConfiguration) extends Bundle { - val in = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))}.flip - val out = Vec.fill(conf.nEndpoints){Decoupled(new PhysicalNetworkIO(dType))} +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))}.flip + val out = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))} } -class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: PhysicalNetworkConfiguration) extends PhysicalNetwork(conf) { - val io = new BasicCrossbarIO(dType) - val rdyVecs = List.fill(conf.nEndpoints)(Vec.fill(conf.nEndpoints)(Bool())) +abstract class PhysicalNetwork extends Module + +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { + val io = new BasicCrossbarIO(n, dType) + + val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.nEndpoints, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits @@ -34,30 +37,26 @@ class BasicCrossbar[T <: Data](dType: T, count: Int = 1)(implicit conf: Physical }} out <> rrarb.io.out }} - for(i <- 0 until conf.nEndpoints) { + for(i <- 0 until n) { io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) } } -case class LogicalNetworkConfiguration(idBits: Int, nMasters: Int, nClients: Int) { - val nEndpoints = nMasters + nClients +abstract class LogicalNetwork extends Module + +class LogicalHeader extends Bundle { + val src = UInt(width = log2Up(params(LNEndpoints))) + val dst = UInt(width = log2Up(params(LNEndpoints))) } -abstract class LogicalNetwork[TileLinkType <: Bundle](implicit conf: LogicalNetworkConfiguration) extends Module - -class LogicalHeader(implicit conf: LogicalNetworkConfiguration) extends Bundle { - val src = UInt(width = conf.idBits) - val dst = UInt(width = conf.idBits) -} - -class LogicalNetworkIO[T <: Data](dType: T)(implicit conf: LogicalNetworkConfiguration) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { val header = new LogicalHeader val payload = dType.clone override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0))(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +68,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]])(implicit conf: LogicalNetworkConfiguration) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 68da8213..d49e264b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,27 +1,29 @@ package uncore import Chisel._ -case class TileLinkConfiguration(co: CoherencePolicyWithUncached, ln: LogicalNetworkConfiguration, addrBits: Int, masterXactIdBits: Int, clientXactIdBits: Int, dataBits: Int, writeMaskBits: Int, wordAddrBits: Int, atomicOpBits: Int) +case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLAddrBits extends Field[Int] +case object TLMasterXactIdBits extends Field[Int] +case object TLClientXactIdBits extends Field[Int] +case object TLDataBits extends Field[Int] +case object TLWriteMaskBits extends Field[Int] +case object TLWordAddrBits extends Field[Int] +case object TLAtomicOpBits extends Field[Int] -abstract trait TileLinkSubBundle extends Bundle { - implicit val tlconf: TileLinkConfiguration - override def clone = this.getClass.getConstructors.head.newInstance(tlconf).asInstanceOf[this.type] +trait HasPhysicalAddress extends Bundle { + val addr = UInt(width = params(TLAddrBits)) } -trait HasPhysicalAddress extends TileLinkSubBundle { - val addr = UInt(width = tlconf.addrBits) +trait HasClientTransactionId extends Bundle { + val client_xact_id = Bits(width = params(TLClientXactIdBits)) } -trait HasClientTransactionId extends TileLinkSubBundle { - val client_xact_id = Bits(width = tlconf.clientXactIdBits) +trait HasMasterTransactionId extends Bundle { + val master_xact_id = Bits(width = params(TLMasterXactIdBits)) } -trait HasMasterTransactionId extends TileLinkSubBundle { - val master_xact_id = Bits(width = tlconf.masterXactIdBits) -} - -trait HasTileLinkData extends TileLinkSubBundle { - val data = Bits(width = tlconf.dataBits) +trait HasTileLinkData extends Bundle { + val data = Bits(width = params(TLDataBits)) } trait SourcedMessage extends Bundle @@ -30,7 +32,7 @@ trait MasterSourcedMessage extends SourcedMessage object Acquire { - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { val acq = new Acquire acq.a_type := a_type acq.addr := addr @@ -41,42 +43,42 @@ object Acquire acq.atomic_opcode := Bits(0) acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id) acq.data := data acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.write_mask := write_mask acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { val acq = apply(a_type, addr, client_xact_id, data) acq.subword_addr := subword_addr acq.atomic_opcode := atomic_opcode acq } - def apply(a: Acquire)(implicit conf: TileLinkConfiguration): Acquire = { + def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a acq } } -class Acquire(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Acquire extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val a_type = UInt(width = tlconf.co.acquireTypeWidth) - val write_mask = Bits(width = tlconf.writeMaskBits) - val subword_addr = Bits(width = tlconf.wordAddrBits) - val atomic_opcode = Bits(width = tlconf.atomicOpBits) + val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) + val write_mask = Bits(width = params(TLWriteMaskBits)) + val subword_addr = Bits(width = params(TLWordAddrBits)) + val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } object Probe { - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr @@ -85,22 +87,22 @@ object Probe } } -class Probe(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { - val p_type = UInt(width = tlconf.co.probeTypeWidth) + val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } object Release { - def apply(r_type: UInt, addr: UInt, data: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.data := data rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr @@ -109,24 +111,24 @@ object Release rel.data := UInt(0) rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Release = { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } } -class Release(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage +class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasMasterTransactionId with HasTileLinkData { - val r_type = UInt(width = tlconf.co.releaseTypeWidth) + val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } object Grant { - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt)(implicit conf: TileLinkConfiguration) = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { val gnt = new Grant gnt.g_type := g_type gnt.client_xact_id := client_xact_id @@ -134,40 +136,35 @@ object Grant gnt.data := UInt(0) gnt } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt)(implicit conf: TileLinkConfiguration): Grant = { + def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { val gnt = apply(g_type, client_xact_id, master_xact_id) gnt.data := data gnt } } -class Grant(implicit val tlconf: TileLinkConfiguration) extends MasterSourcedMessage +class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = tlconf.co.grantTypeWidth) + val g_type = UInt(width = params(TLCoherence).grantTypeWidth) } -class Finish(implicit val tlconf: TileLinkConfiguration) extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(implicit conf: TileLinkConfiguration) extends Bundle { - implicit val ln = conf.ln +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) - override def clone = { new UncachedTileLinkIO().asInstanceOf[this.type] } } -class TileLinkIO(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIO()(conf) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) - override def clone = { new TileLinkIO().asInstanceOf[this.type] } } -abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfiguration) extends Module { - implicit val (ln, co) = (conf.ln, conf.co) - +abstract class TileLinkArbiterLike(val arbN: Int) extends Module { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -178,7 +175,7 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) val arb = Module(new RRArbiter(out.bits.clone, arbN)) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -205,7 +202,8 @@ abstract class TileLinkArbiterLike(val arbN: Int)(implicit conf: TileLinkConfigu } } -abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class UncachedTileLinkIOArbiter(n: Int) + extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO @@ -219,7 +217,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfigur finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } -abstract class TileLinkIOArbiter(n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkArbiterLike(n)(conf) { +abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val io = new Bundle { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO @@ -267,9 +265,9 @@ abstract trait UsesNewId { in.client_xact_id } -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends UncachedTileLinkIOArbiter(n)(conf) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit conf: TileLinkConfiguration) extends TileLinkIOArbiter(n)(conf) with UsesNewId +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 09bb0714..013f84c2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,33 +1,27 @@ package uncore import Chisel._ -trait CoherenceAgentConfiguration { - def tl: TileLinkConfiguration - def nReleaseTransactions: Int - def nAcquireTransactions: Int -} +case object NReleaseTransactors extends Field[Int] +case object NAcquireTransactors extends Field[Int] +case object NTransactors extends Field[Int] +case object NClients extends Field[Int] -abstract class CoherenceAgent(implicit conf: CoherenceAgentConfiguration) extends Module { +abstract class CoherenceAgent extends Module { + val co = params(TLCoherence) val io = new Bundle { - val inner = (new TileLinkIO()(conf.tl)).flip - val outer = new UncachedTileLinkIO()(conf.tl) - val incoherent = Vec.fill(conf.tl.ln.nClients){Bool()}.asInput + val inner = (new TileLinkIO).flip + val outer = new UncachedTileLinkIO + val incoherent = Vec.fill(params(NClients)){Bool()}.asInput } } -case class L2CoherenceAgentConfiguration( - val tl: TileLinkConfiguration, - val nReleaseTransactions: Int, - val nAcquireTransactions: Int) extends CoherenceAgentConfiguration - -class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends CoherenceAgent { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val nTrackers = conf.nReleaseTransactions + conf.nAcquireTransactions - val trackerList = (0 until conf.nReleaseTransactions).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (conf.nReleaseTransactions until nTrackers).map(id => Module(new AcquireTracker(id, bankId))) + val trackerList = (0 until params(NReleaseTransactors)).map(id => + Module(new VoluntaryReleaseTracker(id, bankId))) ++ + (params(NReleaseTransactors) until params(NTransactors)).map(id => + Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -85,12 +79,12 @@ class L2CoherenceAgent(bankId: Int)(implicit conf: CoherenceAgentConfiguration) } -abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends Module { - implicit val (tl, ln, co) = (conf.tl, conf.tl.ln, conf.tl.co) +abstract class XactTracker extends Module { + val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val tile_incoherent = Bits(INPUT, ln.nClients) + val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } @@ -100,15 +94,13 @@ abstract class XactTracker()(implicit conf: CoherenceAgentConfiguration) extends val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) @@ -153,17 +145,16 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int)(implicit conf: Cohere } } -class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentConfiguration) - extends XactTracker()(conf) { +class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(ln.nClients))) + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(ln.nClients))) - val release_count = if (ln.nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(ln.nClients))) - val probe_flags = Reg(init=Bits(0, width = ln.nClients)) + val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) @@ -174,12 +165,12 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC xact.addr, UInt(trackerId), c_rel.payload.data) val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) - val probe_initial_flags = Bits(width = ln.nClients) + val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (ln.nClients > 1) { + if (nClients > 1) { // issue self-probes for uncached read xacts to facilitate I$ coherence val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(ln.nClients)-1,0))) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) } @@ -219,9 +210,9 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(ln.nClients) // TODO: Broadcast only + init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags - if(ln.nClients > 1) { + if(nClients > 1) { release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, Mux(needs_outer_write, s_mem_write, @@ -244,7 +235,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -252,7 +243,7 @@ class AcquireTracker(trackerId: Int, bankId: Int)(implicit conf: CoherenceAgentC } } .otherwise { io.inner.release.ready := Bool(true) - if(ln.nClients > 1) release_count := release_count - UInt(1) + if(nClients > 1) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) From 9ab3a4262cedb689d616c6427f20f88d2e40521d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 11 Aug 2014 18:35:49 -0700 Subject: [PATCH 233/688] Cache utility traits. Completely compiles, asm tests hang. --- uncore/src/main/scala/cache.scala | 113 +++++++++++++++----------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/uncore.scala | 17 ++-- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 247110a5..21cf582e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -3,17 +3,35 @@ import Chisel._ case object NSets extends Field[Int] case object NWays extends Field[Int] -case object IsDM extends Field[Boolean] -case object TagBits extends Field[Int] -case object IdxBits extends Field[Int] -case object OffBits extends Field[Int] -case object UntagBits extends Field[Int] -case object WayBits extends Field[Int] +case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object WordBits extends Field[Int] -case object RefillCycles extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +abstract trait CacheParameters extends UsesParameters { + val paddrBits = params(PAddrBits) + val vaddrBits = params(VAddrBits) + val pgIdxBits = params(PgIdxBits) + val nSets = params(NSets) + val nWays = params(NWays) + val blockOffBits = params(BlockOffBits) + val wordBits = params(WordBits) + val rowBits = params(RowBits) + val wordOffBits = log2Up(wordBits) + val idxBits = log2Up(nSets) + val untagBits = blockOffBits + idxBits + val tagBits = paddrBits - untagBits + val wayBits = log2Up(nWays) + val isDM = nWays == 1 + val rowWords = rowBits/wordBits + val rowBytes = rowBits*8 + val rowOffBits = log2Up(rowBytes) + val refillCycles = params(TLDataBits)/rowBits +} + +abstract class CacheBundle extends Bundle with CacheParameters +abstract class CacheModule extends Module with CacheParameters + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -30,43 +48,43 @@ class RandomReplacement(ways: Int) extends ReplacementPolicy { def hit = {} } -abstract class Metadata extends Bundle { - val tag = Bits(width = params(TagBits)) +abstract class Metadata extends CacheBundle { + val tag = Bits(width = tagBits) val coh: CoherenceMetadata } -class MetaReadReq extends Bundle { - val idx = Bits(width = params(IdxBits)) +class MetaReadReq extends CacheBundle { + val idx = Bits(width = idxBits) } class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { - val way_en = Bits(width = params(WayBits)) + val way_en = Bits(width = nWays) val data = gen.clone override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(params(NWays)){rstVal.clone.asOutput} + val resp = Vec.fill(nWays){rstVal.clone.asOutput} } val metabits = rstVal.getWidth - val rst_cnt = Reg(init=UInt(0, log2Up(params(NSets)+1))) - val rst = rst_cnt < UInt(params(NSets)) + val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) + val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) when (rst) { rst_cnt := rst_cnt+UInt(1) } - val tag_arr = Mem(UInt(width = metabits*params(NWays)), params(NSets), seqRead = true) + val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(params(NWays), wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until params(NWays)) { + for (w <- 0 until nWays) { val m = tags(metabits*(w+1)-1, metabits*w) io.resp(w) := rstVal.clone.fromBits(m) } @@ -75,14 +93,20 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends Module { io.write.ready := !rst } -trait HasL2Id extends Bundle { - val id = UInt(width = log2Up(params(NTransactors))) +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) } -trait HasL2InternalRequestState extends Bundle { +trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() val old_meta = new L2Metadata - val way_en = Bits(width = params(NWays)) + val way_en = Bits(width = nWays) } object L2Metadata { @@ -93,23 +117,22 @@ object L2Metadata { meta } } -class L2Metadata extends Metadata { - val coh = params(TLCoherence).masterMetadataOnFlush.clone +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = params(TagBits)) + val tag = Bits(width = tagBits) } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id -class L2MetaResp extends Bundle +class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState -class L2MetadataArray extends Module { - val (co, ways) = (params(TLCoherence), params(NWays)) +class L2MetadataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2MetaReadReq).flip val write = Decoupled(new L2MetaWriteReq).flip @@ -123,7 +146,7 @@ class L2MetadataArray extends Module { val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until ways).map(f)) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -146,8 +169,8 @@ class L2MetadataArray extends Module { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends Bundle with HasL2Id { - val way_en = Bits(width = params(NWays)) +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) val addr = Bits(width = params(TLAddrBits)) } @@ -160,7 +183,7 @@ class L2DataResp extends Bundle with HasL2Id { val data = Bits(width = params(TLDataBits)) } -class L2DataArray extends Module { +class L2DataArray extends L2HellaCacheModule { val io = new Bundle { val read = Decoupled(new L2DataReadReq).flip val write = Decoupled(new L2DataWriteReq).flip @@ -169,9 +192,9 @@ class L2DataArray extends Module { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(params(WordBits), io.write.bits.wmask) - val resp = (0 until params(NWays)).map { w => - val array = Mem(Bits(width=params(RowBits)), params(NSets)*params(RefillCycles), seqRead = true) + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -185,11 +208,11 @@ class L2DataArray extends Module { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent { +class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { - require(isPow2(params(NSets))) - require(isPow2(params(NWays))) - require(params(RefillCycles) == 1) + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) @@ -207,8 +230,7 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent { } -class TSHRFile(bankId: Int) extends Module { - val (co, nClients) = (params(TLCoherence), params(NClients)) +class TSHRFile(bankId: Int) extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO @@ -234,9 +256,9 @@ class TSHRFile(bankId: Int) extends Module { } // Create TSHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map { id => + val trackerList = (0 until nReleaseTransactors).map { id => Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (params(NReleaseTransactors) until params(NTransactors)).map { id => + } ++ (nReleaseTransactors until nTransactors).map { id => Module(new L2AcquireTracker(id, bankId)) } @@ -301,8 +323,7 @@ class TSHRFile(bankId: Int) extends Module { } -abstract class L2XactTracker extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) +abstract class L2XactTracker extends L2HellaCacheModule { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eba971fd..02d74385 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,7 +209,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val co = params(TLCoherence) val tbits = params(MIFTagBits) val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBits) + val dbeats = params(MIFDataBeats) require(params(TLDataBits) == dbits*dbeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 013f84c2..580f0477 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -3,24 +3,31 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NTransactors extends Field[Int] case object NClients extends Field[Int] -abstract class CoherenceAgent extends Module { +abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) + val nReleaseTransactors = params(NReleaseTransactors) + val nAcquireTransactors = params(NAcquireTransactors) + val nTransactors = nReleaseTransactors + nAcquireTransactors + val nClients = params(NClients) +} + +abstract class CoherenceAgent extends Module + with CoherenceAgentParameters { val io = new Bundle { val inner = (new TileLinkIO).flip val outer = new UncachedTileLinkIO - val incoherent = Vec.fill(params(NClients)){Bool()}.asInput + val incoherent = Vec.fill(nClients){Bool()}.asInput } } class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { // Create SHRs for outstanding transactions - val trackerList = (0 until params(NReleaseTransactors)).map(id => + val trackerList = (0 until nReleaseTransactors).map(id => Module(new VoluntaryReleaseTracker(id, bankId))) ++ - (params(NReleaseTransactors) until params(NTransactors)).map(id => + (nReleaseTransactors until nTransactors).map(id => Module(new AcquireTracker(id, bankId))) // Propagate incoherence flags From e26f8a6f6ae8c1da134939a9c4177752f34a5a3c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 12 Aug 2014 14:55:44 -0700 Subject: [PATCH 234/688] Fix errors in derived cache params --- uncore/src/main/scala/cache.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 21cf582e..d52e692a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -13,18 +13,19 @@ abstract trait CacheParameters extends UsesParameters { val vaddrBits = params(VAddrBits) val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) - val nWays = params(NWays) val blockOffBits = params(BlockOffBits) - val wordBits = params(WordBits) - val rowBits = params(RowBits) - val wordOffBits = log2Up(wordBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = paddrBits - untagBits + val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 + val wordBits = params(WordBits) + val wordBytes = wordBits/8 + val wordOffBits = log2Up(wordBytes) + val rowBits = params(RowBits) val rowWords = rowBits/wordBits - val rowBytes = rowBits*8 + val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) val refillCycles = params(TLDataBits)/rowBits } From e384b33cc36c8ecbb0fe5bf5f87ee7f4e84b29af Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 19 Aug 2014 15:50:20 -0700 Subject: [PATCH 235/688] don't generate a write mask for BigMem if it isn't used not needed for llc data --- uncore/src/main/scala/llc.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 3a21cdca..09ad87bb 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -44,7 +44,12 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = conf.dataBits))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = conf.dataBits))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.getWidth) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt From dc5643b12f2e657aa95712af92dceb4effdc3eff Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 23 Aug 2014 01:19:36 -0700 Subject: [PATCH 236/688] Final parameter refactor. --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d52e692a..c35aeedf 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,6 +1,7 @@ package uncore import Chisel._ +case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] From 17b2359c9ae70882cb4e309afe7bdc47bc11f33f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 24 Aug 2014 19:27:58 -0700 Subject: [PATCH 237/688] htif parameters trait --- uncore/src/main/scala/htif.scala | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 153bad3a..bfe4ef10 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -9,7 +9,18 @@ case object HTIFNSCR extends Field[Int] case object HTIFOffsetBits extends Field[Int] case object HTIFNCores extends Field[Int] -class HostIO(val w: Int) extends Bundle +abstract trait HTIFParameters extends UsesParameters { + val dataBits = params(TLDataBits) + val co = params(TLCoherence) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) +} + +abstract class HTIFBundle extends Bundle with HTIFParameters + +class HostIO extends HTIFBundle { val clk = Bool(OUTPUT) val clk_edge = Bool(OUTPUT) @@ -25,42 +36,34 @@ class PCRReq extends Bundle val data = Bits(width = 64) } -class HTIFIO extends Bundle -{ +class HTIFIO extends HTIFBundle { val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(params(HTIFNCores))) + val id = UInt(INPUT, log2Up(nCores)) val pcr_req = Decoupled(new PCRReq).flip val pcr_rep = Decoupled(Bits(width = 64)) - val ipi_req = Decoupled(Bits(width = log2Up(params(HTIFNCores)))) + val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO extends Bundle -{ - val rdata = Vec.fill(params(HTIFNSCR)){Bits(INPUT, 64)} +class SCRIO extends HTIFBundle { + val rdata = Vec.fill(nSCR){Bits(INPUT, 64)} val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(params(HTIFNSCR))) + val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, 64) } -class HTIF(pcr_RESET: Int) extends Module -{ - val dataBits = params(TLDataBits) - val co = params(TLCoherence) - val w = params(HTIFWidth) - val nSCR = params(HTIFNSCR) - val offsetBits = params(HTIFOffsetBits) - val nCores = params(HTIFNCores) - val io = new Bundle { - val host = new HostIO(w) - val cpu = Vec.fill(nCores){new HTIFIO().flip} +class HTIFModuleIO extends HTIFBundle { + val host = new HostIO + val cpu = Vec.fill(nCores){new HTIFIO}.flip val mem = new TileLinkIO val scr = new SCRIO - } +} +class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { + val io = new HTIFModuleIO io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' From 712f3a754d73c3fa8149e04596d9b93dfd891658 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 2 Sep 2014 12:34:42 -0700 Subject: [PATCH 238/688] merge in master --- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/llc.scala | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index bfe4ef10..8d1b2618 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -104,7 +104,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.width-1, 0) + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 5bc89e38..f34ffac4 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt])(gen: => T) extends Module +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module { class Inputs extends Bundle { val addr = UInt(INPUT, log2Up(n)) @@ -15,8 +15,8 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn val rdata = gen.asOutput } val data = gen - val colMux = if (2*data.width <= leaf.data.width && n > leaf.n) 1 << math.floor(math.log(leaf.data.width/data.width)/math.log(2)).toInt else 1 - val nWide = if (data.width > leaf.data.width) 1+(data.width-1)/leaf.data.width else 1 + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 if (nDeep > 1 || colMux > 1) require(isPow2(n) && isPow2(leaf.n)) @@ -39,12 +39,17 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var dout: Bits = null val ridx = if (postLatency > 0) Reg(Bits()) else null - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.width, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.width*(j+1))-1, leaf.data.width*j)) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) when (in.valid) { - when (in.bits.rw) { mem.write(idx, wdata0, wmask0) } + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } .otherwise { if (postLatency > 0) ridx := idx } } @@ -61,7 +66,7 @@ class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UIn var colMuxOut = rdataWide if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.width*(k+1)-1, gen.width*k))) + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) } @@ -245,7 +250,7 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ val mem_resp_way = UInt(INPUT, log2Up(ways)) } - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf)(Bits(width = params(MIFDataBits)))) + val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) class QEntry extends MemResp { val isWriteback = Bool() override def clone = new QEntry().asInstanceOf[this.type] @@ -268,7 +273,6 @@ class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[ data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt data.io.in.bits.rw := io.req.bits.rw data.io.in.bits.wdata := io.req_data.bits.data - data.io.in.bits.wmask := SInt(-1, io.req_data.bits.data.width) when (valid) { data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt @@ -385,7 +389,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta for (i <- 0 until ways) s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.width-1, s2.addr.width-tagWidth) === t(tagWidth-1, 0)) + val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) val s2_hit_way = OHToUInt(s2_hits) val s2_hit = s2_hits.reduceLeft(_||_) val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) @@ -395,7 +399,7 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta val tag_we = initialize || setDirty || mshr.io.tag.fire() val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.width-1, mshr.io.tag.bits.addr.width-tagWidth)) + val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) @@ -438,10 +442,11 @@ class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, ta replay_s2_rdy := s3_rdy && !tag_we io.cpu.resp <> data.io.resp - io.cpu.req_cmd.ready := !s1_valid && !s2_valid && !replay_s2 && !tag_we io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready io.mem.req_cmd <> memCmdArb.io.out io.mem.req_data <> writeback.io.mem.req_data + io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || + io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module From bfb662968d2e34ad589988866a4638900c5ad39e Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 14:33:58 -0700 Subject: [PATCH 239/688] fixes sbt error during first run --- uncore/chisel-dependent.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index fa8d5490..eeb52c04 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,6 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -val chiselVersion = System.getProperty("chiselVersion", "None") +lazy val chiselVersion = System.getProperty("chiselVersion", "None") libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) From f8821b4cc9a9878c76b6ab849f31914163da4e04 Mon Sep 17 00:00:00 2001 From: Scott Beamer Date: Tue, 2 Sep 2014 15:16:03 -0700 Subject: [PATCH 240/688] better fix with explanation of sbt issue --- uncore/chisel-dependent.sbt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt index eeb52c04..88eb615c 100644 --- a/uncore/chisel-dependent.sbt +++ b/uncore/chisel-dependent.sbt @@ -1,6 +1,8 @@ // Provide a managed dependency on chisel if -DchiselVersion="" is // supplied on the command line. -lazy val chiselVersion = System.getProperty("chiselVersion", "None") +val chiselVersion_u = System.getProperty("chiselVersion", "None") -libraryDependencies ++= ( if (chiselVersion != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion) :: Nil; else Nil) +// _u a temporary fix until sbt 13.6 https://github.com/sbt/sbt/issues/1465 + +libraryDependencies ++= ( if (chiselVersion_u != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion_u) :: Nil; else Nil) From f8d450b4e2156d73cbdb383c4eed19cbbd87ab0c Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 11 Sep 2014 22:06:03 -0700 Subject: [PATCH 241/688] mark DRAMSideLLC as HasKnownBug --- uncore/src/main/scala/llc.scala | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index f34ffac4..1878a356 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -354,7 +354,19 @@ abstract class DRAMSideLLCLike extends Module { } } -class DRAMSideLLC(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike +// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new +// L2$. Stay stuned. +// +// Bug description: +// There's a race condition between the writeback unit (the module which +// sends the data out to the backside interface) and the data unit (the module +// which writes the data into the SRAM in the L2$). The "hit status" +// is saved in a register, however, is updated again when there's +// a transaction coming from the core without waiting until the writeback unit +// has sent out all its data to the outer memory system. This is why the +// problem manifests at a higher probability with the slow backup memory port. + +class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike { val tagWidth = params(MIFAddrBits) - log2Up(sets) val metaWidth = tagWidth + 2 // valid + dirty @@ -505,7 +517,6 @@ object HellaQueue } class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles val size = log2Down(numEntries) + 1 From 0b51d70bd288d9146c5c8d99d5916a208c82789b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:31:38 -0700 Subject: [PATCH 242/688] add LICENSE --- uncore/src/main/scala/cache.scala | 2 ++ uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/consts.scala | 2 ++ uncore/src/main/scala/htif.scala | 2 ++ uncore/src/main/scala/llc.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 2 ++ uncore/src/main/scala/network.scala | 2 ++ uncore/src/main/scala/package.scala | 2 ++ uncore/src/main/scala/slowio.scala | 2 ++ uncore/src/main/scala/tilelink.scala | 2 ++ uncore/src/main/scala/uncore.scala | 2 ++ 11 files changed, 22 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c35aeedf..686f0ac1 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 45a1970c..5fa0b7a6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 64d6c3a7..a402e7db 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore package constants diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8d1b2618..edcd66cb 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala index 1878a356..35553702 100644 --- a/uncore/src/main/scala/llc.scala +++ b/uncore/src/main/scala/llc.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 02d74385..6f42f010 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ import scala.math._ diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 72002d45..d3f4a9e8 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala index 501d511d..2c6c4a5f 100644 --- a/uncore/src/main/scala/package.scala +++ b/uncore/src/main/scala/package.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package object uncore extends uncore.constants.MemoryOpConstants { implicit def toOption[A](a: A) = Option(a) diff --git a/uncore/src/main/scala/slowio.scala b/uncore/src/main/scala/slowio.scala index 9ed0abb7..95ca34e6 100644 --- a/uncore/src/main/scala/slowio.scala +++ b/uncore/src/main/scala/slowio.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d49e264b..af736c97 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 580f0477..8f1a7a67 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -1,3 +1,5 @@ +// See LICENSE for license details. + package uncore import Chisel._ From 49b027db2c8a423b76cb63bf19089f5077823138 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Fri, 12 Sep 2014 15:36:29 -0700 Subject: [PATCH 243/688] forgot to add LICENSE file --- uncore/LICENSE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 uncore/LICENSE diff --git a/uncore/LICENSE b/uncore/LICENSE new file mode 100644 index 00000000..7cff15e4 --- /dev/null +++ b/uncore/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. From f249da1803d958804e5d1e715e324c11b4c1a913 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 17 Sep 2014 11:25:14 -0700 Subject: [PATCH 244/688] update README --- uncore/README | 6 ------ uncore/README.md | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) delete mode 100644 uncore/README create mode 100644 uncore/README.md diff --git a/uncore/README b/uncore/README deleted file mode 100644 index c920e19a..00000000 --- a/uncore/README +++ /dev/null @@ -1,6 +0,0 @@ -This is the repository for uncore components assosciated with UCB_BAR chip -projects. To uses these modules, include this repo as a git submodule within -the your chip repository and add it as Project in your chip's build.scala. -These components are only dependent on Chisel, i.e. -lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) - diff --git a/uncore/README.md b/uncore/README.md new file mode 100644 index 00000000..003f9caa --- /dev/null +++ b/uncore/README.md @@ -0,0 +1,11 @@ +Uncore Library +============== + +This is the repository for uncore components assosciated with Rocket chip +project. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. + + lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + +Documentation about the uncore library will come in the near future. From f7b1e23eadebdc956b68dbd6f5e8eedb04bb6241 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 16:59:48 -0700 Subject: [PATCH 245/688] functional style on MuxBundle --- uncore/src/main/scala/coherence.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5fa0b7a6..1a3c4573 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -5,11 +5,7 @@ import Chisel._ object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - var res = default; - for ((t, v) <- mapping.reverse){ - res = Mux(t, v, res); - } - res + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) } } From faed47d131211fa12c393ae5fd864319cfc90ffc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:01:19 -0700 Subject: [PATCH 246/688] use thunk for dir info --- uncore/src/main/scala/coherence.scala | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1a3c4573..abacdabd 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir.clone + val sharers = c.dir() override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -75,10 +75,12 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation { def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } def none(dummy: Int = 0) = sharers === UInt(0) def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) + def count(dummy: Int = 0) = PopCount(sharers) + def next(dummy: Int = 0) = PriorityEncoder(sharers) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -144,10 +146,10 @@ trait UncachedTransactions { def isUncachedReadTransaction(acq: Acquire): Bool } -abstract class CoherencePolicyWithUncached(dir: DirectoryRepresentation) extends CoherencePolicy(dir) +abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) with UncachedTransactions -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 6 @@ -299,7 +301,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncac def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 7 @@ -470,7 +472,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 def nMasterStates = 3 def nAcquireTypes = 7 @@ -643,7 +645,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnca def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 def nMasterStates = 3 def nAcquireTypes = 7 @@ -822,7 +824,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUnc def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 def nMasterStates = 3 def nAcquireTypes = 8 From 149d51d644d1ffb0ae7914d512f0c7b41951ff71 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Jul 2014 17:10:32 -0700 Subject: [PATCH 247/688] more coherence API cleanup --- uncore/src/main/scala/cache.scala | 11 +-- uncore/src/main/scala/coherence.scala | 102 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 8 +- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 686f0ac1..faa879a9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,7 +376,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) @@ -454,11 +454,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index abacdabd..bd6fecc3 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,7 +28,7 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers := m.sharers.flush() + m.sharers.flush() m } def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { @@ -51,32 +51,36 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val sharers: UInt + val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation def none(dummy: Int = 0): Bool def one(dummy: Int = 0): Bool + def count(dummy: Int = 0): UInt + def next(dummy: Int = 0): UInt } class NullRepresentation extends DirectoryRepresentation { - val sharers = UInt(0) + val internal = UInt(0) def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) + def count(dummy: Int = 0) = UInt(0) + def next(dummy: Int = 0) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val sharers = UInt(width = nClients) - def pop(id: UInt) = { sharers := sharers & ~UIntToOH(id); this } - def push(id: UInt) = { sharers := sharers | UIntToOH(id); this } - def flush(dummy: Int = 0) = { sharers := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = sharers === UInt(0) - def one(dummy: Int = 0) = PopCount(sharers) === UInt(1) - def count(dummy: Int = 0) = PopCount(sharers) - def next(dummy: Int = 0) = PriorityEncoder(sharers) + val internal = UInt(width = nClients) + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def push(id: UInt) = { internal := internal | UIntToOH(id); this } + def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } + def none(dummy: Int = 0) = internal === UInt(0) + def one(dummy: Int = 0) = PopCount(internal) === UInt(1) + def count(dummy: Int = 0) = PopCount(internal) + def next(dummy: Int = 0) = PriorityEncoder(internal) override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] } @@ -113,12 +117,14 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a_type: UInt, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: MasterMetadata): UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt - def getGrantType(a_type: UInt, count: UInt): UInt - def getGrantType(rel: Release, count: UInt): UInt + def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt + def getGrantType(a: Acquire, m: MasterMetadata): UInt + def getGrantType(r: Release, m: MasterMetadata): UInt + //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO + //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool @@ -259,8 +265,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -270,14 +276,14 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, acquireWriteUncached -> probeInvalidate, @@ -427,8 +433,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( acquireReadExclusiveClean -> grantReadExclusive, acquireReadExclusiveDirty -> grantReadExclusive, acquireReadUncached -> grantReadUncached, @@ -438,15 +444,15 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadExclusiveClean -> probeInvalidate, acquireReadExclusiveDirty -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -604,9 +610,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -615,14 +621,14 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -778,9 +784,9 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -789,15 +795,15 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> grantAtomicUncached )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, @@ -975,9 +981,9 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a_type: UInt, count: UInt): UInt = { - MuxLookup(a_type, grantReadUncached, Array( - acquireReadShared -> Mux(count > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, grantReadUncached, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -987,15 +993,15 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? )) } - def getGrantType(rel: Release, count: UInt): UInt = { - MuxLookup(rel.r_type, grantReadUncached, Array( + def getGrantType(r: Release, m: MasterMetadata): UInt = { + MuxLookup(r.r_type, grantReadUncached, Array( releaseVoluntaryInvalidateData -> grantVoluntaryAck )) } - def getProbeType(a_type: UInt, m: MasterMetadata): UInt = { - MuxLookup(a_type, probeCopy, Array( + def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, probeCopy, Array( acquireReadShared -> probeDowngrade, acquireReadExclusive -> probeInvalidate, acquireReadUncached -> probeCopy, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8f1a7a67..2e62d79f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -130,7 +130,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, UInt(0)), + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -161,7 +161,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -195,11 +194,11 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact.a_type, co.masterMetadataOnFlush), + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact.a_type, init_sharer_cnt) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -219,7 +218,6 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) From 53b8d7b03164a97c5bb03cd6910b978168cf1252 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 13 Jul 2014 22:29:15 -0700 Subject: [PATCH 248/688] use new coherence methods in l2, ready to query dir logic --- uncore/src/main/scala/cache.scala | 83 ++++++++++++++++++++------- uncore/src/main/scala/coherence.scala | 2 +- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index faa879a9..3de8ba4a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -109,7 +109,7 @@ trait HasL2Id extends Bundle with CoherenceAgentParameters { trait HasL2InternalRequestState extends L2HellaCacheBundle { val tag_match = Bool() - val old_meta = new L2Metadata + val meta = new L2Metadata val way_en = Bits(width = nWays) } @@ -167,7 +167,7 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) io.resp.bits.tag_match := s2_tag_match - io.resp.bits.old_meta := Mux(s2_tag_match, + io.resp.bits.meta := Mux(s2_tag_match, L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) @@ -354,18 +354,16 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val new_meta = Reg(new L2Metadata) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), @@ -376,32 +374,47 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) xact.client_xact_id, UInt(trackerId)) + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src + xact := c_rel.payload + init_client_id := c_rel.header.src state := s_mem } - } -/* + }/* is(s_meta_read) { when(io.meta_read.ready) state := s_meta_resp } is(s_meta_resp) { when(io.meta_resp.valid) { - new_meta := L2Metadata(io.meta.resp.bits.old_meta.tag, io.meta.resp.bits.old_meta.sharers, io.meta.resp.bits - old_meta := io.meta.resp.bits.old_meta + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) state := Mux(s_meta_write -Mux(co.messageHasData(xact), s_mem, s_ack) - } - */ + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ is(s_mem) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } @@ -417,11 +430,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - - val init_sharer_cnt = Reg(init=UInt(0, width = log2Up(nClients))) - val release_count = if(nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) @@ -447,7 +460,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready @@ -459,7 +471,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.addr, UInt(trackerId)) - val grant_type = co.getGrantType(xact, conf.tl.co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id @@ -471,6 +483,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -479,7 +513,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - init_sharer_cnt := UInt(nClients) // TODO: Broadcast only probe_flags := probe_initial_flags if(nClients > 1) { release_count := PopCount(probe_initial_flags) @@ -490,6 +523,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(needs_outer_read, s_mem_read, s_make_grant)) } } + // s_read_meta + // xact_internal := resp is(s_probe) { // Generate probes io.inner.probe.valid := probe_flags.orR @@ -499,6 +534,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data to be written back when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index bd6fecc3..7dbff4aa 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -74,7 +74,7 @@ class NullRepresentation extends DirectoryRepresentation { class FullRepresentation(nClients: Int) extends DirectoryRepresentation { val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } + def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? def push(id: UInt) = { internal := internal | UIntToOH(id); this } def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } def none(dummy: Int = 0) = internal === UInt(0) From 82fe22f958a812cf8ff1d7755d5d4cb01cb7c242 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 11:14:36 -0700 Subject: [PATCH 249/688] support for multiple tilelink paramerterizations in same design Conflicts: src/main/scala/cache.scala --- uncore/src/main/scala/cache.scala | 46 ++++++++++++++++------------ uncore/src/main/scala/tilelink.scala | 5 +-- uncore/src/main/scala/uncore.scala | 44 ++++++++++++++------------ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3de8ba4a..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -212,13 +212,14 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParameters { +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) require(refillCycles == 1) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) val data = Module(new L2DataArray) @@ -234,10 +235,10 @@ class L2HellaCache(bankId: Int) extends CoherenceAgent with L2HellaCacheParamete } -class TSHRFile(bankId: Int) extends L2HellaCacheModule { +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput val meta_read = Decoupled(new L2MetaReadReq) val meta_write = Decoupled(new L2MetaWriteReq) @@ -261,9 +262,9 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } // Propagate incoherence flags @@ -312,7 +313,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { ack.ready := Bool(true) // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out @@ -327,10 +329,10 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule { } -abstract class L2XactTracker extends L2HellaCacheModule { +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -350,7 +352,7 @@ abstract class L2XactTracker extends L2HellaCacheModule { } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -364,10 +366,11 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -426,7 +429,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -440,11 +443,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af736c97..8bb98f0a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ +case object TLId extends Field[String] case object TLCoherence extends Field[CoherencePolicyWithUncached] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] @@ -155,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO extends UncachedTileLinkIO { +class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 2e62d79f..37e535f8 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,22 +15,23 @@ abstract trait CoherenceAgentParameters extends UsesParameters { val nClients = params(NClients) } -abstract class CoherenceAgent extends Module +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module with CoherenceAgentParameters { val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput } } -class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { +class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) { // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId))) + Module(new AcquireTracker(id, bankId, innerId, outerId))) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -82,17 +83,18 @@ class L2CoherenceAgent(bankId: Int) extends CoherenceAgent { ack.ready := Bool(true) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size)) + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out } -abstract class XactTracker extends Module { +abstract class XactTracker(innerId: String, outerId: String) extends Module { val (co, nClients) = (params(TLCoherence),params(NClients)) val io = new Bundle { - val inner = (new TileLinkIO).flip - val outer = new UncachedTileLinkIO + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, params(NClients)) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) @@ -105,7 +107,7 @@ abstract class XactTracker extends Module { val m_gnt = io.outer.grant.bits } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { +class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } @@ -120,10 +122,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), - xact.data) + xact.data), + { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -154,7 +157,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends XactTracker { } } -class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { +class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } @@ -167,11 +170,14 @@ class AcquireTracker(trackerId: Int, bankId: Int) extends XactTracker { val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data) - val outer_write_rel = Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data) - val outer_read = Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) From 7571695320f3804387a87aec570cf7ba80c01da6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 24 Sep 2014 15:11:24 -0700 Subject: [PATCH 250/688] Removed broken or unfinished modules, new MemPipeIO converter --- uncore/src/main/scala/bigmem.scala | 80 ++++ uncore/src/main/scala/cache.scala | 499 ----------------------- uncore/src/main/scala/llc.scala | 554 -------------------------- uncore/src/main/scala/memserdes.scala | 109 +++++ 4 files changed, 189 insertions(+), 1053 deletions(-) create mode 100644 uncore/src/main/scala/bigmem.scala delete mode 100644 uncore/src/main/scala/llc.scala diff --git a/uncore/src/main/scala/bigmem.scala b/uncore/src/main/scala/bigmem.scala new file mode 100644 index 00000000..fbbc4c07 --- /dev/null +++ b/uncore/src/main/scala/bigmem.scala @@ -0,0 +1,80 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module +{ + class Inputs extends Bundle { + val addr = UInt(INPUT, log2Up(n)) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = Valid(new Inputs).flip + val rdata = gen.asOutput + } + val data = gen + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} + for (i <- 0 until nDeep) { + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(next=ren) + val rdata = Vec.fill(nWide){Bits()} + + val r = Pipe(ren, in.bits.addr, postLatency) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val ridx = if (postLatency > 0) Reg(Bits()) else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + when (in.valid) { + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } + .otherwise { if (postLatency > 0) ridx := idx } + } + + if (postLatency == 0) { + dout = mem(idx) + } else if (postLatency == 1) { + dout = mem(ridx) + } else + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits + + rdata(j) := dout + } + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := r.valid + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..26157a75 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,502 +96,3 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } - -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters - -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters - -trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) -} - -trait HasL2InternalRequestState extends L2HellaCacheBundle { - val tag_match = Bool() - val meta = new L2Metadata - val way_en = Bits(width = nWays) -} - -object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { - val meta = new L2Metadata - meta.tag := tag - meta.coh := coh - meta - } -} -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone -} - -class L2MetaReadReq extends MetaReadReq with HasL2Id { - val tag = Bits(width = tagBits) -} - -class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - -class L2MetaResp extends L2HellaCacheBundle - with HasL2Id - with HasL2InternalRequestState - -class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } - - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) - meta.io.read <> io.read - meta.io.write <> io.write - - val s1_clk_en = Reg(next = io.read.fire()) - val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) - val s1_id = RegEnable(io.read.bits.id, io.read.valid) - def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) - val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits - val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) - val s2_tag_match = s2_tag_match_way.orR - val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) - - val replacer = params(Replacer)() - val s1_replaced_way_en = UIntToOH(replacer.way) - val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) - val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => - RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - - io.resp.valid := Reg(next = s1_clk_en) - io.resp.bits.id := RegEnable(s1_id, s1_clk_en) - io.resp.bits.tag_match := s2_tag_match - io.resp.bits.meta := Mux(s2_tag_match, - L2Metadata(s2_repl_meta.tag, s2_hit_coh), - s2_repl_meta) - io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) -} - -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { - val way_en = Bits(width = nWays) - val addr = Bits(width = params(TLAddrBits)) -} - -class L2DataWriteReq extends L2DataReadReq { - val wmask = Bits(width = params(TLWriteMaskBits)) - val data = Bits(width = params(TLDataBits)) -} - -class L2DataResp extends Bundle with HasL2Id { - val data = Bits(width = params(TLDataBits)) -} - -class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } - - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) - } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) - - io.read.ready := Bool(true) - io.write.ready := Bool(true) -} - -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - - require(isPow2(nSets)) - require(isPow2(nWays)) - require(refillCycles == 1) - - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) - - tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp - io.outer <> tshrfile.io.outer - io.incoherent <> tshrfile.io.incoherent -} - - -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } - } - - // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) - } - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Handle acquire transaction initiation - val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires - - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - - // Handle releases, which might be voluntary and might have data - val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Arbitrate for the outer memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - io.outer <> outer_arb.io.out - - // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) - -} - - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip - } - - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - - io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) - xact.client_xact_id, - UInt(trackerId)) - - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - state := s_mem - } - }/* - is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp - } - is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } - } - is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } - } - } -} - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) - val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) - - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - m_gnt.payload.data) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) - when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) - } - } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) - when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) - } - } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } - } - } -} diff --git a/uncore/src/main/scala/llc.scala b/uncore/src/main/scala/llc.scala deleted file mode 100644 index 35553702..00000000 --- a/uncore/src/main/scala/llc.scala +++ /dev/null @@ -1,554 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module -{ - class Inputs extends Bundle { - val addr = UInt(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = Valid(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 - val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec.fill(nDeep){Bits()} - val rdataSel = Vec.fill(nDeep){Bool()} - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(next=ren) - val rdata = Vec.fill(nWide){Bits()} - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val ridx = if (postLatency > 0) Reg(Bits()) else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - when (in.valid) { - when (in.bits.rw) { - if (noMask) - mem.write(idx, wdata0) - else - mem.write(idx, wdata0, wmask0) - } - .otherwise { if (postLatency > 0) ridx := idx } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = mem(ridx) - } else - dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} - -class LLCDataReq(ways: Int) extends MemReqCmd -{ - val way = UInt(width = log2Up(ways)) - val isWriteback = Bool() - override def clone = new LLCDataReq(ways).asInstanceOf[this.type] -} - -class LLCTagReq(ways: Int) extends HasMemAddr -{ - val way = UInt(width = log2Up(ways)) - override def clone = new LLCTagReq(ways).asInstanceOf[this.type] -} - -class LLCMSHRFile(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Decoupled(new MemReqCmd).flip - val repl_way = UInt(INPUT, log2Up(ways)) - val repl_dirty = Bool(INPUT) - val repl_tag = UInt(INPUT, params(MIFAddrBits) - log2Up(sets)) - val data = Decoupled(new LLCDataReq(ways)) - val tag = Decoupled(new LLCTagReq(ways)) - val mem = new MemPipeIO - val mem_resp_set = UInt(OUTPUT, log2Up(sets)) - val mem_resp_way = UInt(OUTPUT, log2Up(ways)) - } - - class MSHR extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) - val way = UInt(width = log2Up(ways)) - val tag = io.cpu.bits.tag.clone - val refilled = Bool() - val refillCount = UInt(width = log2Up(refill_cycles)) - val requested = Bool() - val old_dirty = Bool() - val old_tag = UInt(width = params(MIFAddrBits) - log2Up(sets)) - val wb_busy = Bool() - - override def clone = new MSHR().asInstanceOf[this.type] - } - - val valid = Vec.fill(outstanding){Reg(init=Bool(false))} - val validBits = valid.toBits - val freeId = PriorityEncoder(~validBits) - val mshr = Vec.fill(outstanding){Reg(new MSHR)} - when (io.cpu.valid && io.cpu.ready) { - valid(freeId) := Bool(true) - mshr(freeId).addr := io.cpu.bits.addr - mshr(freeId).tag := io.cpu.bits.tag - mshr(freeId).way := io.repl_way - mshr(freeId).old_dirty := io.repl_dirty - mshr(freeId).old_tag := io.repl_tag - mshr(freeId).wb_busy := Bool(false) - mshr(freeId).requested := Bool(false) - mshr(freeId).refillCount := UInt(0) - mshr(freeId).refilled := Bool(false) - } - - val requests = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && !mshr(i).old_dirty && !mshr(i).wb_busy && !mshr(i).requested):_*) - val request = requests.orR && io.data.ready // allow in-flight hits to drain - val requestId = PriorityEncoder(requests) - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { mshr(requestId).requested := Bool(true) } - - val refillId = io.mem.resp.bits.tag(log2Up(outstanding)-1, 0) - val refillCount = mshr(refillId).refillCount - when (io.mem.resp.valid) { - mshr(refillId).refillCount := refillCount + UInt(1) - when (refillCount === UInt(refill_cycles-1)) { mshr(refillId).refilled := Bool(true) } - } - - val replays = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).refilled):_*) - val replay = replays.orR - val replayId = PriorityEncoder(replays) - when (replay && io.data.ready && io.tag.ready) { valid(replayId) := Bool(false) } - - val writebacks = Cat(Bits(0), (outstanding-1 to 0 by -1).map(i => valid(i) && mshr(i).old_dirty):_*) - val writeback = writebacks.orR - val writebackId = PriorityEncoder(writebacks) - when (writeback && io.data.ready && !replay) { - mshr(writebackId).old_dirty := Bool(false) - mshr(writebackId).wb_busy := Bool(true) - } - mshr.foreach(m => when (m.wb_busy && io.data.ready) { m.wb_busy := Bool(false) }) - - val conflicts = Cat(Bits(0), (0 until outstanding).map(i => valid(i) && io.cpu.bits.addr(log2Up(sets)-1, 0) === mshr(i).addr(log2Up(sets)-1, 0)):_*) - io.cpu.ready := !conflicts.orR && !validBits.andR - - io.data.valid := writeback - io.data.bits.rw := Bool(false) - io.data.bits.tag := mshr(replayId).tag - io.data.bits.isWriteback := Bool(true) - io.data.bits.addr := Cat(mshr(writebackId).old_tag, mshr(writebackId).addr(log2Up(sets)-1, 0)).toUInt - io.data.bits.way := mshr(writebackId).way - when (replay) { - io.data.valid := io.tag.ready - io.data.bits.isWriteback := Bool(false) - io.data.bits.addr := mshr(replayId).addr - io.data.bits.way := mshr(replayId).way - } - io.tag.valid := replay && io.data.ready - io.tag.bits.addr := io.data.bits.addr - io.tag.bits.way := io.data.bits.way - - io.mem.req_cmd.valid := request - io.mem.req_cmd.bits.rw := Bool(false) - io.mem.req_cmd.bits.addr := mshr(requestId).addr - io.mem.req_cmd.bits.tag := requestId - io.mem_resp_set := mshr(refillId).addr - io.mem_resp_way := mshr(refillId).way -} - -class LLCWriteback(requestors: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val req = Vec.fill(requestors){Decoupled(UInt(width = params(MIFAddrBits))).flip } - val data = Vec.fill(requestors){Decoupled(new MemData).flip } - val mem = new MemPipeIO - } - - val valid = Reg(init=Bool(false)) - val who = Reg(UInt()) - val addr = Reg(UInt()) - val cmd_sent = Reg(Bool()) - val data_sent = Reg(Bool()) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - - var anyReq = Bool(false) - for (i <- 0 until requestors) { - io.req(i).ready := !valid && !anyReq - io.data(i).ready := valid && who === UInt(i) && io.mem.req_data.ready - anyReq = anyReq || io.req(i).valid - } - - val nextWho = PriorityEncoder(io.req.map(_.valid)) - when (!valid && io.req.map(_.valid).reduceLeft(_||_)) { - valid := Bool(true) - cmd_sent := Bool(false) - data_sent := Bool(false) - who := nextWho - addr := io.req(nextWho).bits - } - - when (io.mem.req_data.valid && io.mem.req_data.ready) { - count := count + UInt(1) - when (count === UInt(refill_cycles-1)) { - data_sent := Bool(true) - when (cmd_sent) { valid := Bool(false) } - } - } - when (io.mem.req_cmd.valid && io.mem.req_cmd.ready) { cmd_sent := Bool(true) } - when (valid && cmd_sent && data_sent) { valid := Bool(false) } - - io.mem.req_cmd.valid := valid && !cmd_sent - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := Bool(true) - - io.mem.req_data.valid := valid && !data_sent && io.data(who).valid - io.mem.req_data.bits := io.data(who).bits -} - -class LLCData(latency: Int, sets: Int, ways: Int, refill_cycles: Int, leaf: Mem[UInt]) extends Module -{ - val io = new Bundle { - val req = Decoupled(new LLCDataReq(ways)).flip - val req_data = Decoupled(new MemData).flip - val writeback = Decoupled(UInt(width = params(MIFAddrBits))) - val writeback_data = Decoupled(new MemData) - val resp = Decoupled(new MemResp) - val mem_resp = Valid(new MemResp).flip - val mem_resp_set = UInt(INPUT, log2Up(sets)) - val mem_resp_way = UInt(INPUT, log2Up(ways)) - } - - val data = Module(new BigMem(sets*ways*refill_cycles, 1, latency-1, leaf, true)(Bits(width = params(MIFDataBits)))) - class QEntry extends MemResp { - val isWriteback = Bool() - override def clone = new QEntry().asInstanceOf[this.type] - } - val q = Module(new Queue(new QEntry, latency+2)) - val qReady = q.io.count <= UInt(q.entries-latency-1) - val valid = Reg(init=Bool(false)) - val req = Reg(io.req.bits.clone) - val count = Reg(init=UInt(0, log2Up(refill_cycles))) - val refillCount = Reg(init=UInt(0, log2Up(refill_cycles))) - - when (data.io.in.valid && !io.mem_resp.valid) { - count := count + UInt(1) - when (valid && count === UInt(refill_cycles-1)) { valid := Bool(false) } - } - when (io.req.valid && io.req.ready) { valid := Bool(true); req := io.req.bits } - when (io.mem_resp.valid) { refillCount := refillCount + UInt(1) } - - data.io.in.valid := io.req.valid && io.req.ready && Mux(io.req.bits.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(io.req.bits.way, io.req.bits.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := io.req.bits.rw - data.io.in.bits.wdata := io.req_data.bits.data - when (valid) { - data.io.in.valid := Mux(req.rw, io.req_data.valid, qReady) - data.io.in.bits.addr := Cat(req.way, req.addr(log2Up(sets)-1, 0), count).toUInt - data.io.in.bits.rw := req.rw - } - when (io.mem_resp.valid) { - data.io.in.valid := Bool(true) - data.io.in.bits.addr := Cat(io.mem_resp_way, io.mem_resp_set, refillCount).toUInt - data.io.in.bits.rw := Bool(true) - data.io.in.bits.wdata := io.mem_resp.bits.data - } - - val tagPipe = Pipe(data.io.in.valid && !data.io.in.bits.rw, Mux(valid, req.tag, io.req.bits.tag), latency) - q.io.enq.valid := tagPipe.valid - q.io.enq.bits.tag := tagPipe.bits - q.io.enq.bits.isWriteback := Pipe(Mux(valid, req.isWriteback, io.req.bits.isWriteback), Bool(false), latency).valid - q.io.enq.bits.data := data.io.rdata - - io.req.ready := !valid && Mux(io.req.bits.isWriteback, io.writeback.ready, Bool(true)) - io.req_data.ready := !io.mem_resp.valid && Mux(valid, req.rw, io.req.valid && io.req.bits.rw) - - io.writeback.valid := io.req.valid && io.req.ready && io.req.bits.isWriteback - io.writeback.bits := io.req.bits.addr - - q.io.deq.ready := Mux(q.io.deq.bits.isWriteback, io.writeback_data.ready, io.resp.ready) - io.resp.valid := q.io.deq.valid && !q.io.deq.bits.isWriteback - io.resp.bits := q.io.deq.bits - io.writeback_data.valid := q.io.deq.valid && q.io.deq.bits.isWriteback - io.writeback_data.bits := q.io.deq.bits -} - -class MemReqArb(n: Int, refill_cycles: Int) extends Module -{ - val io = new Bundle { - val cpu = Vec.fill(n){new MemIO().flip} - val mem = new MemIO - } - - val lock = Reg(init=Bool(false)) - val locker = Reg(UInt()) - - val arb = Module(new RRArbiter(new MemReqCmd, n)) - val respWho = io.mem.resp.bits.tag(log2Up(n)-1,0) - val respTag = io.mem.resp.bits.tag >> UInt(log2Up(n)) - for (i <- 0 until n) { - val me = UInt(i, log2Up(n)) - arb.io.in(i).valid := io.cpu(i).req_cmd.valid - arb.io.in(i).bits := io.cpu(i).req_cmd.bits - arb.io.in(i).bits.tag := Cat(io.cpu(i).req_cmd.bits.tag, me) - io.cpu(i).req_cmd.ready := arb.io.in(i).ready - io.cpu(i).req_data.ready := Bool(false) - - val getLock = io.cpu(i).req_cmd.fire() && io.cpu(i).req_cmd.bits.rw && !lock - val haveLock = lock && locker === me - when (getLock) { - lock := Bool(true) - locker := UInt(i) - } - when (getLock || haveLock) { - io.cpu(i).req_data.ready := io.mem.req_data.ready - io.mem.req_data.valid := Bool(true) - io.mem.req_data.bits := io.cpu(i).req_data.bits - } - - io.cpu(i).resp.valid := io.mem.resp.valid && respWho === me - io.cpu(i).resp.bits := io.mem.resp.bits - io.cpu(i).resp.bits.tag := respTag - } - io.mem.resp.ready := io.cpu(respWho).resp.ready - - val unlock = Counter(io.mem.req_data.fire(), refill_cycles)._2 - when (unlock) { lock := Bool(false) } -} - -abstract class DRAMSideLLCLike extends Module { - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } -} - -// DRAMSideLLC has a known bug now. DO NOT USE. We are working on a brand new -// L2$. Stay stuned. -// -// Bug description: -// There's a race condition between the writeback unit (the module which -// sends the data out to the backside interface) and the data unit (the module -// which writes the data into the SRAM in the L2$). The "hit status" -// is saved in a register, however, is updated again when there's -// a transaction coming from the core without waiting until the writeback unit -// has sent out all its data to the outer memory system. This is why the -// problem manifests at a higher probability with the slow backup memory port. - -class DRAMSideLLC_HasKnownBug(sets: Int, ways: Int, outstanding: Int, refill_cycles: Int, tagLeaf: Mem[UInt], dataLeaf: Mem[UInt]) extends DRAMSideLLCLike -{ - val tagWidth = params(MIFAddrBits) - log2Up(sets) - val metaWidth = tagWidth + 2 // valid + dirty - - val memCmdArb = Module(new Arbiter(new MemReqCmd, 2)) - val dataArb = Module(new Arbiter(new LLCDataReq(ways), 2)) - val mshr = Module(new LLCMSHRFile(sets, ways, outstanding, refill_cycles)) - val tags = Module(new BigMem(sets, 0, 1, tagLeaf)(Bits(width = metaWidth*ways))) - val data = Module(new LLCData(4, sets, ways, refill_cycles, dataLeaf)) - val writeback = Module(new LLCWriteback(2, refill_cycles)) - - val initCount = Reg(init=UInt(0, log2Up(sets+1))) - val initialize = !initCount(log2Up(sets)) - when (initialize) { initCount := initCount + UInt(1) } - - val replay_s2 = Reg(init=Bool(false)) - val s2_valid = Reg(init=Bool(false)) - val s2 = Reg(new MemReqCmd) - val s3_rdy = Bool() - val replay_s2_rdy = Bool() - - val s1_valid = Reg(next = io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy, init = Bool(false)) - val s1 = Reg(new MemReqCmd) - when (io.cpu.req_cmd.fire()) { s1 := io.cpu.req_cmd.bits } - when (replay_s2 && replay_s2_rdy) { s1 := s2 } - - s2_valid := s1_valid - replay_s2 := s2_valid && !s3_rdy || replay_s2 && !replay_s2_rdy - val s2_tags = Vec.fill(ways){Reg(Bits(width = metaWidth))} - when (s1_valid) { - s2 := s1 - for (i <- 0 until ways) - s2_tags(i) := tags.io.rdata(metaWidth*(i+1)-1, metaWidth*i) - } - val s2_hits = s2_tags.map(t => t(tagWidth) && s2.addr(s2.addr.getWidth-1, s2.addr.getWidth-tagWidth) === t(tagWidth-1, 0)) - val s2_hit_way = OHToUInt(s2_hits) - val s2_hit = s2_hits.reduceLeft(_||_) - val s2_hit_dirty = s2_tags(s2_hit_way)(tagWidth+1) - val repl_way = LFSR16(s2_valid)(log2Up(ways)-1, 0) - val repl_tag = s2_tags(repl_way).toUInt - val setDirty = s2_valid && s2.rw && s2_hit && !s2_hit_dirty - - val tag_we = initialize || setDirty || mshr.io.tag.fire() - val tag_waddr = Mux(initialize, initCount, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)) - val tag_wdata = Cat(setDirty, !initialize, Mux(setDirty, s2.addr, mshr.io.tag.bits.addr)(mshr.io.tag.bits.addr.getWidth-1, mshr.io.tag.bits.addr.getWidth-tagWidth)) - val tag_wmask = Mux(initialize, SInt(-1, ways), UIntToOH(Mux(setDirty, s2_hit_way, mshr.io.tag.bits.way))) - tags.io.in.valid := io.cpu.req_cmd.fire() || replay_s2 && replay_s2_rdy || tag_we - tags.io.in.bits.addr := Mux(tag_we, tag_waddr, Mux(replay_s2, s2.addr, io.cpu.req_cmd.bits.addr)(log2Up(sets)-1,0)) - tags.io.in.bits.rw := tag_we - tags.io.in.bits.wdata := Fill(ways, tag_wdata) - tags.io.in.bits.wmask := FillInterleaved(metaWidth, tag_wmask) - - mshr.io.cpu.valid := s2_valid && !s2_hit && !s2.rw - mshr.io.cpu.bits := s2 - mshr.io.repl_way := repl_way - mshr.io.repl_dirty := repl_tag(tagWidth+1, tagWidth).andR - mshr.io.repl_tag := repl_tag - mshr.io.mem.resp := io.mem.resp - mshr.io.tag.ready := !s1_valid && !s2_valid - - data.io.req <> dataArb.io.out - data.io.mem_resp := io.mem.resp - data.io.mem_resp_set := mshr.io.mem_resp_set - data.io.mem_resp_way := mshr.io.mem_resp_way - data.io.req_data.bits := io.cpu.req_data.bits - data.io.req_data.valid := io.cpu.req_data.valid - - writeback.io.req(0) <> data.io.writeback - writeback.io.data(0) <> data.io.writeback_data - writeback.io.req(1).valid := s2_valid && !s2_hit && s2.rw && mshr.io.cpu.ready - writeback.io.req(1).bits := s2.addr - writeback.io.data(1).valid := io.cpu.req_data.valid - writeback.io.data(1).bits := io.cpu.req_data.bits - - memCmdArb.io.in(0) <> mshr.io.mem.req_cmd - memCmdArb.io.in(1) <> writeback.io.mem.req_cmd - - dataArb.io.in(0) <> mshr.io.data - dataArb.io.in(1).valid := s2_valid && s2_hit && mshr.io.cpu.ready - dataArb.io.in(1).bits := s2 - dataArb.io.in(1).bits.way := s2_hit_way - dataArb.io.in(1).bits.isWriteback := Bool(false) - - s3_rdy := mshr.io.cpu.ready && Mux(s2_hit, dataArb.io.in(1).ready, !s2.rw || writeback.io.req(1).ready) - replay_s2_rdy := s3_rdy && !tag_we - - io.cpu.resp <> data.io.resp - io.cpu.req_data.ready := writeback.io.data(1).ready || data.io.req_data.ready - io.mem.req_cmd <> memCmdArb.io.out - io.mem.req_data <> writeback.io.mem.req_data - io.cpu.req_cmd.ready := !(s1_valid || s2_valid || replay_s2 || tag_we || - io.cpu.req_cmd.bits.rw && io.cpu.req_data.ready) -} - -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) - - val do_flow = Bool() - val do_enq = io.enq.fire() && !do_flow - val do_deq = io.deq.fire() && !do_flow - - val maybe_full = Reg(init=Bool(false)) - val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 - when (do_enq != do_deq) { maybe_full := do_enq } - - val ptr_match = enq_ptr === deq_ptr - val empty = ptr_match && !maybe_full - val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) - do_flow := empty && io.deq.ready - - val ram = Mem(data, entries, seqRead = true) - val ram_addr = Reg(Bits()) - val ram_out_valid = Reg(Bool()) - ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits } - when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { - ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) - } - - io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) - io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) -} - -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - - val fq = Module(new HellaFlowQueue(entries)(data)) - io.enq <> fq.io.enq - io.deq <> Queue(fq.io.deq, 1, pipe = true) -} - -object HellaQueue -{ - def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { - val q = Module((new HellaQueue(entries)) { enq.bits.clone }) - q.io.enq.valid := enq.valid // not using <> so that override is allowed - q.io.enq.bits := enq.bits - enq.ready := q.io.enq.ready - q.io.deq - } -} - -class DRAMSideLLCNull(numRequests: Int, refillCycles: Int) extends DRAMSideLLCLike { - val numEntries = numRequests * refillCycles - val size = log2Down(numEntries) + 1 - - val inc = Bool() - val dec = Bool() - val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) - - when (inc && !dec) { - count := count + UInt(1) - } - when (!inc && dec) { - count := count - UInt(refillCycles) - } - when (inc && dec) { - count := count - UInt(refillCycles-1) - } - - val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark - - io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask - io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask - io.mem.req_cmd.bits := io.cpu.req_cmd.bits - - io.mem.req_data <> io.cpu.req_data - - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq - - inc := resp_dataq.io.deq.fire() - dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw -} diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 6f42f010..b470df49 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -286,3 +286,112 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { } } +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + require(isPow2(entries) && entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(init=Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val deq_ptr = Counter(do_deq, entries)._1 + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) + do_flow := empty && io.deq.ready + + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + + val fq = Module(new HellaFlowQueue(entries)(data)) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val cpu = new MemIO().flip + val mem = new MemPipeIO + } + + val numEntries = numRequests * refillCycles + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(init=UInt(numEntries, size)) + val watermark = count >= UInt(refillCycles) + + when (inc && !dec) { + count := count + UInt(1) + } + when (!inc && dec) { + count := count - UInt(refillCycles) + } + when (inc && dec) { + count := count - UInt(refillCycles-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_dataq.io.enq <> io.mem.resp + io.cpu.resp <> resp_dataq.io.deq + + inc := resp_dataq.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} + +class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { + val io = new Bundle { + val uncached = new UncachedTileLinkIO().flip + val mem = new MemPipeIO + } + + val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + a.io.uncached <> io.uncached + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + a.io.mem.resp <> b.io.cpu.resp + b.io.mem <> io.mem +} From d735f641102afb7805401dbc32c7e36a0b265904 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 2 Oct 2014 16:47:35 -0700 Subject: [PATCH 251/688] Parameter API update --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 8bb98f0a..51864fe1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -156,13 +156,13 @@ class Grant extends MasterSourcedMessage class Finish extends ClientSourcedMessage with HasMasterTransactionId -class UncachedTileLinkIO(p: Option[Parameters] = None) extends Bundle()(p) { +class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } -class TileLinkIO(p: Option[Parameters] = None) extends UncachedTileLinkIO(p) { +class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } From dc1a61264df319c05ac4c1d19eadabce9adfb7b6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 30 Sep 2014 14:48:02 -0700 Subject: [PATCH 252/688] initial version, acts like old hub --- uncore/src/main/scala/cache.scala | 499 ++++++++++++++++++++++++++++++ 1 file changed, 499 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 26157a75..ca6b5c3f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -96,3 +96,502 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } + +abstract trait L2HellaCacheParameters extends CacheParameters + with CoherenceAgentParameters + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors)) +} + +trait HasL2InternalRequestState extends L2HellaCacheBundle { + val tag_match = Bool() + val meta = new L2Metadata + val way_en = Bits(width = nWays) +} + +object L2Metadata { + def apply(tag: Bits, coh: MasterMetadata) = { + val meta = new L2Metadata + meta.tag := tag + meta.coh := coh + meta + } +} +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = co.masterMetadataOnFlush.clone +} + +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = tagBits) +} + +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) + with HasL2Id + +class L2MetaResp extends L2HellaCacheBundle + with HasL2Id + with HasL2InternalRequestState + +class L2MetadataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2MetaReadReq).flip + val write = Decoupled(new L2MetaWriteReq).flip + val resp = Valid(new L2MetaResp) + } + + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) + + val replacer = params(Replacer)() + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.meta := Mux(s2_tag_match, + L2Metadata(s2_repl_meta.tag, s2_hit_coh), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { + val way_en = Bits(width = nWays) + val addr = Bits(width = params(TLAddrBits)) +} + +class L2DataWriteReq extends L2DataReadReq { + val wmask = Bits(width = params(TLWriteMaskBits)) + val data = Bits(width = params(TLDataBits)) +} + +class L2DataResp extends Bundle with HasL2Id { + val data = Bits(width = params(TLDataBits)) +} + +class L2DataArray extends L2HellaCacheModule { + val io = new Bundle { + val read = Decoupled(new L2DataReadReq).flip + val write = Decoupled(new L2DataWriteReq).flip + val resp = Valid(new L2DataResp) + } + + val waddr = io.write.bits.addr + val raddr = io.read.bits.addr + val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val resp = (0 until nWays).map { w => + val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + when (io.write.bits.way_en(w) && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + } + array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + } + io.resp.valid := ShiftRegister(io.read.valid, 2) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + + io.read.ready := Bool(true) + io.write.ready := Bool(true) +} + +class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends + CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { + + require(isPow2(nSets)) + require(isPow2(nWays)) + require(refillCycles == 1) + + val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val meta = Module(new L2MetadataArray) + val data = Module(new L2DataArray) + + tshrfile.io.inner <> io.inner + tshrfile.io.meta_read <> meta.io.read + tshrfile.io.meta_write <> meta.io.write + tshrfile.io.meta_resp <> meta.io.resp + tshrfile.io.data_read <> data.io.read + tshrfile.io.data_write <> data.io.write + tshrfile.io.data_resp <> data.io.resp + io.outer <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent +} + + +class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + // Wiring helper funcs + def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + } + + // Create TSHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map { id => + Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + } ++ (nReleaseTransactors until nTransactors).map { id => + Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + } + + // Propagate incoherence flags + trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + + // Handle acquire transaction initiation + val acquire = io.inner.acquire + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + alloc_arb.io.out.ready := acquire.valid && !block_acquires + + // Handle probe requests + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Handle releases, which might be voluntary and might have data + val release = io.inner.release + val voluntary = co.isVoluntary(release.bits.payload) + val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) + val block_releases = Bool(false) + val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} + //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response + val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + + // Reply to initial requestor + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + + // Free finished transactions + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Arbitrate for the outer memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + {case TLId => outerId}) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + io.outer <> outer_arb.io.out + + // Local memory + doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) + doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) + doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) + doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) + doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) + doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + +} + + +abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_conflict = Bool(OUTPUT) + val meta_read = Decoupled(new L2MetaReadReq) + val meta_write = Decoupled(new L2MetaWriteReq) + val meta_resp = Valid(new L2MetaResp).flip + val data_read = Decoupled(new L2DataReadReq) + val data_write = Decoupled(new L2DataWriteReq) + val data_resp = Valid(new L2DataResp).flip + } + + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + val xact = Reg{ new Release } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + io.has_acquire_conflict := Bool(false) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) + + io.outer.grant.ready := Bool(false) + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, + UInt(trackerId), + xact.data), + { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + xact.client_xact_id, + UInt(trackerId)) + + io.data_read.valid := Bool(false) + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact := c_rel.payload + init_client_id := c_rel.header.src + state := s_mem + } + }/* + is(s_meta_read) { + when(io.meta_read.ready) state := s_meta_resp + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) + state := Mux(s_meta_write + Mux(co.messageHasData(xact), s_mem, s_ack) + }*/ + is(s_mem) { + io.outer.acquire.valid := Bool(true) + when(io.outer.acquire.ready) { state := s_ack } + } + is(s_ack) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val state = Reg(init=s_idle) + val xact = Reg{ new Acquire } + val xact_internal = Reg{ new L2MetaResp } + val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + //TODO: Will need id reg for merged release xacts + + val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + + val pending_outer_write = co.messageHasData(xact) + val pending_outer_read = co.requiresOuterRead(xact.a_type) + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, + xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), + { case TLId => outerId }) + + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + if (nClients > 1) { + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) + } + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := outer_read + io.outer.grant.ready := io.inner.grant.ready + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ + co.masterMetadataOnFlush), + xact.addr, + UInt(trackerId)) + + val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.payload := Grant(grant_type, + xact.client_xact_id, + UInt(trackerId), + m_gnt.payload.data) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + + io.data_read.valid := Bool(false) + io.data_read.bits.id := UInt(trackerId) + io.data_read.bits.way_en := xact_internal.way_en + io.data_read.bits.addr := xact.addr + io.data_write.valid := Bool(false) + io.data_write.bits.id := UInt(trackerId) + io.data_write.bits.way_en := xact_internal.way_en + io.data_write.bits.addr := xact.addr + io.data_write.bits.wmask := SInt(-1) + io.data_write.bits.data := xact.data + io.meta_read.valid := Bool(false) + io.meta_read.bits.id := UInt(trackerId) + io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_read.bits.tag := xact.addr >> UInt(untagBits) + io.meta_write.valid := Bool(false) + io.meta_write.bits.id := UInt(trackerId) + io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta_write.bits.way_en := xact_internal.way_en + io.meta_write.bits.data := xact_internal.meta + + when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + val needs_outer_write = co.messageHasData(c_acq.payload) + val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + when( io.inner.acquire.valid ) { + xact := c_acq.payload + init_client_id := c_acq.header.src + probe_flags := probe_initial_flags + if(nClients > 1) { + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) + } else state := Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant)) + } + } + // s_read_meta + // xact_internal := resp + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + when(io.inner.release.valid) { + //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( + // io.inner.release.bits.payload, + // xact_internal.meta.coh, + // init_client_id) + when(co.messageHasData(c_rel.payload)) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } .otherwise { + io.inner.release.ready := Bool(true) + if(nClients > 1) release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_read) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_mem_write) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_acq + when(io.outer.acquire.ready) { + state := Mux(pending_outer_read, s_mem_read, s_make_grant) + } + } + is(s_make_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + } + } + is(s_busy) { // Nothing left to do but wait for transaction to complete + when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + io.inner.grant.valid := Bool(true) + } + when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + state := s_idle + } + } + } +} From 394eb38a963588427fe900c7bffa369dc494ab88 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Oct 2014 01:06:33 -0700 Subject: [PATCH 253/688] temp; converted voluntary wb tracker --- uncore/src/main/scala/cache.scala | 56 ++++++++++++++------------- uncore/src/main/scala/coherence.scala | 4 +- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ca6b5c3f..b4e210b0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -122,7 +122,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = co.masterMetadataOnFlush.clone + val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -306,11 +306,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Reply to initial requestor doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - // Free finished transactions - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + // Free finished transactions on ack + val finish = io.inner.finish + val finish_idx = finish.bits.payload.master_xact_id + trackerList.zipWithIndex.map { case (t, i) => + t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) + } + trackerList.map(_.io.inner.finish.bits := finish.bits) + finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Arbitrate for the outer memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), @@ -353,7 +356,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -365,19 +368,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, - UInt(trackerId), - xact.data), - { case TLId => outerId }) + io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush),// TODO xact_internal.meta) + io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), xact.client_xact_id, UInt(trackerId)) @@ -397,8 +395,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid) { xact_internal := io.meta_resp.bits } + io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -406,25 +403,32 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou when( io.inner.release.valid ) { xact := c_rel.payload init_client_id := c_rel.header.src - state := s_mem + state := s_meta_read } - }/* + } is(s_meta_read) { - when(io.meta_read.ready) state := s_meta_resp + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal.meta := tl.co.masterMetadataOnRelease(xact, xact_internal.meta, init_client_id)) - state := Mux(s_meta_write - Mux(co.messageHasData(xact), s_mem, s_ack) - }*/ - is(s_mem) { - io.outer.acquire.valid := Bool(true) + xact_internal := io.meta_resp.bits + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) when(io.outer.acquire.ready) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + when(io.inner.grant.ready) { + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + s_busy, s_idle) + } + } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..778a04f6 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -62,10 +62,10 @@ abstract class DirectoryRepresentation extends Bundle { } class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) + val internal = UInt() def pop(id: UInt) = this def push(id: UInt) = this - def flush(dummy: Int = 0) = this + def flush(dummy: Int = 0) = { internal := UInt(0); this } def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) def count(dummy: Int = 0) = UInt(0) From 86bdbd65358e253f7742927f74da69c7e9af494e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 7 Oct 2014 22:33:10 -0700 Subject: [PATCH 254/688] new tshrs, compiles but does not elaborate --- uncore/src/main/scala/cache.scala | 275 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 204 ++++++++++--------- uncore/src/main/scala/tilelink.scala | 4 + 3 files changed, 293 insertions(+), 190 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b4e210b0..9cc99e4e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -353,10 +353,14 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + def mergeData(acq: Acquire, data: UInt): UInt = { + //TODO wite mask + Mux(co.messageHasData(acq), acq.data, data) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_ack :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -412,18 +416,26 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := s_meta_write + xact_internal := co.masterMetadataOnRelease(xact, + io.meta_resp.bits.meta.coh, + init_client_id) + state := Mux(io.meta_resp.bits.tag_match, + Mux(co.messageHasData(xact), s_data_write, s_meta_write), + s_grant) } } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { state := s_meta_write } + } is(s_meta_write) { io.meta_write.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + when(io.meta_write.ready) { state := s_grant } } - is(s_ack) { + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload.g_type), + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), s_busy, s_idle) } } @@ -434,61 +446,71 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) + val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val pending_probes = Reg(init = co.dir()) + val curr_p_id = pending_probes.next() + + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = co.needsWriteback(xact_internal.meta.coh) + val is_hit = co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + val c_rel_had_data = Reg{Bool()} + val c_rel_was_voluntary = Reg{Bool()} + val wb_buffer = Reg{xact.data.clone} + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) - + xact.addr, + UInt(trackerId), + xact.data), { case TLId => outerId }) + val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, + Cat(xact_internal.meta.tag, + xact.addr(untagBits-1,blockOffBits)), + UInt(trackerId), + wb_buffer), { case TLId => outerId }) + val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, + xact.addr, + UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.acquire.bits.payload := outer_read - io.outer.grant.ready := io.inner.grant.ready + io.outer.grant.ready := Bool(true) //grant.data -> xact.data + val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + xact.addr, + UInt(trackerId)) + val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, + xact.addr, + UInt(trackerId)) + //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, /*xact_internal.meta), TODO*/ - co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Mux(needs_writeback, + inner_probe_wb, + inner_probe_cacq) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush)// TODO xact_internal.meta) + val grant_type = co.getGrantType(xact, xact_internal.meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(grant_type, xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + xact.data) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -511,91 +533,154 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta - - when(io.meta_resp.valid && io.meta_resp.bits.id === UInt(trackerId)) { xact_internal := io.meta_resp.bits } + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := xact_internal.meta.coh switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src - probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + state := s_meta_read } } - // s_read_meta - // xact_internal := resp - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + is(s_meta_read) { + io.meta_read.valid := Bool(true) + when(io.meta_read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta_resp.valid) { + val coh = io.meta_resp.bits.meta.coh + val _tag_match = io.meta_resp.bits.tag_match + val _needs_writeback = co.needsWriteback(coh) + val _is_hit = co.isHit(xact, coh) + val _needs_probes = co.requiresProbes(xact.a_type, coh) + xact_internal := io.meta_resp.bits + when(!_needs_writeback) { + xact_internal.meta.coh := co.masterMetadataOnFlush + } + when(_needs_probes) { + pending_probes := coh.sharers + release_count := coh.sharers.count() + c_rel_had_data := Bool(false) + c_rel_was_voluntary := Bool(false) + } + state := Mux(_tag_match, + Mux(_is_hit, + Mux(_needs_probes, s_probe, s_data_read_hit), + Mux(_needs_probes, s_probe, s_outer_read)), + Mux(_needs_writeback, + Mux(_needs_probes, s_probe, s_data_read_wb), + s_outer_read)) } + } + is(s_probe) { + val skip = io.tile_incoherent(curr_p_id) || + ((curr_p_id === init_client_id) && + !co.requiresSelfProbe(xact.a_type)) + io.inner.probe.valid := !(pending_probes.none() || skip) + when(io.inner.probe.ready || skip) { + pending_probes.pop(curr_p_id) + } + when(skip) { release_count := release_count - UInt(1) } - // Handle releases, which may have data to be written back + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - //xact_internal.meta.coh := tl.co.masterMetadataOnRelease( - // io.inner.release.bits.payload, - // xact_internal.meta.coh, - // init_client_id) + xact_internal.meta.coh := co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) when(co.messageHasData(c_rel.payload)) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } + c_rel_had_data := Bool(true) + when(tag_match) { + xact.data := mergeData(xact, io.inner.release.bits.payload.data) + } .otherwise { + wb_buffer := io.inner.release.bits.payload.data } - } .otherwise { - io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + } + when(co.isVoluntary(c_rel.payload)) { + c_rel_was_voluntary := Bool(true) + } + when(!co.isVoluntary(c_rel.payload)) { + release_count := release_count - Mux(skip, UInt(2), UInt(1)) when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + state := Mux(tag_match, + Mux(is_hit, + Mux(c_rel_had_data, s_meta_write, s_data_read_hit), + s_outer_read), + Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) } } } } - is(s_mem_read) { + is(s_data_read_wb) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_wb + } + } + is(s_data_resp_wb) { + when(io.data_resp.valid) { + wb_buffer := io.data_resp.bits + state := s_outer_write_wb + } + } + is(s_outer_write_wb) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_wb + when(io.outer.acquire.ready) { + state := s_outer_read + } + } + is(s_outer_read) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := s_outer_resp } } - is(s_mem_write) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) + is(s_outer_resp) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + xact.data := mergeData(xact, io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_internal.meta.coh + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) } } - is(s_make_grant) { + is(s_data_read_hit) { + io.data_read.valid := Bool(true) + when(io.data_read.ready) { + state := s_data_resp_hit + } + } + is(s_data_resp_hit) { + when(io.data_resp.valid) { + xact.data := mergeData(xact, io.data_resp.bits.data) + state := s_meta_write + } + } + is(s_data_write) { + io.data_write.valid := Bool(true) + when(io.data_write.ready) { + state := s_meta_write + } + } + is(s_meta_write) { + io.meta_write.valid := Bool(true) + when(io.meta_write.ready) { state := s_grant } + } + is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(c_gnt.payload.g_type), + s_busy, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { - state := s_idle - } + is(s_busy) { + when(io.inner.finish.valid) { state := s_idle } } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 778a04f6..5dd87b35 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -106,6 +105,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool + def needsWriteback(m: MasterMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata @@ -114,33 +114,33 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata def masterMetadataOnFlush: MasterMetadata def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata + def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) def messageHasData (rel: SourcedMessage): Bool def messageUpdatesDataArray (reply: Grant): Bool def messageIsUncached(acq: Acquire): Bool - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool + def requiresOuterRead(a_type: UInt): Bool def requiresOuterWrite(a_type: UInt): Bool def requiresSelfProbe(a_type: UInt): Bool + def requiresProbes(a_type: UInt, m: MasterMetadata): Bool def requiresAckForGrant(g_type: UInt): Bool def requiresAckForRelease(r_type: UInt): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } trait UncachedTransactions { @@ -191,6 +191,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -214,16 +215,19 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached )) } @@ -292,6 +296,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -299,11 +304,10 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -346,6 +350,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -374,18 +379,20 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata(Mux(popped.none(), masterInvalid, masterValid), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -462,6 +469,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -469,25 +477,25 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 3 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -522,6 +530,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -549,20 +558,21 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -612,7 +622,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -635,6 +645,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi acquireWriteUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -642,25 +653,25 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 4 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 7 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -695,6 +706,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) @@ -723,20 +735,21 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW probeCopy -> m.state )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached )) } @@ -786,7 +799,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -813,6 +826,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW acquireAtomicUncached -> probeInvalidate )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached) @@ -820,26 +834,26 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { def nClientStates = 7 - def nMasterStates = 3 + def nMasterStates = 2 def nAcquireTypes = 8 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 9 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -873,6 +887,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } + def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -914,25 +929,24 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientMigratoryDirty -> clientInvalid)) )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt) = { - val popped = m.sharers.pop(src) - val next = MasterMetadata( - Mux(popped.none(), masterInvalid, - Mux(popped.one(), masterExclusive, masterShared)), - popped)(this) - def is(r: UInt) = incoming.r_type === r + def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { + val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) MuxBundle(m, Array( - is(releaseVoluntaryInvalidateData) -> next, - is(releaseInvalidateData) -> next, - is(releaseDowngradeData) -> m, - is(releaseCopyData) -> m, - is(releaseInvalidateAck) -> next, - is(releaseDowngradeAck) -> m, - is(releaseCopyAck) -> m, - is(releaseDowngradeDataMigratory) -> m, - is(releaseDowngradeAckHasCopy) -> m, - is(releaseInvalidateDataMigratory) -> next, - is(releaseInvalidateAckMigratory) -> next + r.is(releaseVoluntaryInvalidateData) -> next, + r.is(releaseInvalidateData) -> next, + r.is(releaseInvalidateAck) -> next, + r.is(releaseInvalidateDataMigratory) -> next, + r.is(releaseInvalidateAckMigratory) -> next + )) + } + def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { + val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val uncached = MasterMetadata(masterValid, m.sharers)(this) + MuxBundle(uncached, Array( + g.is(grantReadShared) -> cached, + g.is(grantReadExclusive) -> cached, + g.is(grantReadExclusiveAck) -> cached, + g.is(grantReadMigratory) -> cached )) } @@ -983,7 +997,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1012,6 +1026,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> probeInvalidateOthers )) } + def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate def requiresOuterRead(a_type: UInt) = { (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) @@ -1019,12 +1034,11 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresOuterWrite(a_type: UInt) = { (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached + def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..46ae899e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -77,6 +77,7 @@ class Acquire extends ClientSourcedMessage val write_mask = Bits(width = params(TLWriteMaskBits)) val subword_addr = Bits(width = params(TLWordAddrBits)) val atomic_opcode = Bits(width = params(TLAtomicOpBits)) + def is(t: UInt) = a_type === t } object Probe @@ -94,6 +95,7 @@ class Probe extends MasterSourcedMessage with HasPhysicalAddress with HasMasterTransactionId { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + def is(t: UInt) = p_type === t } object Release @@ -127,6 +129,7 @@ class Release extends ClientSourcedMessage with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + def is(t: UInt) = r_type === t } object Grant @@ -151,6 +154,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + def is(t: UInt) = g_type === t } class Finish extends ClientSourcedMessage with HasMasterTransactionId From 044b19dbc1de7dfeb1d9c15a76e937ef4bdc0a0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 15 Oct 2014 11:46:35 -0700 Subject: [PATCH 255/688] Compiles and elaborates, does not pass asm tests --- uncore/src/main/scala/cache.scala | 31 +++++---- uncore/src/main/scala/coherence.scala | 93 +++++++++++++-------------- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9cc99e4e..1c6fc044 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -130,8 +130,9 @@ class L2MetaReadReq extends MetaReadReq with HasL2Id { } class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) - with HasL2Id - + with HasL2Id { + override def clone = new L2MetaWriteReq().asInstanceOf[this.type] +} class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -360,7 +361,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -399,7 +400,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data := xact_internal.meta - io.meta_resp.valid := Bool(true) switch (state) { is(s_idle) { @@ -450,21 +450,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } + val test = Reg{UInt()} val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts val release_count = Reg(init = UInt(0, width = log2Up(nClients))) - val pending_probes = Reg(init = co.dir()) - val curr_p_id = pending_probes.next() + val pending_probes = Reg(init = co.dir().flush) + val curr_p_id = co.dir().next(pending_probes) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - val c_rel_had_data = Reg{Bool()} - val c_rel_was_voluntary = Reg{Bool()} + val c_rel_had_data = Reg(init = Bool(false)) + val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && @@ -557,12 +558,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits + test := UInt(0) when(!_needs_writeback) { - xact_internal.meta.coh := co.masterMetadataOnFlush +// xact_internal.meta.coh := co.masterMetadataOnFlush + test := UInt(12) } when(_needs_probes) { pending_probes := coh.sharers - release_count := coh.sharers.count() + release_count := co.dir().count(coh.sharers) c_rel_had_data := Bool(false) c_rel_was_voluntary := Bool(false) } @@ -579,19 +582,21 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val skip = io.tile_incoherent(curr_p_id) || ((curr_p_id === init_client_id) && !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(pending_probes.none() || skip) + io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) when(io.inner.probe.ready || skip) { - pending_probes.pop(curr_p_id) + co.dir().pop(pending_probes, curr_p_id) } when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { +/* xact_internal.meta.coh := co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, c_rel.header.src) +*/ when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { @@ -623,7 +628,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_resp_wb) { when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits + wb_buffer := io.data_resp.bits.data state := s_outer_write_wb } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 5dd87b35..2bb6ec01 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -28,10 +28,10 @@ object MasterMetadata { def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = new MasterMetadata m.state := state - m.sharers.flush() + m.sharers := c.dir().flush m } - def apply(state: UInt, sharers: DirectoryRepresentation)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { val m = apply(state) m.sharers := sharers m @@ -39,7 +39,7 @@ object MasterMetadata { } class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { val state = UInt(width = c.masterStateWidth) - val sharers = c.dir() + val sharers = UInt(width = c.dir().width) override def clone = new MasterMetadata()(c).asInstanceOf[this.type] } /* @@ -50,37 +50,34 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe } */ -abstract class DirectoryRepresentation extends Bundle { - def pop(id: UInt): DirectoryRepresentation - def push(id: UInt): DirectoryRepresentation - def flush(dummy: Int = 0): DirectoryRepresentation - def none(dummy: Int = 0): Bool - def one(dummy: Int = 0): Bool - def count(dummy: Int = 0): UInt - def next(dummy: Int = 0): UInt +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt() - def pop(id: UInt) = this - def push(id: UInt) = this - def flush(dummy: Int = 0) = { internal := UInt(0); this } - def none(dummy: Int = 0) = Bool(false) - def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) - def next(dummy: Int = 0) = UInt(0) +class NullRepresentation extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(0) + def next(s: UInt) = UInt(0) } -class FullRepresentation(nClients: Int) extends DirectoryRepresentation { - val internal = UInt(width = nClients) - def pop(id: UInt) = { internal := internal & ~UIntToOH(id); this } // make new FullRep to return? - def push(id: UInt) = { internal := internal | UIntToOH(id); this } - def flush(dummy: Int = 0) = { internal := UInt(0, width = nClients); this } - def none(dummy: Int = 0) = internal === UInt(0) - def one(dummy: Int = 0) = PopCount(internal) === UInt(1) - def count(dummy: Int = 0) = PopCount(internal) - def next(dummy: Int = 0) = PriorityEncoder(internal) - override def clone = new FullRepresentation(nClients).asInstanceOf[this.type] +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -216,7 +213,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -224,7 +221,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached @@ -307,7 +304,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -380,7 +377,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -388,7 +385,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadExclusive) -> cached, @@ -480,7 +477,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -559,7 +556,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -567,7 +564,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -622,7 +619,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -656,7 +653,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -736,7 +733,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -744,7 +741,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -799,7 +796,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -838,7 +835,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -930,7 +927,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, m.sharers.pop(src))(this) + val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -940,7 +937,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, m.sharers.push(dst))(this) + val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) MuxBundle(uncached, Array( g.is(grantReadShared) -> cached, @@ -997,7 +994,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(!m.sharers.none(), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? + acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? acquireReadExclusive -> grantReadExclusive, acquireReadUncached -> grantReadUncached, acquireWriteUncached -> grantWriteUncached, @@ -1038,7 +1035,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck def requiresAckForRelease(r_type: UInt) = Bool(false) def requiresSelfProbe(a_type: UInt) = a_type === acquireReadUncached - def requiresProbes(a_type: UInt, m: MasterMetadata) = !m.sharers.none() + def requiresProbes(a_type: UInt, m: MasterMetadata) = !dir().none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From a891ba1d460c832d97d8e5b6e7865344eb997755 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Oct 2014 17:40:30 -0700 Subject: [PATCH 256/688] more correct handling of internal state --- uncore/src/main/scala/cache.scala | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1c6fc044..4ca6b53e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,11 +459,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) + + val next_coh_on_release = co.masterMetadataOnRelease( + c_rel.payload, + xact_internal.meta.coh, + c_rel.header.src) + val next_coh_on_grant = co.masterMetadataOnGrant( + c_gnt.payload, + xact_internal.meta.coh, + c_gnt.header.dst) val is_uncached = co.messageIsUncached(xact) val tag_match = xact_internal.tag_match val needs_writeback = co.needsWriteback(xact_internal.meta.coh) val is_hit = co.isHit(xact, xact_internal.meta.coh) val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + val c_rel_had_data = Reg(init = Bool(false)) val c_rel_was_voluntary = Reg(init = Bool(false)) val wb_buffer = Reg{xact.data.clone} @@ -535,7 +546,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := xact_internal.meta.coh + io.meta_write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -558,11 +569,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits - test := UInt(0) - when(!_needs_writeback) { -// xact_internal.meta.coh := co.masterMetadataOnFlush - test := UInt(12) - } when(_needs_probes) { pending_probes := coh.sharers release_count := co.dir().count(coh.sharers) @@ -591,12 +597,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { -/* - xact_internal.meta.coh := co.masterMetadataOnRelease( - c_rel.payload, - xact_internal.meta.coh, - c_rel.header.src) -*/ + xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { c_rel_had_data := Bool(true) when(tag_match) { From 27c72e5eedd708fe98c73aabd39124f48a099177 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 23 Oct 2014 21:50:03 -0700 Subject: [PATCH 257/688] nearly all isa tests pass --- uncore/src/main/scala/cache.scala | 152 ++++++++++++++------------ uncore/src/main/scala/coherence.scala | 14 +++ uncore/src/main/scala/memserdes.scala | 2 +- 3 files changed, 97 insertions(+), 71 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4ca6b53e..6b5e9ba8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -157,13 +157,13 @@ class L2MetadataArray extends L2HellaCacheModule { val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - //val s2_hit = s2_tag_match && tl.co.isHit(s2_req.cmd, s2_hit_state) && s2_hit_state === tl.co.newStateOnHit(s2_req.cmd, s2_hit_state) val replacer = params(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + when(!s2_tag_match) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) @@ -197,9 +197,9 @@ class L2DataArray extends L2HellaCacheModule { val waddr = io.write.bits.addr val raddr = io.read.bits.addr - val wmask = FillInterleaved(wordBits, io.write.bits.wmask) + val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=params(RowBits)), nSets*refillCycles, seqRead = true) + val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } @@ -258,7 +258,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := UInt(i) === in.bits.id } + outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } } // Create TSHRs for outstanding transactions @@ -295,8 +295,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -361,7 +360,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_meta_write :: s_data_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg{ new Release } val xact_internal = Reg{ new L2MetaResp } @@ -373,11 +372,12 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id io.inner.grant.bits.payload := Grant(co.getGrantType(xact, xact_internal.meta.coh), @@ -399,7 +399,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta_write.bits.id := UInt(trackerId) io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data := xact_internal.meta + io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + xact_internal.meta.coh, + init_client_id) switch (state) { is(s_idle) { @@ -416,9 +419,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta_resp.valid) { - xact_internal := co.masterMetadataOnRelease(xact, - io.meta_resp.bits.meta.coh, - init_client_id) + xact_internal := io.meta_resp.bits state := Mux(io.meta_resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) @@ -440,6 +441,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } @@ -450,16 +452,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact = Reg{ new Acquire } val xact_internal = Reg{ new L2MetaResp } - val test = Reg{UInt()} - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - + val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_had_data = Reg(init = Bool(false)) + val crel_was_voluntary = Reg(init = Bool(false)) + val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) + val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) + val wb_buffer = Reg{xact.data.clone} - val release_count = Reg(init = UInt(0, width = log2Up(nClients))) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - + val is_uncached = co.messageIsUncached(xact) + val tag_match = xact_internal.tag_match + val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) + val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) + val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) + //TODO: does allocate + + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + (state != s_idle) //TODO: Also indexes + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (state != s_idle) //TODO: Also indexes? + val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, xact_internal.meta.coh, @@ -468,29 +483,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St c_gnt.payload, xact_internal.meta.coh, c_gnt.header.dst) - val is_uncached = co.messageIsUncached(xact) - val tag_match = xact_internal.tag_match - val needs_writeback = co.needsWriteback(xact_internal.meta.coh) - val is_hit = co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact.a_type, xact_internal.meta.coh) - //TODO: does allocate - - val c_rel_had_data = Reg(init = Bool(false)) - val c_rel_was_voluntary = Reg(init = Bool(false)) - val wb_buffer = Reg{xact.data.clone} - - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(Acquire(co.getUncachedWriteAcquireType, - Cat(xact_internal.meta.tag, - xact.addr(untagBits-1,blockOffBits)), + addr_wb, UInt(trackerId), wb_buffer), { case TLId => outerId }) val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, @@ -501,31 +501,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val inner_probe_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr, UInt(trackerId)) - val inner_probe_wb = Probe(co.getProbeTypeOnVoluntaryWriteback, - xact.addr, + val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, + addr_wb, UInt(trackerId)) //TODO inner_probe_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(needs_writeback, - inner_probe_wb, - inner_probe_cacq) + io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, + cprb_for_mwb, + cprb_for_cacq) - val grant_type = co.getGrantType(xact, xact_internal.meta.coh) + val cgnt_for_cacq = Grant(co.getGrantType(xact, xact_internal.meta.coh), + xact.client_xact_id, + UInt(trackerId), + xact.data) + val cgnt_for_cwb = Grant(crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, - xact.client_xact_id, - UInt(trackerId), - xact.data) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) io.data_read.valid := Bool(false) io.data_read.bits.id := UInt(trackerId) @@ -565,15 +567,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(io.meta_resp.valid) { val coh = io.meta_resp.bits.meta.coh val _tag_match = io.meta_resp.bits.tag_match - val _needs_writeback = co.needsWriteback(coh) - val _is_hit = co.isHit(xact, coh) + val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact.a_type, coh) xact_internal := io.meta_resp.bits when(_needs_probes) { - pending_probes := coh.sharers - release_count := co.dir().count(coh.sharers) - c_rel_had_data := Bool(false) - c_rel_was_voluntary := Bool(false) + val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_self = mask_incoherent & + ~(!(co.requiresSelfProbe(xact.a_type) || _needs_writeback) << init_client_id) + pending_probes := mask_self + release_count := co.dir().count(mask_self) + crel_had_data := Bool(false) + crel_was_voluntary := Bool(false) } state := Mux(_tag_match, Mux(_is_hit, @@ -585,21 +590,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - val skip = io.tile_incoherent(curr_p_id) || - ((curr_p_id === init_client_id) && - !co.requiresSelfProbe(xact.a_type)) - io.inner.probe.valid := !(co.dir().none(pending_probes) || skip) - when(io.inner.probe.ready || skip) { - co.dir().pop(pending_probes, curr_p_id) + io.inner.probe.valid := !co.dir().none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir().pop(pending_probes, curr_p_id) } - when(skip) { release_count := release_count - UInt(1) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_internal.meta.coh := next_coh_on_release when(co.messageHasData(c_rel.payload)) { - c_rel_had_data := Bool(true) + crel_had_data := Bool(true) when(tag_match) { xact.data := mergeData(xact, io.inner.release.bits.payload.data) } .otherwise { @@ -607,19 +608,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } when(co.isVoluntary(c_rel.payload)) { - c_rel_was_voluntary := Bool(true) + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantType(c_rel.payload, xact_internal.meta.coh) } when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - Mux(skip, UInt(2), UInt(1)) - when(release_count === UInt(1)) { - state := Mux(tag_match, - Mux(is_hit, - Mux(c_rel_had_data, s_meta_write, s_data_read_hit), - s_outer_read), - Mux(c_rel_had_data, s_outer_write_wb, s_data_read_wb)) - } + release_count := release_count - UInt(1) } } + + when(crel_was_voluntary) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + crel_was_voluntary := Bool(false) + } + } + + when(release_count === UInt(0) && !crel_was_voluntary) { + state := Mux(tag_match, + Mux(is_hit, + Mux(crel_had_data, s_data_write, s_data_read_hit), + s_outer_read), + Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) + } } is(s_data_read_wb) { io.data_read.valid := Bool(true) @@ -686,6 +697,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_busy) { + io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2bb6ec01..5afcccfb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -58,6 +58,7 @@ abstract class DirectoryRepresentation(val width: Int) { def one(s: UInt): Bool def count(s: UInt): UInt def next(s: UInt): UInt + def full(s: UInt): UInt } class NullRepresentation extends DirectoryRepresentation(1) { @@ -68,6 +69,7 @@ class NullRepresentation extends DirectoryRepresentation(1) { def one(s: UInt) = Bool(false) def count(s: UInt) = UInt(0) def next(s: UInt) = UInt(0) + def full(s: UInt) = UInt(0) } class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { @@ -78,6 +80,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def one(s: UInt) = PopCount(s) === UInt(1) def count(s: UInt) = PopCount(s) def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s } abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { @@ -147,6 +150,7 @@ trait UncachedTransactions { def getUncachedWriteWordAcquireType: UInt def getUncachedAtomicAcquireType: UInt def isUncachedReadTransaction(acq: Acquire): Bool + def getUncachedReadGrantType: UInt } abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) @@ -234,6 +238,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -399,6 +405,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -579,6 +587,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -756,6 +766,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck @@ -954,6 +966,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getUncachedWriteWordAcquireType = acquireWriteWordUncached def getUncachedAtomicAcquireType = acquireAtomicUncached def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached + def getUncachedReadGrantType = grantReadUncached + def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..fc4e0906 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(co.getUncachedReadGrantType, tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) From 10309849b7b7080b91281bf3cb5c95d5b5680900 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 5 Nov 2014 14:21:38 -0800 Subject: [PATCH 258/688] Remove master_xact_id from Probe and Release --- uncore/src/main/scala/tilelink.scala | 35 ++++++++++------------------ uncore/src/main/scala/uncore.scala | 8 +++---- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 51864fe1..6738deaf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -79,52 +79,41 @@ class Acquire extends ClientSourcedMessage val atomic_opcode = Bits(width = params(TLAtomicOpBits)) } -object Probe -{ - def apply(p_type: UInt, addr: UInt, master_xact_id: UInt) = { + +object Probe { + def apply(p_type: UInt, addr: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr := addr - prb.master_xact_id := master_xact_id prb } } class Probe extends MasterSourcedMessage - with HasPhysicalAddress - with HasMasterTransactionId { + with HasPhysicalAddress { val p_type = UInt(width = params(TLCoherence).probeTypeWidth) } -object Release -{ - def apply(r_type: UInt, addr: UInt, data: UInt): Release = { - val rel = new Release - rel.r_type := r_type - rel.addr := addr - rel.data := data - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt): Release = { +object Release { + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { val rel = new Release rel.r_type := r_type rel.addr := addr rel.client_xact_id := client_xact_id - rel.master_xact_id := master_xact_id - rel.data := UInt(0) - rel - } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Release = { - val rel = apply(r_type, addr, client_xact_id, master_xact_id) rel.data := data rel } + def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { + apply(r_type, addr, client_xact_id, UInt(0)) + } + def apply(r_type: UInt, addr: UInt): Release = { + apply(r_type, addr, UInt(0), UInt(0)) + } } class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId - with HasMasterTransactionId with HasTileLinkData { val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 37e535f8..e1f59e08 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -62,8 +62,8 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - //val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), release.bits.payload.master_xact_id) // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response - val release_idx = Mux(voluntary, UInt(0), release.bits.payload.master_xact_id) + val release_idx = Mux(voluntary, UInt(0), conflict_idx) + // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits @@ -200,9 +200,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), - xact.addr, - UInt(trackerId)) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) From 35553cc0b71ffe63b1ea0b9d775e691582376db5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 16:05:25 -0800 Subject: [PATCH 259/688] NullDirectory sharers.count fix --- uncore/src/main/scala/coherence.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 7dbff4aa..9f94583a 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -51,7 +51,6 @@ class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends Cohe */ abstract class DirectoryRepresentation extends Bundle { - val internal: UInt def pop(id: UInt): DirectoryRepresentation def push(id: UInt): DirectoryRepresentation def flush(dummy: Int = 0): DirectoryRepresentation @@ -61,14 +60,13 @@ abstract class DirectoryRepresentation extends Bundle { def next(dummy: Int = 0): UInt } -class NullRepresentation extends DirectoryRepresentation { - val internal = UInt(0) +class NullRepresentation(nClients: Int) extends DirectoryRepresentation { def pop(id: UInt) = this def push(id: UInt) = this def flush(dummy: Int = 0) = this def none(dummy: Int = 0) = Bool(false) def one(dummy: Int = 0) = Bool(false) - def count(dummy: Int = 0) = UInt(0) + def count(dummy: Int = 0) = UInt(nClients) def next(dummy: Int = 0) = UInt(0) } From 82155f333e96bfb5eaa3b3b25ff7184609e39a17 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 11 Nov 2014 17:36:55 -0800 Subject: [PATCH 260/688] Major tilelink revision for uncached message types --- uncore/src/main/scala/coherence.scala | 566 ++++++++++---------------- uncore/src/main/scala/consts.scala | 2 + uncore/src/main/scala/htif.scala | 6 +- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 179 +++++--- uncore/src/main/scala/uncore.scala | 31 +- 6 files changed, 351 insertions(+), 435 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9f94583a..a2af20cb 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -120,59 +120,48 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantType(r: Release, m: MasterMetadata): UInt - //def getGrantType(a: Acquire) = getGrantType(a, new NullRepresentation) // TODO - //def getGrantType(r: Release) = getGrantType(r, new NullRepresentation) + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt - def messageHasData (rel: SourcedMessage): Bool - def messageUpdatesDataArray (reply: Grant): Bool - def messageIsUncached(acq: Acquire): Bool + def messageHasData(rel: SourcedMessage): Bool + def messageUpdatesDataArray(g: Grant): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterWrite(a_type: UInt, m: MasterMetadata): Bool - def requiresOuterRead(a_type: UInt): Bool - def requiresOuterWrite(a_type: UInt): Bool - def requiresSelfProbe(a_type: UInt): Bool - def requiresAckForGrant(g_type: UInt): Bool - def requiresAckForRelease(r_type: UInt): Bool + def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresSelfProbe(a: Acquire): Bool + def requiresAckForGrant(g: Grant): Bool + def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool + + def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) + } } -trait UncachedTransactions { - def getUncachedReadAcquireType: UInt - def getUncachedWriteAcquireType: UInt - def getUncachedReadWordAcquireType: UInt - def getUncachedWriteWordAcquireType: UInt - def getUncachedAtomicAcquireType: UInt - def isUncachedReadTransaction(acq: Acquire): Bool -} - -abstract class CoherencePolicyWithUncached(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) - with UncachedTransactions - -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 - def nAcquireTypes = 6 + def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 - def nGrantTypes = 7 + def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid @@ -197,15 +186,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit M_CLN -> clientValid )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> clientValid, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -225,14 +207,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive @@ -251,80 +227,61 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWit } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } + def getGrantType(a: Acquire, m: MasterMetadata): UInt = + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 8 + def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid @@ -332,7 +289,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -355,16 +312,9 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusiveDirty, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -387,14 +337,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) @@ -419,83 +363,62 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadExclusiveClean -> grantReadExclusive, - acquireReadExclusiveDirty -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadExclusiveClean -> probeInvalidate, - acquireReadExclusiveDirty -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), probeInvalidate) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), @@ -508,7 +431,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -530,16 +453,9 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )))(this) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> clientExclusiveDirty, - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + Mux(incoming.g_type === grantReadShared, clientShared, + clientExclusiveDirty)))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -564,14 +480,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -596,79 +506,69 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWi } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) - } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeInvalidate, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 3 - def nAcquireTypes = 7 + def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 - def nGrantTypes = 9 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) def isHit (cmd: UInt, m: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), @@ -681,7 +581,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -704,16 +604,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantReadExclusiveAck -> clientExclusiveDirty + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -738,14 +635,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW )) } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) @@ -770,83 +661,68 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyW } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = { - (reply.g_type === grantReadShared || reply.g_type === grantReadExclusive) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) } - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + Mux(a.a_type === acquireReadShared, + Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + grantReadExclusive)) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicyWithUncached(dir) { +class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 3 - def nAcquireTypes = 8 + def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 - def nGrantTypes = 9 + def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val acquireReadShared :: acquireReadExclusive :: acquireReadUncached :: acquireWriteUncached :: acquireReadWordUncached :: acquireWriteWordUncached :: acquireAtomicUncached :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadUncached :: grantWriteUncached :: grantReadExclusiveAck :: grantReadWordUncached :: grantWriteWordUncached :: grantAtomicUncached :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val uncachedAcquireTypeVec = Vec(acquireReadUncached, acquireWriteUncached, acquireReadWordUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataAcquireTypeVec = Vec(acquireWriteUncached, acquireWriteWordUncached, acquireAtomicUncached) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadUncached, grantReadMigratory, grantReadWordUncached, grantAtomicUncached) + val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) def isHit (cmd: UInt, m: ClientMetadata): Bool = { @@ -859,7 +735,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def isValid (m: MasterMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && messageIsUncached(outstanding)) || + (isRead(cmd) && outstanding.uncached) || (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) } def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -883,22 +759,18 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )))(this) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadUncached -> clientInvalid, - grantWriteUncached -> clientInvalid, - grantReadWordUncached -> clientInvalid, - grantWriteWordUncached -> clientInvalid, - grantAtomicUncached -> clientInvalid, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))(this) + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantReadShared -> clientShared, + grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireReadExclusive -> clientExclusiveDirty, + acquireReadShared -> clientExclusiveClean)), + grantReadExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireReadExclusive -> clientMigratoryDirty, + acquireReadShared -> clientMigratoryClean)) + ))))(this) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -935,14 +807,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } - def getUncachedReadAcquireType = acquireReadUncached - def getUncachedWriteAcquireType = acquireWriteUncached - def getUncachedReadWordAcquireType = acquireReadWordUncached - def getUncachedWriteWordAcquireType = acquireWriteWordUncached - def getUncachedAtomicAcquireType = acquireAtomicUncached - def isUncachedReadTransaction(acq: Acquire) = acq.a_type === acquireReadUncached def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = gnt.g_type === grantVoluntaryAck + def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) @@ -969,60 +835,50 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo } def messageHasData(msg: SourcedMessage) = msg match { - case acq: Acquire => hasDataAcquireTypeVec.contains(acq.a_type) - case grant: Grant => hasDataGrantTypeVec.contains(grant.g_type) + case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) + case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) case _ => Bool(false) } - def messageUpdatesDataArray (reply: Grant): Bool = Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(reply.g_type) - def messageIsUncached(acq: Acquire): Bool = uncachedAcquireTypeVec.contains(acq.a_type) + def messageUpdatesDataArray(g: Grant): Bool = { + Mux(g.uncached, Bool(false), + Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, grantReadUncached, Array( - acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), //TODO: what is count? Depend on release.p_type??? - acquireReadExclusive -> grantReadExclusive, - acquireReadUncached -> grantReadUncached, - acquireWriteUncached -> grantWriteUncached, - acquireReadWordUncached -> grantReadWordUncached, - acquireWriteWordUncached -> grantWriteWordUncached, - acquireAtomicUncached -> grantAtomicUncached, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI? - )) + Mux(a.uncached, getGrantTypeForUncached(a, m), + MuxLookup(a.a_type, grantReadShared, Array( + acquireReadShared -> Mux(m.sharers.count() > UInt(0), grantReadShared, grantReadExclusive), + acquireReadExclusive -> grantReadExclusive, + acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? + ))) } - def getGrantType(r: Release, m: MasterMetadata): UInt = { - MuxLookup(r.r_type, grantReadUncached, Array( - releaseVoluntaryInvalidateData -> grantVoluntaryAck - )) - } - + def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck def getProbeType(a: Acquire, m: MasterMetadata): UInt = { - MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireReadUncached -> probeCopy, - acquireWriteUncached -> probeInvalidate, - acquireReadWordUncached -> probeCopy, - acquireWriteWordUncached -> probeInvalidate, - acquireAtomicUncached -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - )) + Mux(a.uncached, + MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedRead -> probeCopy, + Acquire.uncachedWrite -> probeInvalidate, + Acquire.uncachedAtomic -> probeInvalidate + )), + MuxLookup(a.a_type, probeCopy, Array( + acquireReadShared -> probeDowngrade, + acquireReadExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers + ))) } - def requiresOuterRead(a_type: UInt) = { - (a_type != acquireWriteUncached && a_type != acquireInvalidateOthers) - } - def requiresOuterWrite(a_type: UInt) = { - (a_type === acquireWriteUncached || a_type === acquireWriteWordUncached || a_type === acquireAtomicUncached) - } - def requiresOuterRead(a_type: UInt, m: MasterMetadata) = requiresOuterRead(a_type) - def requiresOuterWrite(a_type: UInt, m: MasterMetadata) = requiresOuterWrite(a_type) + def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g_type: UInt) = g_type != grantVoluntaryAck - def requiresAckForRelease(r_type: UInt) = Bool(false) - def requiresSelfProbe(a_type: UInt) = Bool(false) + def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck + def requiresAckForRelease(r: Release) = Bool(false) + def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index a402e7db..fe647d07 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,9 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") + val MT_CB = Bits("b111") // cache block + val NUM_XA_OPS = 9 val M_SZ = 5 val M_X = Bits("b?????"); val M_XRD = Bits("b00000"); // int load diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index edcd66cb..405380f2 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -135,7 +135,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.master_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload.g_type) + mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } io.mem.grant.ready := Bool(true) @@ -193,8 +193,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - Acquire(co.getUncachedWriteAcquireType, init_addr, UInt(0)), - Acquire(co.getUncachedReadAcquireType, init_addr, UInt(0))) + UncachedWrite(init_addr, UInt(0)), + UncachedRead(init_addr)) io.mem.acquire.valid := acq_q.io.deq.valid acq_q.io.deq.ready := io.mem.acquire.ready io.mem.acquire.bits.payload := acq_q.io.deq.bits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b470df49..02e651b8 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -268,7 +268,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { // Aggregate incoming MemIO responses into TL Grants io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(UInt(0), tag_in, UInt(0), buf_in) + io.uncached.grant.bits.payload := Grant(Bool(true), UInt(0), tag_in, UInt(0), buf_in) when(!active_in && io.mem.resp.valid) { active_in := Bool(true) cnt_in := UInt(1) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6738deaf..3b1bf26b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -2,65 +2,89 @@ package uncore import Chisel._ +import scala.math.max case object TLId extends Field[String] -case object TLCoherence extends Field[CoherencePolicyWithUncached] +case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] -case object TLWriteMaskBits extends Field[Int] -case object TLWordAddrBits extends Field[Int] -case object TLAtomicOpBits extends Field[Int] -trait HasPhysicalAddress extends Bundle { - val addr = UInt(width = params(TLAddrBits)) +abstract trait TileLinkParameters extends UsesParameters { + val tlAddrBits = params(TLAddrBits) + val tlClientXactIdBits = params(TLClientXactIdBits) + val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlDataBits = params(TLDataBits) + val tlWriteMaskBits = tlDataBits/8 + val tlSubblockAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlUncachedOperandSizeBits = MT_SZ + val tlSubblockUnionBits = max(tlWriteMaskBits, + (tlSubblockAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) } -trait HasClientTransactionId extends Bundle { - val client_xact_id = Bits(width = params(TLClientXactIdBits)) +class TLBundle extends Bundle with TileLinkParameters + +trait HasPhysicalAddress extends TLBundle { + val addr = UInt(width = tlAddrBits) } -trait HasMasterTransactionId extends Bundle { - val master_xact_id = Bits(width = params(TLMasterXactIdBits)) +trait HasClientTransactionId extends TLBundle { + val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasTileLinkData extends Bundle { - val data = Bits(width = params(TLDataBits)) +trait HasMasterTransactionId extends TLBundle { + val master_xact_id = Bits(width = tlMasterXactIdBits) } -trait SourcedMessage extends Bundle +trait HasTileLinkData extends TLBundle { + val data = UInt(width = tlDataBits) +} + +trait SourcedMessage extends TLBundle trait ClientSourcedMessage extends SourcedMessage trait MasterSourcedMessage extends SourcedMessage -object Acquire -{ - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { +class Acquire extends ClientSourcedMessage + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val uncached = Bool() + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val subblock = Bits(width = tlSubblockUnionBits) + val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits + val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) + def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) +} + +object Acquire { + val nUncachedAcquireTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite + def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { val acq = new Acquire + acq.uncached := Bool(false) acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id - acq.data := Bits(0) - acq.write_mask := Bits(0) - acq.subword_addr := Bits(0) - acq.atomic_opcode := Bits(0) - acq - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id) acq.data := data + acq.subblock := UInt(0) acq } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.write_mask := write_mask - acq - } - def apply(a_type: UInt, addr: UInt, client_xact_id: UInt, subword_addr: UInt, atomic_opcode: UInt, data: UInt): Acquire = { - val acq = apply(a_type, addr, client_xact_id, data) - acq.subword_addr := subword_addr - acq.atomic_opcode := atomic_opcode - acq + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { + apply(a_type, addr, client_xact_id, UInt(0)) } def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -69,16 +93,45 @@ object Acquire } } -class Acquire extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val a_type = UInt(width = params(TLCoherence).acquireTypeWidth) - val write_mask = Bits(width = params(TLWriteMaskBits)) - val subword_addr = Bits(width = params(TLWordAddrBits)) - val atomic_opcode = Bits(width = params(TLAtomicOpBits)) +object UncachedRead { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) + acq.uncached := Bool(true) + acq.subblock := Cat(subblock_addr, operand_sz) + acq + } + def apply(addr: UInt, client_xact_id: UInt): Acquire = { + apply(addr, client_xact_id, UInt(0), MT_CB) + } + def apply(addr: UInt): Acquire = { + apply(addr, UInt(0), UInt(0), MT_CB) + } } +object UncachedWrite { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := write_mask + acq + } + def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(addr, client_xact_id, SInt(-1), data) + } + def apply(addr: UInt, data: UInt): Acquire = { + apply(addr, UInt(0), data) + } +} + +object UncachedAtomic { + def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, + subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { + val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) + acq.uncached := Bool(true) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq + } +} object Probe { def apply(p_type: UInt, addr: UInt) = { @@ -118,28 +171,34 @@ class Release extends ClientSourcedMessage val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) } -object Grant -{ - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - val gnt = new Grant - gnt.g_type := g_type - gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id - gnt.data := UInt(0) - gnt - } - def apply(g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { - val gnt = apply(g_type, client_xact_id, master_xact_id) - gnt.data := data - gnt - } -} - class Grant extends MasterSourcedMessage with HasTileLinkData with HasClientTransactionId with HasMasterTransactionId { - val g_type = UInt(width = params(TLCoherence).grantTypeWidth) + val uncached = Bool() + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) +} + +object Grant { + val nUncachedGrantTypes = 3 + //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + def uncachedRead = UInt(0) + def uncachedWrite = UInt(1) + def uncachedAtomic = UInt(2) + def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + val gnt = new Grant + gnt.uncached := uncached + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.master_xact_id := master_xact_id + gnt.data := data + gnt + } + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + } } class Finish extends ClientSourcedMessage with HasMasterTransactionId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e1f59e08..6ea8cd25 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -122,7 +122,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.header.src := UInt(bankId) //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(Acquire(co.getUncachedWriteAcquireType, + io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), xact.data), @@ -133,7 +133,8 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(co.getGrantType(xact, co.masterMetadataOnFlush), + io.inner.grant.bits.payload := Grant(Bool(false), + co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId)) @@ -169,15 +170,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact.a_type) - val outer_write_acq = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), xact.data), - { case TLId => outerId }) - val outer_write_rel = Bundle(Acquire(co.getUncachedWriteAcquireType, - xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(Acquire(co.getUncachedReadAcquireType, xact.addr, UInt(trackerId)), - { case TLId => outerId }) + val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), + { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -202,11 +201,11 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.bits.header.dst := curr_p_id io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) - val grant_type = co.getGrantType(xact, co.masterMetadataOnFlush) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(grant_type, + io.inner.grant.bits.payload := Grant(xact.uncached, + co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), m_gnt.payload.data) @@ -218,7 +217,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload.a_type) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src @@ -266,7 +265,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_mem_write) { @@ -279,7 +278,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_make_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(grant_type), s_busy, s_idle) + state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) } } is(s_busy) { // Nothing left to do but wait for transaction to complete From cb7e7125993c001c08d5159aa1e4d8bda3837c00 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 12 Nov 2014 12:55:07 -0800 Subject: [PATCH 261/688] Added uncached write data queue to coherence hub --- uncore/src/main/scala/tilelink.scala | 2 +- uncore/src/main/scala/uncore.scala | 59 +++++++++++++++++++++------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b1bf26b..ec4c15df 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,7 +16,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) - val tlWriteMaskBits = tlDataBits/8 + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) val tlUncachedOperandSizeBits = MT_SZ diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6ea8cd25..858d603c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -5,14 +5,17 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] abstract trait CoherenceAgentParameters extends UsesParameters { val co = params(TLCoherence) - val nReleaseTransactors = params(NReleaseTransactors) + val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) + val sdqDepth = params(L2StoreDataQueueDepth) + val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,11 +30,19 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { + // Queue to store impending UncachedWrite data + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} + when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId))) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId))) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) @@ -46,10 +57,11 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) t.acquire.valid := alloc_arb.io.in(i).ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires // Handle probe request generation val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) @@ -67,12 +79,19 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } + when(voluntary && release.fire()) { + vwbdq(release_idx) := release.bits.payload.data + } + // Reply to initial requestor val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -84,9 +103,22 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - {case TLId => outerId}) + { case TLId => outerId; case TLDataBits => sdqIdBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) + val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) + val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) + val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) + val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq + io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), + Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } } @@ -112,10 +144,9 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val incoming_rel = io.inner.release.bits io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, incoming_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -140,11 +171,11 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute switch (state) { is(s_idle) { - io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := incoming_rel.payload - init_client_id := incoming_rel.header.src - state := Mux(co.messageHasData(incoming_rel.payload), s_mem, s_ack) + io.inner.release.ready := Bool(true) + xact := c_rel.payload + init_client_id := c_rel.header.src + state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) } } is(s_mem) { @@ -173,7 +204,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) @@ -208,7 +239,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri co.getGrantType(xact, co.masterMetadataOnFlush), xact.client_xact_id, UInt(trackerId), - m_gnt.payload.data) + UInt(0)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 05b5188ad97607b5534b60a56513acc8d853fcea Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 19 Nov 2014 15:55:25 -0800 Subject: [PATCH 262/688] meta and data bundle refactor --- uncore/src/main/scala/cache.scala | 198 +++++++++++++++--------------- 1 file changed, 100 insertions(+), 98 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 10d00210..ee375784 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -138,12 +138,19 @@ class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState +trait HasL2MetaReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2MetaReadReq) + val resp = Valid(new L2MetaResp).flip +} + +trait HasL2MetaWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2MetaWriteReq) +} + +class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaWriteIO + class L2MetadataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2MetaReadReq).flip - val write = Decoupled(new L2MetaWriteReq).flip - val resp = Valid(new L2MetaResp) - } + val io = new L2MetaRWIO().flip val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) meta.io.read <> io.read @@ -189,12 +196,19 @@ class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { val data = Bits(width = tlDataBits) } +trait HasL2DataReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2DataReadReq) + val resp = Valid(new L2DataResp).flip +} + +trait HasL2DataWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2DataWriteReq) +} + +class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO + class L2DataArray extends L2HellaCacheModule { - val io = new Bundle { - val read = Decoupled(new L2DataReadReq).flip - val write = Decoupled(new L2DataWriteReq).flip - val resp = Valid(new L2DataResp) - } + val io = new L2DataRWIO().flip val waddr = io.write.bits.addr val raddr = io.read.bits.addr @@ -226,12 +240,8 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends val data = Module(new L2DataArray) tshrfile.io.inner <> io.inner - tshrfile.io.meta_read <> meta.io.read - tshrfile.io.meta_write <> meta.io.write - tshrfile.io.meta_resp <> meta.io.resp - tshrfile.io.data_read <> data.io.read - tshrfile.io.data_write <> data.io.write - tshrfile.io.data_resp <> data.io.resp + tshrfile.io.meta <> meta.io + tshrfile.io.data <> data.io io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent } @@ -242,12 +252,8 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val meta = new L2MetaRWIO + val data = new L2DataRWIO } // Wiring helper funcs @@ -323,12 +329,12 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac io.outer <> outer_arb.io.out // Local memory - doOutputArbitration(io.meta_read, trackerList.map(_.io.meta_read)) - doOutputArbitration(io.meta_write, trackerList.map(_.io.meta_write)) - doOutputArbitration(io.data_read, trackerList.map(_.io.data_read)) - doOutputArbitration(io.data_write, trackerList.map(_.io.data_write)) - doInputRouting(io.meta_resp, trackerList.map(_.io.meta_resp)) - doInputRouting(io.data_resp, trackerList.map(_.io.data_resp)) + doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) } @@ -340,12 +346,8 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) - val meta_read = Decoupled(new L2MetaReadReq) - val meta_write = Decoupled(new L2MetaWriteReq) - val meta_resp = Valid(new L2MetaResp).flip - val data_read = Decoupled(new L2DataReadReq) - val data_write = Decoupled(new L2DataWriteReq) - val data_resp = Valid(new L2DataResp).flip + val data = new L2DataRWIO + val meta = new L2MetaRWIO } val c_acq = io.inner.acquire.bits @@ -386,23 +388,23 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact.client_xact_id, UInt(trackerId)) - io.data_read.valid := Bool(false) - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_internal.meta.coh, init_client_id) @@ -416,24 +418,24 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - xact_internal := io.meta_resp.bits - state := Mux(io.meta_resp.bits.tag_match, + when(io.meta.resp.valid) { + xact_internal := io.meta.resp.bits + state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { state := s_meta_write } + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) @@ -522,26 +524,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) - io.data_read.valid := Bool(false) - io.data_read.bits.id := UInt(trackerId) - io.data_read.bits.way_en := xact_internal.way_en - io.data_read.bits.addr := xact.addr - io.data_write.valid := Bool(false) - io.data_write.bits.id := UInt(trackerId) - io.data_write.bits.way_en := xact_internal.way_en - io.data_write.bits.addr := xact.addr - io.data_write.bits.wmask := SInt(-1) - io.data_write.bits.data := xact.data - io.meta_read.valid := Bool(false) - io.meta_read.bits.id := UInt(trackerId) - io.meta_read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_read.bits.tag := xact.addr >> UInt(untagBits) - io.meta_write.valid := Bool(false) - io.meta_write.bits.id := UInt(trackerId) - io.meta_write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta_write.bits.way_en := xact_internal.way_en - io.meta_write.bits.data.tag := xact.addr >> UInt(untagBits) - io.meta_write.bits.data.coh := next_coh_on_grant + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_internal.way_en + io.data.read.bits.addr := xact.addr + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_internal.way_en + io.data.write.bits.addr := xact.addr + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := xact.data + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_internal.way_en + io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.data.coh := next_coh_on_grant switch (state) { is(s_idle) { @@ -553,17 +555,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_meta_read) { - io.meta_read.valid := Bool(true) - when(io.meta_read.ready) { state := s_meta_resp } + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { - when(io.meta_resp.valid) { - val coh = io.meta_resp.bits.meta.coh - val _tag_match = io.meta_resp.bits.tag_match + when(io.meta.resp.valid) { + val coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta_resp.bits + xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & @@ -626,14 +628,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_wb) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_wb } } is(s_data_resp_wb) { - when(io.data_resp.valid) { - wb_buffer := io.data_resp.bits.data + when(io.data.resp.valid) { + wb_buffer := io.data.resp.bits.data state := s_outer_write_wb } } @@ -661,26 +663,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read_hit) { - io.data_read.valid := Bool(true) - when(io.data_read.ready) { + io.data.read.valid := Bool(true) + when(io.data.read.ready) { state := s_data_resp_hit } } is(s_data_resp_hit) { - when(io.data_resp.valid) { - xact.data := mergeData(xact, io.data_resp.bits.data) + when(io.data.resp.valid) { + xact.data := mergeData(xact, io.data.resp.bits.data) state := s_meta_write } } is(s_data_write) { - io.data_write.valid := Bool(true) - when(io.data_write.ready) { + io.data.write.valid := Bool(true) + when(io.data.write.ready) { state := s_meta_write } } is(s_meta_write) { - io.meta_write.valid := Bool(true) - when(io.meta_write.ready) { state := s_grant } + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { state := s_grant } } is(s_grant) { io.inner.grant.valid := Bool(true) From 404773eb9f1f29277b205eb4ace622ca67eb0d9d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 3 Dec 2014 14:22:39 -0800 Subject: [PATCH 263/688] fix wb bug --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ee375784..0a717604 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -462,6 +462,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Reg{xact.data.clone} + val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) @@ -476,8 +477,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) //TODO: Also indexes - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && - (state != s_idle) //TODO: Also indexes? + io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && + (state != s_idle) //TODO: Also indexes? val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, @@ -488,7 +490,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_internal.meta.coh, c_gnt.header.dst) - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), { case TLId => outerId }) val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), From 2f733a60db1d564f2da11632acc6e3308910460c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 02:57:44 -0800 Subject: [PATCH 264/688] Begin adding TLDataBeats to uncore --- uncore/src/main/scala/network.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index d3f4a9e8..a34e91c1 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -25,13 +25,13 @@ class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { abstract class PhysicalNetwork extends Module -class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1) extends PhysicalNetwork { +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { arb.valid := in.valid && (in.bits.header.dst === UInt(i)) arb.bits := in.bits diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index a5066f39..1e535507 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -10,12 +10,14 @@ case object TLAddrBits extends Field[Int] case object TLMasterXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] +case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlMasterXactIdBits = params(TLMasterXactIdBits) val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits val tlSubblockAddrBits = log2Up(tlWriteMaskBits) val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) From 3026c46a9c81709bd1c49e1f9737af1095cf918c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 7 Dec 2014 03:02:20 -0800 Subject: [PATCH 265/688] Finish adding TLDataBeats to uncore & hub --- uncore/src/main/scala/cache.scala | 4 +- uncore/src/main/scala/coherence.scala | 6 - uncore/src/main/scala/htif.scala | 33 +++-- uncore/src/main/scala/memserdes.scala | 133 ++++++++++++------ uncore/src/main/scala/tilelink.scala | 18 +-- uncore/src/main/scala/uncore.scala | 193 ++++++++++++++++---------- uncore/src/main/scala/util.scala | 88 ++++++++++++ 7 files changed, 325 insertions(+), 150 deletions(-) create mode 100644 uncore/src/main/scala/util.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0a717604..b96aab70 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,7 +30,8 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCycles = params(TLDataBits)/rowBits + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +100,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters - with TileLinkParameters abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 57545fda..280e20ad 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,12 +3,6 @@ package uncore import Chisel._ -object MuxBundle { - def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) - } -} - abstract class CoherenceMetadata extends Bundle object ClientMetadata { diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 405380f2..378ca3b1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -13,6 +13,7 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) + val dataBeats = params(TLDataBeats) val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) @@ -71,7 +72,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 - val long_request_bits = short_request_bits + dataBits + val long_request_bits = short_request_bits + dataBits*dataBeats require(short_request_bits % w == 0) val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits @@ -150,12 +151,13 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state_tx))) } - val acq_q = Module(new Queue(new Acquire, 1)) - when (state === state_mem_wreq && acq_q.io.enq.ready) { - state := state_mem_wresp + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) + when (state === state_mem_wreq) { + when (cnt_done) { state := state_mem_wresp } } - when (state === state_mem_rreq && acq_q.io.enq.ready) { - state := state_mem_rresp + when (state === state_mem_rreq) { + when(io.mem.acquire.ready) { state := state_mem_rresp } } when (state === state_mem_wresp) { when (mem_acked) { @@ -164,10 +166,10 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } when (state === state_mem_rresp) { - when (io.mem.grant.valid) { + when (cnt_done) { state := state_mem_finish + mem_acked := Bool(false) } - mem_acked := Bool(false) } when (state === state_mem_finish && io.mem.finish.ready) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) @@ -182,22 +184,19 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: Bits = null + var mem_req_data: UInt = null for (i <- 0 until dataBits/short_request_bits) { - val idx = UInt(i, log2Up(dataBits/short_request_bits)) + val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) when (state === state_mem_rresp && io.mem.grant.valid) { packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } mem_req_data = Cat(packet_ram(idx), mem_req_data) } - acq_q.io.enq.valid := state === state_mem_rreq || state === state_mem_wreq val init_addr = addr.toUInt >> UInt(offsetBits-3) - acq_q.io.enq.bits := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, UInt(0)), + io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq + io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + UncachedWrite(init_addr, mem_req_data), UncachedRead(init_addr)) - io.mem.acquire.valid := acq_q.io.deq.valid - acq_q.io.deq.ready := io.mem.acquire.ready - io.mem.acquire.bits.payload := acq_q.io.deq.bits io.mem.acquire.bits.payload.data := mem_req_data io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module @@ -255,7 +254,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits/8) << acq_q.io.enq.bits.addr.getWidth) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index ea41256e..d6133fc0 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -209,81 +209,122 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val co = params(TLCoherence) - val tbits = params(MIFTagBits) - val dbits = params(MIFDataBits) - val dbeats = params(MIFDataBeats) - require(params(TLDataBits) == dbits*dbeats) + val mifTagBits = params(MIFTagBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) + val tlDataBits = params(TLDataBits) + val tlDataBeats = params(TLDataBeats) + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + // Decompose outgoing TL Acquires into MemIO cmd and data val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) val mem_data_q = Module(new Queue(new MemData, qDepth)) - val cnt_max = dbeats - val cnt_out = Reg(UInt(width = log2Up(cnt_max+1))) + io.uncached.acquire.ready := Bool(false) + io.uncached.grant.valid := Bool(false) + io.mem.resp.ready := Bool(false) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + + val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val buf_out = Reg(Bits()) + val tl_done_out = Reg(init=Bool(false)) + val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) - val cnt_in = Reg(UInt(width = log2Up(cnt_max+1))) - val active_in = Reg(init=Bool(false)) - val buf_in = Reg(Bits()) - val tag_in = Reg(UInt(width = tbits)) - - // Decompose outgoing TL Acquires into MemIO cmd and data - when(!active_out && io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - cnt_out := UInt(0) - buf_out := io.uncached.acquire.bits.payload.data - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr - has_data := co.messageHasData(io.uncached.acquire.bits.payload) + when(!active_out){ + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.uncached.acquire.bits.payload.client_xact_id + addr_out := io.uncached.acquire.bits.payload.addr + has_data := acq_has_data + tl_done_out := tl_wrap_out + mif_done_out := Bool(false) + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } } when(active_out) { + when(!cmd_sent_out) { + mem_cmd_q.io.enq.valid := Bool(true) + } + when(has_data) { + when(!tl_done_out) { + io.uncached.acquire.ready := Bool(true) + when(io.uncached.acquire.valid) { + tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + } + } when(mem_cmd_q.io.enq.fire()) { cmd_sent_out := Bool(true) } - when(mem_data_q.io.enq.fire()) { - cnt_out := cnt_out + UInt(1) - buf_out := buf_out >> UInt(dbits) - } - when(cmd_sent_out && (!has_data || cnt_out === UInt(cnt_max))) { + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(cmd_sent_out && (!has_data || mif_done_out)) { active_out := Bool(false) } } - io.uncached.acquire.ready := !active_out - mem_cmd_q.io.enq.valid := active_out && !cmd_sent_out mem_cmd_q.io.enq.bits.rw := has_data mem_cmd_q.io.enq.bits.tag := tag_out mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.valid := active_out && has_data && cnt_out < UInt(cnt_max) - mem_data_q.io.enq.bits.data := buf_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data io.mem.req_cmd <> mem_cmd_q.io.deq io.mem.req_data <> mem_data_q.io.deq // Aggregate incoming MemIO responses into TL Grants - io.mem.resp.ready := !active_in || cnt_in < UInt(cnt_max) - io.uncached.grant.valid := active_in && (cnt_in === UInt(cnt_max)) - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), buf_in) - when(!active_in && io.mem.resp.valid) { - active_in := Bool(true) - cnt_in := UInt(1) - buf_in := io.mem.resp.bits.data << UInt(dbits*(cnt_max-1)) - tag_in := io.mem.resp.bits.tag + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) + val active_in = Reg(init=Bool(false)) + val mif_done_in = Reg(init=Bool(false)) + val tag_in = Reg(UInt(width = mifTagBits)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + + when(!active_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data + } } + when(active_in) { - when(io.uncached.grant.fire()) { - active_in := Bool(false) - } - when(io.mem.resp.fire()) { - buf_in := Cat(io.mem.resp.bits.data, buf_in(cnt_max*dbits-1,dbits)) - cnt_in := cnt_in + UInt(1) + io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } } + + io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), + tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -390,8 +431,8 @@ class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) val a = Module(new MemIOUncachedTileLinkIOConverter(2)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) a.io.uncached <> io.uncached - b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles) + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1e535507..678efb6a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -26,9 +26,11 @@ abstract trait TileLinkParameters extends UsesParameters { (tlSubblockAddrBits + tlUncachedOperandSizeBits + tlAtomicOpcodeBits)) + val co = params(TLCoherence) } -class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLModule extends Module with TileLinkParameters trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) @@ -55,7 +57,7 @@ class Acquire extends ClientSourcedMessage with HasClientTransactionId with HasTileLinkData { val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), params(TLCoherence).acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff @@ -147,7 +149,7 @@ object Probe { class Probe extends MasterSourcedMessage with HasPhysicalAddress { - val p_type = UInt(width = params(TLCoherence).probeTypeWidth) + val p_type = UInt(width = co.probeTypeWidth) def is(t: UInt) = p_type === t } @@ -172,7 +174,7 @@ class Release extends ClientSourcedMessage with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = params(TLCoherence).releaseTypeWidth) + val r_type = UInt(width = co.releaseTypeWidth) def is(t: UInt) = r_type === t } @@ -181,7 +183,7 @@ class Grant extends MasterSourcedMessage with HasClientTransactionId with HasMasterTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), params(TLCoherence).grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t } @@ -221,7 +223,7 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -abstract class TileLinkArbiterLike(val arbN: Int) extends Module { +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId @@ -232,8 +234,8 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends Module { def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = params(TLCoherence).messageHasData(m.payload) - val arb = Module(new RRArbiter(out.bits.clone, arbN)) + def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { arb.valid := req.valid diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 858d603c..69b13800 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -8,14 +8,16 @@ case object NAcquireTransactors extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] case object NClients extends Field[Int] -abstract trait CoherenceAgentParameters extends UsesParameters { - val co = params(TLCoherence) +abstract trait CoherenceAgentParameters extends UsesParameters + with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) - val sdqDepth = params(L2StoreDataQueueDepth) - val sdqIdBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + 1 + val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + + log2Ceil(tlDataBeats) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } abstract class CoherenceAgent(innerId: String, outerId: String) extends Module @@ -27,75 +29,93 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } +class DataQueueLocation extends Bundle with CoherenceAgentParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { - // Queue to store impending UncachedWrite data - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val(sdqDepth-1,0)) - val sdq_rdy = !sdq_val.andR - val sdq_enq = io.inner.acquire.valid && io.inner.acquire.ready && co.messageHasData(io.inner.acquire.bits.payload) - val sdq = Vec.fill(sdqDepth){Reg(io.inner.acquire.bits.payload.data)} - when (sdq_enq) { sdq(sdq_alloc_id) := io.inner.acquire.bits.payload.data } + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) ++ + Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => sdqIdBits})) + Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - // Handle acquire transaction initiation + // Queue to store impending UncachedWrite data val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner alloc_arb.io.in(i).valid := t.acquire.ready t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := Cat(sdq_alloc_id, UInt(1)) + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits t.acquire.valid := alloc_arb.io.in(i).ready } acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - // Handle probe request generation - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Handle releases, which might be voluntary and might have data + // Queue to store impending Voluntary Release data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) + val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width + val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) val block_releases = Bool(false) val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} val release_idx = Mux(voluntary, UInt(0), conflict_idx) - // TODO: Add merging logic to allow allocated AcquireTracker to handle conflicts, send all necessary grants, use first sufficient response for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) Cat(UInt(i), UInt(2)) else UInt(0)) + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - val vwbdq = Vec.fill(nReleaseTransactors){ Reg(release.bits.payload.data) } - when(voluntary && release.fire()) { - vwbdq(release_idx) := release.bits.payload.data - } + // Wire probe requests to clients + val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) + io.inner.probe <> probe_arb.io.out + probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - // Reply to initial requestor - val grant_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), trackerList.size)) + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Free finished transactions + // Wire finished transaction acks val ack = io.inner.finish trackerList.map(_.io.inner.finish.valid := ack.valid) trackerList.map(_.io.inner.finish.bits := ack.bits) @@ -103,27 +123,28 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => sdqIdBits }) + { case TLId => outerId; case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val is_in_sdq = outer_arb.io.out.acquire.bits.payload.data(0) - val is_in_vwbdq = outer_arb.io.out.acquire.bits.payload.data(1) - val free_sdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(1) - val free_vwbdq_id = outer_arb.io.out.acquire.bits.payload.data >> UInt(2) - val free_sdq = io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload) && is_in_sdq - io.outer.acquire.bits.payload.data := Mux(is_in_sdq, sdq(free_sdq_id), - Mux(is_in_vwbdq, vwbdq(free_vwbdq_id), release.bits.payload.data)) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + co.messageHasData(io.outer.acquire.bits.payload) && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out // Update SDQ valid bits when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(free_sdq_id) & Fill(sdqDepth, free_sdq)) | + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) } } abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients) = (params(TLCoherence),params(NClients)) + val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) @@ -137,13 +158,20 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val c_gnt = io.inner.grant.bits val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits + } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_mem :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact = Reg{ new Release } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && @@ -156,7 +184,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact.addr, UInt(trackerId), - xact.data), + data_ptrs(outer_data_cnt)), { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -169,18 +197,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact.client_xact_id, UInt(trackerId)) + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_ptrs(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { + io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - io.inner.release.ready := Bool(true) xact := c_rel.payload init_client_id := c_rel.header.src - state := Mux(co.messageHasData(c_rel.payload), s_mem, s_ack) + data_ptrs(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) + state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } - is(s_mem) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { state := s_ack } + is(s_outer) { + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + when(outer_data_done) { state := s_ack } } is(s_ack) { io.inner.grant.valid := Bool(true) @@ -195,6 +233,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact = Reg{ new Acquire } val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) //TODO: Will need id reg for merged release xacts + val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (outer_data_cnt, outer_data_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) @@ -202,23 +246,21 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), UInt(0)), // Special SQDId + val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), { case TLId => outerId }) val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) - if (nClients > 1) { - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = Bool(true) //co.needsSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - } + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) + val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) io.outer.acquire.valid := Bool(false) @@ -244,6 +286,14 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) + when(collect_inner_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_ptrs(inner_data_cnt) := c_acq.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) @@ -252,14 +302,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when( io.inner.acquire.valid ) { xact := c_acq.payload init_client_id := c_acq.header.src + data_ptrs(UInt(0)) := c_acq.payload.data + collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags - if(nClients > 1) { - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } else state := Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant)) + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(needs_outer_write, s_mem_write, + Mux(needs_outer_read, s_mem_read, s_make_grant))) } } is(s_probe) { @@ -276,15 +325,17 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) + when(outer_data_done) { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } } } } .otherwise { io.inner.release.ready := Bool(true) - if(nClients > 1) release_count := release_count - UInt(1) + release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, Mux(pending_outer_read, s_mem_read, s_make_grant)) @@ -300,9 +351,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := Bool(true) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq - when(io.outer.acquire.ready) { + when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala new file mode 100644 index 00000000..c28778aa --- /dev/null +++ b/uncore/src/main/scala/util.scala @@ -0,0 +1,88 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import scala.math._ + +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) + } +} + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { + val io = new Bundle { + val in = Decoupled(gen.clone).flip + val out = Decoupled(gen.clone) + val cnt = UInt(OUTPUT, log2Up(n)) + val done = Bool(OUTPUT) + } + val narrowWidth = io.in.bits.payload.data.getWidth / n + require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + + if(n == 1) { + io.in <> io.out + io.cnt := UInt(width = 0) + io.done := Bool(true) + } else { + val cnt = Reg(init=UInt(0, width = log2Up(n))) + val wrap = cnt === UInt(n-1) + val rbits = Reg(init=io.in.bits) + val active = Reg(init=Bool(false)) + + val shifter = Vec.fill(n){Bits(width = narrowWidth)} + (0 until n).foreach { + i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + } + + io.done := Bool(false) + io.cnt := cnt + io.in.ready := !active + io.out.valid := active || io.in.valid + io.out.bits := io.in.bits + when(!active && io.in.valid) { + when(doSer(io.in.bits.payload)) { + cnt := Mux(io.out.ready, UInt(1), UInt(0)) + rbits := io.in.bits + active := Bool(true) + } + io.done := !doSer(io.in.bits.payload) + } + when(active) { + io.out.bits := rbits + io.out.bits.payload.data := shifter(cnt) + when(io.out.ready) { + cnt := cnt + UInt(1) + when(wrap) { + cnt := UInt(0) + io.done := Bool(true) + active := Bool(false) + } + } + } + } +} From 424df2368f6c93f34800f61bf7789d84d7762663 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 01:11:08 -0800 Subject: [PATCH 266/688] 1R/W L2 data array? Add TLDataBeats to new LLC; all bmarks pass --- uncore/src/main/scala/cache.scala | 335 +++++++++++++++++--------- uncore/src/main/scala/coherence.scala | 26 +- uncore/src/main/scala/tilelink.scala | 9 +- uncore/src/main/scala/uncore.scala | 142 ++++++----- 4 files changed, 308 insertions(+), 204 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b96aab70..97a0c9a5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -134,6 +134,7 @@ class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id { override def clone = new L2MetaWriteReq().asInstanceOf[this.type] } + class L2MetaResp extends L2HellaCacheBundle with HasL2Id with HasL2InternalRequestState @@ -156,10 +157,10 @@ class L2MetadataArray extends L2HellaCacheModule { meta.io.read <> io.read meta.io.write <> io.write - val s1_clk_en = Reg(next = io.read.fire()) val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) @@ -220,11 +221,11 @@ class L2DataArray extends L2HellaCacheModule { } array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) } - io.resp.valid := ShiftRegister(io.read.valid, 2) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 2) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 2), resp) + io.resp.valid := ShiftRegister(io.read.fire(), 1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - io.read.ready := Bool(true) + io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? io.write.ready := Bool(true) } @@ -233,7 +234,6 @@ class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends require(isPow2(nSets)) require(isPow2(nWays)) - require(refillCycles == 1) val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) val meta = Module(new L2MetadataArray) @@ -257,8 +257,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + def doOutputArbitration[T <: Data](out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } @@ -282,7 +285,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac val acquire = io.inner.acquire val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner @@ -293,9 +295,6 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle probe requests - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) - // Handle releases, which might be voluntary and might have data val release = io.inner.release val voluntary = co.isVoluntary(release.bits.payload) @@ -310,10 +309,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases - // Reply to initial requestor - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - - // Free finished transactions on ack + // Wire finished transaction acks val finish = io.inner.finish val finish_idx = finish.bits.payload.master_xact_id trackerList.zipWithIndex.map { case (t, i) => @@ -322,20 +318,26 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac trackerList.map(_.io.inner.finish.bits := finish.bits) finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - // Arbitrate for the outer memory port + // Wire probe requests to clients + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + // Wire grant reply to initiating client + def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + + // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), {case TLId => outerId}) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } io.outer <> outer_arb.io.out - // Local memory + // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) - } @@ -356,21 +358,34 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val c_ack = io.inner.finish.bits val m_gnt = io.outer.grant.bits - def mergeData(acq: Acquire, data: UInt): UInt = { - //TODO wite mask - Mux(co.messageHasData(acq), acq.data, data) + def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { + //TODO apply acq's write mask + Mux(co.messageHasData(acq), old_data, new_data) } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + + val collect_inner_data = Reg(init=Bool(false)) + val (inner_data_cnt, inner_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (local_data_cnt, local_data_done) = + Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) @@ -382,38 +397,50 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.finish.ready := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh), - xact.client_xact_id, + co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), + xact_client_xact_id, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, - xact_internal.meta.coh, - init_client_id) + xact_meta.coh, + xact_src) + + when(collect_inner_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_data(inner_data_cnt) := c_rel.payload.data + } + when(inner_data_done) { collect_inner_data := Bool(false) } + } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data + collect_inner_data := co.messageHasData(c_rel.payload) state := s_meta_read } } @@ -423,15 +450,18 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } is(s_meta_resp) { when(io.meta.resp.valid) { - xact_internal := io.meta.resp.bits + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(co.messageHasData(xact), s_data_write, s_meta_write), s_grant) } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { state := s_meta_write } + io.data.write.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (local_data_cnt < inner_data_cnt)) + when(local_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -454,72 +484,100 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val xact_internal = Reg{ new L2MetaResp } - val init_client_id = Reg(init = UInt(0, width = log2Up(nClients))) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val crel_had_data = Reg(init = Bool(false)) val crel_was_voluntary = Reg(init = Bool(false)) val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Reg{xact.data.clone} - val addr_wb = Cat(xact_internal.meta.tag, xact.addr(untagBits-1,blockOffBits)) + val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + + val collect_cacq_data = Reg(init=Bool(false)) + //TODO: zero width wires + val (cacq_data_cnt, cacq_data_done) = + Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (outer_data_resp_cnt, outer_data_resp_done) = + Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = co.dir().flush) val curr_p_id = co.dir().next(pending_probes) - val is_uncached = xact.uncached - val tag_match = xact_internal.tag_match - val needs_writeback = !tag_match && co.needsWriteback(xact_internal.meta.coh) - val is_hit = tag_match && co.isHit(xact, xact_internal.meta.coh) - val needs_probes = co.requiresProbes(xact, xact_internal.meta.coh) - //TODO: does allocate + val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) + val needs_probes = co.requiresProbes(xact, xact_meta.coh) + //TODO: uncached does or does not allocate - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - (state != s_idle) //TODO: Also indexes + //TODO: Are there any races between lines with the same idx? + //TODO: Allow hit under miss for stores + io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && + xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + (state != s_idle) && + !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(addr_wb, c_rel.payload.addr)) && - (state != s_idle) //TODO: Also indexes? + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state != s_idle) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, - xact_internal.meta.coh, + xact_meta.coh, c_rel.header.src) val next_coh_on_grant = co.masterMetadataOnGrant( c_gnt.payload, - xact_internal.meta.coh, + xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), xact.data), + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(addr_wb, UInt(trackerId), wb_buffer), + val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact.addr, UInt(trackerId)), { case TLId => outerId }) + val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_internal.meta.coh), xact.addr) - val cprb_for_mwb = Probe(co.getProbeTypeOnVoluntaryWriteback, addr_wb) - //TODO inner_probe_mprb + val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) + //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!tag_match && needs_writeback, - cprb_for_mwb, + io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, + cprb_for_mvwb, cprb_for_cacq) - val cgnt_for_cacq = Grant(xact.uncached, co.getGrantType(xact, xact_internal.meta.coh), - xact.client_xact_id, + val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, UInt(trackerId), - xact.data) - val cgnt_for_cwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) + xact_data(cgnt_data_cnt)) + val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, init_client_id) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) + io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -527,31 +585,45 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_internal.way_en - io.data.read.bits.addr := xact.addr + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_internal.way_en - io.data.write.bits.addr := xact.addr + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact.data + io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact.addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.read.bits.tag := xact_addr >> UInt(untagBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr(untagBits-1,blockOffBits) - io.meta.write.bits.way_en := xact_internal.way_en - io.meta.write.bits.data.tag := xact.addr >> UInt(untagBits) + io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) io.meta.write.bits.data.coh := next_coh_on_grant + when(collect_cacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_data(cacq_data_cnt) := c_acq.payload.data + } + when(cacq_data_done) { collect_cacq_data := Bool(false) } + } + switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src + collect_cacq_data := co.messageHasData(c_acq.payload) state := s_meta_read } } @@ -561,21 +633,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_meta_resp) { when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) - xact_internal := io.meta.resp.bits when(_needs_probes) { val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << init_client_id) + ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self release_count := co.dir().count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) - } + } state := Mux(_tag_match, Mux(_is_hit, Mux(_needs_probes, s_probe, s_data_read_hit), @@ -594,25 +668,41 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_internal.meta.coh := next_coh_on_release + xact_meta.coh := next_coh_on_release + + // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(tag_match) { - xact.data := mergeData(xact, io.inner.release.bits.payload.data) - } .otherwise { - wb_buffer := io.inner.release.bits.payload.data + when(xact_tag_match) { // Hit, so merge new write with released data + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + io.inner.release.bits.payload.data) + } .otherwise { // Miss, we're voluntarily evicting this data + wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data } } + + // Voluntary releases don't count against the release countdown + // because we will get a further release ack from that client in + // response to our probe. We don't send a grant acknowledging + // a writeback or decrement release_count until we've received + // all the data beats. when(co.isVoluntary(c_rel.payload)) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_internal.meta.coh) - } - when(!co.isVoluntary(c_rel.payload)) { - release_count := release_count - UInt(1) + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + crel_was_voluntary := Bool(true) + crel_wb_src := c_rel.header.src + crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) + } + } .otherwise { + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } } } + // If we saw a voluntary writeback, we need to send an extra grant + // to acknowledge it. when(crel_was_voluntary) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { @@ -621,7 +711,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(tag_match, + state := Mux(xact_tag_match, Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read_hit), s_outer_read), @@ -630,20 +720,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_data_read_wb) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_wb + when(local_data_read_done) { state := s_data_resp_wb } + when(io.data.resp.valid) { + wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } } is(s_data_resp_wb) { - when(io.data.resp.valid) { - wb_buffer := io.data.resp.bits.data - state := s_outer_write_wb - } + when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write_wb } } is(s_outer_write_wb) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_wb - when(io.outer.acquire.ready) { + when(outer_data_write_done) { state := s_outer_read } } @@ -657,27 +746,35 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_outer_resp) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - xact.data := mergeData(xact, io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_internal.meta.coh - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + //TODO make sure cacq data is actually present before merging + xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), + io.outer.grant.bits.payload.data) + //TODO: set pending client state in xact_meta.coh + when(outer_data_resp_done) { + state := Mux(co.messageHasData(io.outer.grant.bits.payload), + s_data_write, s_data_read_hit) + } } } is(s_data_read_hit) { io.data.read.valid := Bool(true) - when(io.data.read.ready) { - state := s_data_resp_hit + when(io.data.resp.valid) { + //TODO make sure cacq data is actually present before merging + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_read_done) { state := s_data_resp_hit } } is(s_data_resp_hit) { when(io.data.resp.valid) { - xact.data := mergeData(xact, io.data.resp.bits.data) - state := s_meta_write + xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), + io.data.resp.bits.data) } + when(local_data_resp_done) { state := s_meta_write } } is(s_data_write) { io.data.write.valid := Bool(true) - when(io.data.write.ready) { + when(local_data_write_done) { state := s_meta_write } } @@ -687,7 +784,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { + when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { state := Mux(co.requiresAckForGrant(c_gnt.payload), s_busy, s_idle) } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 280e20ad..9d6d768b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -205,9 +205,7 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -339,9 +337,7 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -485,10 +481,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -644,11 +637,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached - )) + Mux(g.uncached, uncached, cached) } def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData @@ -817,12 +806,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) val uncached = MasterMetadata(masterValid, m.sharers)(this) - MuxBundle(uncached, Array( - g.is(grantReadShared) -> cached, - g.is(grantReadExclusive) -> cached, - g.is(grantReadExclusiveAck) -> cached, - g.is(grantReadMigratory) -> cached - )) + Mux(g.uncached, uncached, cached) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 678efb6a..45c6fa59 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -78,16 +78,19 @@ object Acquire { def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { val acq = new Acquire - acq.uncached := Bool(false) + acq.uncached := uncached acq.a_type := a_type acq.addr := addr acq.client_xact_id := client_xact_id acq.data := data - acq.subblock := UInt(0) + acq.subblock := subblock acq } + def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { + apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) + } def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { apply(a_type, addr, client_xact_id, UInt(0)) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 69b13800..fcdbdd9f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -15,20 +15,10 @@ abstract trait CoherenceAgentParameters extends UsesParameters val nTransactors = nReleaseTransactors + nAcquireTransactors val nClients = params(NClients) val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(params(L2StoreDataQueueDepth))) + - log2Ceil(tlDataBeats) + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } -} - class DataQueueLocation extends Bundle with CoherenceAgentParameters { val idx = UInt(width = dqIdxBits) val loc = UInt(width = log2Ceil(nDataQueueLocations)) @@ -42,6 +32,15 @@ object DataQueueLocation { } } +abstract class CoherenceAgent(innerId: String, outerId: String) extends Module + with CoherenceAgentParameters { + val io = new Bundle { + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val incoherent = Vec.fill(nClients){Bool()}.asInput + } +} + class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { @@ -103,6 +102,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends } release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + // Wire probe requests to clients val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) io.inner.probe <> probe_arb.io.out @@ -115,12 +120,6 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - // Create an arbiter for the one memory port val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), { case TLId => outerId; case TLDataBits => internalDataBits }) @@ -164,9 +163,13 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact = Reg{ new Release } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact_r_type = Reg(io.inner.release.bits.payload.r_type) + val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) @@ -174,33 +177,35 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state != s_idle) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact.addr, - UInt(trackerId), - data_ptrs(outer_data_cnt)), - { case TLId => outerId }) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id + io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.acquire.bits.payload := Bundle(UncachedWrite( + xact_addr, + UInt(trackerId), + xact_data(outer_data_cnt)), + { case TLId => outerId }) + when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - data_ptrs(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := c_rel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -209,15 +214,18 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact := c_rel.payload - init_client_id := c_rel.header.src - data_ptrs(UInt(0)) := c_rel.payload.data + xact_src := c_rel.header.src + xact_r_type := c_rel.payload.r_type + xact_addr := c_rel.payload.addr + xact_client_xact_id := c_rel.payload.client_xact_id + xact_data(UInt(0)) := c_rel.payload.data collect_inner_data := co.messageHasData(c_rel.payload) state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) } } is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) when(outer_data_done) { state := s_ack } } is(s_ack) { @@ -230,28 +238,28 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) val state = Reg(init=s_idle) - val xact = Reg{ new Acquire } - val init_client_id = Reg(init=UInt(0, width = log2Up(nClients))) - //TODO: Will need id reg for merged release xacts - val data_ptrs = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val collect_inner_data = Reg(init=Bool(false)) val (inner_data_cnt, inner_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) val (outer_data_cnt, outer_data_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val release_count = if (nClients == 1) UInt(0) else Reg(init=UInt(0, width = log2Up(nClients))) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) - val outer_write_acq = Bundle(UncachedWrite(xact.addr, UInt(trackerId), data_ptrs(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact.addr, UInt(trackerId), c_rel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact.addr, UInt(trackerId)), - { case TLId => outerId }) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -260,26 +268,34 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state != s_idle) + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + (state != s_idle) && + !collect_inner_data + io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state != s_idle) + + val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + { case TLId => outerId }) + val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default io.outer.acquire.bits.header.src := UInt(bankId) - //io.outer.acquire.bits.header.dst TODO - io.outer.acquire.bits.payload := outer_read io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact.addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := init_client_id - io.inner.grant.bits.payload := Grant(xact.uncached, + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, co.masterMetadataOnFlush), - xact.client_xact_id, + xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -289,7 +305,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - data_ptrs(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := c_acq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -300,9 +316,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = co.messageHasData(c_acq.payload) val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) when( io.inner.acquire.valid ) { - xact := c_acq.payload - init_client_id := c_acq.header.src - data_ptrs(UInt(0)) := c_acq.payload.data + xact_uncached := c_acq.payload.uncached + xact_a_type := c_acq.payload.a_type + xact_addr := c_acq.payload.addr + xact_client_xact_id := c_acq.payload.client_xact_id + xact_data(UInt(0)) := c_acq.payload.data + xact_subblock := c_acq.payload.subblock + xact_src := c_acq.header.src collect_inner_data := co.messageHasData(c_acq.payload) probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) @@ -319,12 +339,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back + io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready when(io.inner.release.valid) { when(co.messageHasData(c_rel.payload)) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { - io.inner.release.ready := Bool(true) when(outer_data_done) { release_count := release_count - UInt(1) when(release_count === UInt(1)) { @@ -334,7 +354,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } .otherwise { - io.inner.release.ready := Bool(true) release_count := release_count - UInt(1) when(release_count === UInt(1)) { state := Mux(pending_outer_write, s_mem_write, @@ -351,7 +370,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } is(s_mem_write) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) + io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq when(outer_data_done) { state := Mux(pending_outer_read, s_mem_read, s_make_grant) From 6a8b66231ca484cd40269a8aa8170c0ef318b5fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 12 Dec 2014 12:07:04 -0800 Subject: [PATCH 267/688] Add uncached->cached tilelink converter --- uncore/src/main/scala/tilelink.scala | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45c6fa59..4e5ac961 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -226,6 +226,25 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} +object TileLinkIOWrapper { + def apply[T <: Data](uncached: UncachedTileLinkIO) = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> uncached + conv.io.out + } +} + abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId From d04da83f96b58c7957483059ea7508d4d38925cd Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 15 Dec 2014 17:36:17 -0800 Subject: [PATCH 268/688] Make data RAMs 1RW instead of 1R1W --- uncore/src/main/scala/cache.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 97a0c9a5..09d04521 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -216,10 +216,13 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val resp = (0 until nWays).map { w => val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) + val reg_raddr = Reg(UInt()) when (io.write.bits.way_en(w) && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { + reg_raddr := raddr } - array(RegEnable(raddr, io.read.bits.way_en(w) && io.read.valid)) + array(reg_raddr) } io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) From ab39cbb15dbe4068f96882d7ab1923ad51c524e9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 15 Dec 2014 19:23:13 -0800 Subject: [PATCH 269/688] cleanup DirectoryRepresentation and coherence params --- uncore/src/main/scala/cache.scala | 14 +-- uncore/src/main/scala/coherence.scala | 145 ++++++++++++-------------- uncore/src/main/scala/uncore.scala | 17 ++- 3 files changed, 86 insertions(+), 90 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 09d04521..08533ef9 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -123,7 +123,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata()(co) //co.masterMetadataOnFlush.clone + val coh = new MasterMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -524,8 +524,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir().flush) - val curr_p_id = co.dir().next(pending_probes) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) @@ -645,11 +645,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _is_hit = _tag_match && co.isHit(xact, coh) val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir().full(coh.sharers) & ~io.tile_incoherent + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent val mask_self = mask_incoherent & ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) pending_probes := mask_self - release_count := co.dir().count(mask_self) + release_count := co.dir.count(mask_self) crel_had_data := Bool(false) crel_was_voluntary := Bool(false) } @@ -663,9 +663,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_probe) { - io.inner.probe.valid := !co.dir().none(pending_probes) + io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { - pending_probes := co.dir().pop(pending_probes, curr_p_id) + pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 9d6d768b..d7bbd25f 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,46 +3,35 @@ package uncore import Chisel._ -abstract class CoherenceMetadata extends Bundle +abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + def ===(right: ClientMetadata): Bool = this.state === right.state + override def clone = new ClientMetadata().asInstanceOf[this.type] +} object ClientMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy) = { + def apply(state: UInt) = { val m = new ClientMetadata m.state := state m } } -class ClientMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata()(c).asInstanceOf[this.type] -} +class MasterMetadata extends CoherenceMetadata { + val state = UInt(width = co.masterStateWidth) + val sharers = UInt(width = co.dir.width) + override def clone = new MasterMetadata().asInstanceOf[this.type] +} object MasterMetadata { - def apply(state: UInt)(implicit c: CoherencePolicy): MasterMetadata = { + def apply(state: UInt, sharers: UInt) = { val m = new MasterMetadata m.state := state - m.sharers := c.dir().flush - m - } - def apply(state: UInt, sharers: UInt)(implicit c: CoherencePolicy): MasterMetadata = { - val m = apply(state) m.sharers := sharers m } + def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) } -class MasterMetadata(implicit c: CoherencePolicy) extends CoherenceMetadata { - val state = UInt(width = c.masterStateWidth) - val sharers = UInt(width = c.dir().width) - override def clone = new MasterMetadata()(c).asInstanceOf[this.type] -} -/* -class MixedMetadata(inner: CoherencePolicy, outer: CoherencePolicy) extends CoherenceMetadata { - val cstate = UInt(width = outer.clientStateWidth) - val mstate = UInt(width = inner.masterStateWidth) - val sharers = inner.dir.sharers.clone -} -*/ abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt @@ -77,7 +66,7 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } -abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { +abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int def nMasterStates: Int def nAcquireTypes: Int @@ -143,7 +132,7 @@ abstract class CoherencePolicy(val dir: () => DirectoryRepresentation) { } } -class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 def nMasterStates = 2 def nAcquireTypes = 1 @@ -184,18 +173,18 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientValid - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid))(this) + ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers, src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -203,8 +192,8 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -263,11 +252,11 @@ class MICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(di def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -307,27 +296,27 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientExclusiveClean - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean)))(this) + clientExclusiveClean))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientExclusiveClean, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -335,8 +324,8 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -401,12 +390,12 @@ class MEICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 def nMasterStates = 2 def nAcquireTypes = 2 @@ -452,26 +441,26 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty)))(this) + clientExclusiveDirty))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -479,8 +468,8 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -525,7 +514,7 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -552,12 +541,12 @@ class MSICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 def nMasterStates = 2 def nAcquireTypes = 2 @@ -603,13 +592,13 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def needsWriteback(m: MasterMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state))(this) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -618,16 +607,16 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, clientExclusiveDirty, clientExclusiveClean), grantReadExclusiveAck -> clientExclusiveDirty - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, probeCopy -> m.state - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -635,8 +624,8 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -681,7 +670,7 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, - Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck @@ -708,12 +697,12 @@ class MESICoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy( def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } -class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePolicy(dir) { +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 def nMasterStates = 2 def nAcquireTypes = 3 @@ -760,12 +749,12 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state))(this) + clientMigratoryClean -> clientMigratoryDirty)), m.state)) def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( MuxLookup(cmd, clientInvalid, Array( M_INV -> clientInvalid, M_CLN -> clientShared - )))(this) + ))) def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( Mux(incoming.uncached, clientInvalid, @@ -779,7 +768,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo acquireInvalidateOthers -> clientMigratoryDirty, acquireReadExclusive -> clientMigratoryDirty, acquireReadShared -> clientMigratoryClean)) - ))))(this) + )))) def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, m.state, Array( probeInvalidate -> clientInvalid, @@ -791,10 +780,10 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo clientSharedByTwo -> clientShared, clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) - )))(this) - def masterMetadataOnFlush = MasterMetadata(masterInvalid)(this) + ))) + def masterMetadataOnFlush = MasterMetadata(masterInvalid) def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir().pop(m.sharers,src))(this) + val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -804,8 +793,8 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo )) } def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir().push(m.sharers, dst))(this) - val uncached = MasterMetadata(masterValid, m.sharers)(this) + val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = MasterMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -853,7 +842,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def getGrantType(a: Acquire, m: MasterMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir().none(m.sharers), grantReadShared, grantReadExclusive), + acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), acquireReadExclusive -> grantReadExclusive, acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) @@ -883,7 +872,7 @@ class MigratoryCoherence(dir: () => DirectoryRepresentation) extends CoherencePo def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir().none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index fcdbdd9f..411d9517 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -4,16 +4,22 @@ package uncore import Chisel._ case object NReleaseTransactors extends Field[Int] +case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object NIncoherentClients extends Field[Int] +case object NCoherentClients extends Field[Int] case object L2StoreDataQueueDepth extends Field[Int] -case object NClients extends Field[Int] +case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CoherenceAgentParameters extends UsesParameters with TileLinkParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nClients = params(NClients) + val nCoherentClients = params(NCoherentClients) + val nIncoherentClients = params(NIncoherentClients) + val nClients = nCoherentClients + nIncoherentClients val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases @@ -41,7 +47,7 @@ abstract class CoherenceAgent(innerId: String, outerId: String) extends Module } } -class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends +class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) { val internalDataBits = new DataQueueLocation().getWidth @@ -143,11 +149,12 @@ class L2CoherenceAgent(bankId: Int, innerId: String, outerId: String) extends abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, nClients, tlDataBeats) = (params(TLCoherence),params(NClients),params(TLDataBeats)) + val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) + val nClients = params(NCoherentClients) + params(NIncoherentClients) val io = new Bundle { val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, params(NClients)) + val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) val has_release_conflict = Bool(OUTPUT) } From bfcfc3fe18ba88c45fec946324ac3e8bcff1c417 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 17 Dec 2014 14:28:14 -0800 Subject: [PATCH 270/688] refactor cache params --- uncore/src/main/scala/cache.scala | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08533ef9..a63fafff 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -30,8 +30,6 @@ abstract trait CacheParameters extends UsesParameters { val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) } abstract class CacheBundle extends Bundle with CacheParameters @@ -99,7 +97,12 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { } abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters + with CoherenceAgentParameters { + val idxMSB = idxBits-1 + val idxLSB = 0 + val refillCyclesPerBeat = params(TLDataBits)/rowBits + val refillCycles = refillCyclesPerBeat*params(TLDataBeats) +} abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters @@ -415,13 +418,13 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.bits.data := xact_data(local_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) @@ -505,7 +508,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(untagBits-1,blockOffBits)) + val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires @@ -535,7 +538,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(untagBits-1,0) === c_acq.payload.addr(untagBits-1,0) && + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || @@ -598,13 +601,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.write.bits.data := xact_data(local_data_write_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(untagBits-1,blockOffBits) - io.meta.read.bits.tag := xact_addr >> UInt(untagBits) + io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(untagBits-1,blockOffBits) + io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(untagBits) + io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_grant when(collect_cacq_data) { From d121af7f942e3d1f25084e9d3a7000ef3f03fd77 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 18 Dec 2014 17:12:29 -0800 Subject: [PATCH 271/688] Simplify release handling --- uncore/src/main/scala/cache.scala | 22 +++++++++------------- uncore/src/main/scala/uncore.scala | 17 +++++++---------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a63fafff..e3a8db22 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -303,17 +303,13 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle releases, which might be voluntary and might have data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, Mux(any_release_conflict, conflict_idx, UInt(0)), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -353,7 +349,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO } @@ -391,8 +387,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -541,9 +536,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data - io.has_release_conflict := (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && - (state != s_idle) + io.has_release_match := !co.isVoluntary(c_rel.payload) && + (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || + co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + (state === s_probe) val next_coh_on_release = co.masterMetadataOnRelease( c_rel.payload, diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 411d9517..d98bf980 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -94,19 +94,16 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } // Handle releases, which might be voluntary and might have data - val any_release_conflict = trackerList.tail.map(_.io.has_release_conflict).reduce(_||_) - val block_releases = Bool(false) - val conflict_idx = Vec(trackerList.map(_.io.has_release_conflict)).lastIndexWhere{b: Bool => b} - val release_idx = Mux(voluntary, UInt(0), conflict_idx) + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} for( i <- 0 until trackerList.size ) { val t = trackerList(i).io.inner t.release.bits := release.bits t.release.bits.payload.data := (if (i < nReleaseTransactors) DataQueueLocation(rel_data_cnt, inVolWBQueue) else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) && !block_releases + t.release.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) && !block_releases + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) // Wire finished transaction acks val ack = io.inner.finish @@ -156,7 +153,7 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) - val has_release_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) } val c_acq = io.inner.acquire.bits @@ -184,8 +181,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - (state != s_idle) + io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -278,7 +274,8 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && (state != s_idle) && !collect_inner_data - io.has_release_conflict := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + !co.isVoluntary(c_rel.payload) && (state != s_idle) val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), From f234fe65ce4eb88d09fd10c0707a6bcf2d6f1269 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 03:03:53 -0800 Subject: [PATCH 272/688] Initial verison of L2WritebackUnit, passes MiT2 bmark tests --- uncore/src/main/scala/cache.scala | 377 ++++++++++++++++---------- uncore/src/main/scala/coherence.scala | 21 +- 2 files changed, 253 insertions(+), 145 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3a8db22..3cbc8967 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -108,7 +108,7 @@ abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { - val id = UInt(width = log2Up(nTransactors)) + val id = UInt(width = log2Up(nTransactors + 1)) } trait HasL2InternalRequestState extends L2HellaCacheBundle { @@ -284,32 +284,36 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac Module(new L2AcquireTracker(id, bankId, innerId, outerId)) } + val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) + (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.valid := alloc_arb.io.in(i).ready + val acquireList = trackerList.map(_.io.inner.acquire) + acquireList zip alloc_arb.io.in map { case(acq, arb) => + arb.valid := acq.ready + acq.bits := acquire.bits + acq.valid := arb.ready } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && !block_acquires + acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires - // Handle releases, which might be voluntary and might have data + // Wire releases from clients val release = io.inner.release - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.valid := release.valid && (release_idx === UInt(i)) + val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ + wb.io.has_release_match).indexWhere{b: Bool => b} + val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + releaseList.zipWithIndex.map { case(r, i) => + r.bits := release.bits + r.valid := release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + release.ready := Vec(releaseList.map(_.ready)).read(release_idx) // Wire finished transaction acks val finish = io.inner.finish @@ -321,27 +325,174 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) // Wire probe requests to clients - doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outerList = trackerList.map(_.io.outer) :+ wb.io.outer + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), {case TLId => outerId}) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read)) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp)) + doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } +class L2WritebackReq extends L2HellaCacheBundle + with HasL2Id { + val addr = UInt(width = tlAddrBits) + val coh = new MasterMetadata + val way_en = Bits(width = nWays) +} + +class L2WritebackResp extends L2HellaCacheBundle with HasL2Id + +class L2WritebackIO extends L2HellaCacheBundle { + val req = Decoupled(new L2WritebackReq) + val resp = Valid(new L2WritebackResp).flip +} + +class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { + val io = new Bundle { + val wb = new L2WritebackIO().flip + val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip + val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) + val tile_incoherent = Bits(INPUT, nClients) + val has_acquire_conflict = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) + val data = new L2DataRWIO + } + val c_acq = io.inner.acquire.bits + val c_rel = io.inner.release.bits + val c_gnt = io.inner.grant.bits + val c_ack = io.inner.finish.bits + val m_gnt = io.outer.grant.bits + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + + val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_coh = Reg{ new MasterMetadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_id = Reg{ UInt() } + + val crel_had_data = Reg(init = Bool(false)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = co.dir.flush) + val curr_p_id = co.dir.next(pending_probes) + + val (crel_data_cnt, crel_data_done) = + Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + val (outer_data_write_cnt, outer_data_write_done) = + Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) + val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + + io.has_release_match := !co.isVoluntary(c_rel.payload) && + co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + (state === s_probe) + + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, + UInt(trackerId), + xact_data(outer_data_write_cnt)), + { case TLId => outerId }) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // Never gets mgnts + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.write.valid := Bool(false) + + io.wb.req.ready := Bool(false) + io.wb.resp.valid := Bool(false) + io.wb.resp.bits.id := xact_id + + switch (state) { + is(s_idle) { + io.wb.req.ready := Bool(true) + when(io.wb.req.valid) { + xact_addr := io.wb.req.bits.addr + xact_coh := io.wb.req.bits.coh + xact_way_en := io.wb.req.bits.way_en + xact_id := io.wb.req.bits.id + val coh = io.wb.req.bits.coh + val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + when(needs_probes) { + val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent + pending_probes := mask_incoherent + release_count := co.dir.count(mask_incoherent) + crel_had_data := Bool(false) + } + state := Mux(needs_probes, s_probe, s_data_read) + } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := !co.dir.none(pending_probes) + when(io.inner.probe.ready) { + pending_probes := co.dir.pop(pending_probes, curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + xact_coh := next_coh_on_rel + // Handle released dirty data + when(co.messageHasData(c_rel.payload)) { + crel_had_data := Bool(true) + xact_data(crel_data_cnt) := c_rel.payload.data + } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(crel_had_data, s_outer_write, s_data_read) + } + } + is(s_data_read) { + io.data.read.valid := Bool(true) + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_read_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } + when(local_data_resp_done) { state := s_outer_write } + } + is(s_outer_write) { + io.outer.acquire.valid := Bool(true) + when(outer_data_write_done) { + io.wb.resp.valid := Bool(true) + state := s_idle + } + } + } +} abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { val io = new Bundle { @@ -352,6 +503,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO + val wb = new L2WritebackIO } val c_acq = io.inner.acquire.bits @@ -423,6 +575,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, xact_meta.coh, xact_src) + io.wb.req.valid := Bool(false) when(collect_inner_data) { io.inner.release.ready := Bool(true) @@ -482,8 +635,9 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou } } + class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_probe :: s_data_read_wb :: s_data_resp_wb :: s_outer_write_wb :: s_outer_read :: s_outer_resp :: s_data_read_hit :: s_data_resp_hit :: s_data_write :: s_outer_write_acq :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -499,32 +653,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) val crel_had_data = Reg(init = Bool(false)) - val crel_was_voluntary = Reg(init = Bool(false)) - val crel_wb_src = Reg(init = UInt(0, width = log2Up(nClients))) - val crel_wb_g_type = Reg(init = UInt(0, width = co.grantTypeWidth)) - val wb_buffer = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val wb_addr = Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) + val curr_p_id = co.dir.next(pending_probes) + val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) + val mask_self = Mux(co.requiresSelfProbe(xact), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.tile_incoherent val collect_cacq_data = Reg(init=Bool(false)) //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = + val (cacq_data_cnt, cacq_data_done) = Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = + val (crel_data_cnt, crel_data_done) = Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = + val (cgnt_data_cnt, cgnt_data_done) = Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = + val (outer_data_write_cnt, outer_data_write_done) = Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = + val (outer_data_resp_cnt, outer_data_resp_done) = Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) @@ -537,22 +690,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (state != s_idle) && !collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - (co.isCoherenceConflict(xact.addr, c_rel.payload.addr) || - co.isCoherenceConflict(wb_addr, c_rel.payload.addr)) && + co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_release = co.masterMetadataOnRelease( - c_rel.payload, - xact_meta.coh, - c_rel.header.src) - val next_coh_on_grant = co.masterMetadataOnGrant( - c_gnt.payload, - xact_meta.coh, + val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_write_wb = Bundle(UncachedWrite(wb_addr, UInt(trackerId), wb_buffer(outer_data_write_cnt)), + val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) @@ -561,25 +706,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data - val cprb_for_cacq = Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cprb_for_mvwb = Probe(co.getProbeTypeOnVoluntaryWriteback, wb_addr) - //TODO cprb_for_mprb io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Mux(!xact_tag_match && needs_writeback, - cprb_for_mvwb, - cprb_for_cacq) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) - val cgnt_for_cacq = Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) - val cgnt_for_cvwb = Grant(Bool(false), crel_wb_g_type, UInt(0), UInt(trackerId), UInt(0)) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := Mux(crel_was_voluntary, crel_wb_src, xact_src) - io.inner.grant.bits.payload := Mux(crel_was_voluntary, cgnt_for_cvwb, cgnt_for_cacq) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), + xact_client_xact_id, + UInt(trackerId), + xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -604,7 +742,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_grant + io.meta.write.bits.data.coh := next_coh_on_gnt + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) @@ -641,101 +785,50 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val _needs_probes = co.requiresProbes(xact, coh) when(_needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - val mask_self = mask_incoherent & - ~(!(co.requiresSelfProbe(xact) || _needs_writeback) << xact_src) - pending_probes := mask_self - release_count := co.dir.count(mask_self) + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := co.dir.count(mask_incoherent) crel_had_data := Bool(false) - crel_was_voluntary := Bool(false) } state := Mux(_tag_match, - Mux(_is_hit, - Mux(_needs_probes, s_probe, s_data_read_hit), - Mux(_needs_probes, s_probe, s_outer_read)), - Mux(_needs_writeback, - Mux(_needs_probes, s_probe, s_data_read_wb), - s_outer_read)) + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe } } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_read } + } is(s_probe) { + // Send probes io.inner.probe.valid := !co.dir.none(pending_probes) when(io.inner.probe.ready) { pending_probes := co.dir.pop(pending_probes, curr_p_id) } - // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_release - + xact_meta.coh := next_coh_on_rel // Handle released dirty data when(co.messageHasData(c_rel.payload)) { crel_had_data := Bool(true) - when(xact_tag_match) { // Hit, so merge new write with released data - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - io.inner.release.bits.payload.data) - } .otherwise { // Miss, we're voluntarily evicting this data - wb_buffer(crel_data_cnt) := io.inner.release.bits.payload.data - } + //TODO make sure cacq data is actually present before merging + xact_data(crel_data_cnt) := mergeData(xact, + xact_data(crel_data_cnt), + c_rel.payload.data) } - - // Voluntary releases don't count against the release countdown - // because we will get a further release ack from that client in - // response to our probe. We don't send a grant acknowledging - // a writeback or decrement release_count until we've received - // all the data beats. - when(co.isVoluntary(c_rel.payload)) { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - crel_was_voluntary := Bool(true) - crel_wb_src := c_rel.header.src - crel_wb_g_type := co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh) - } - } .otherwise { - when(!co.messageHasData(c_rel.payload) || crel_data_done) { - release_count := release_count - UInt(1) - } + // We don't decrement release_count until we've received all the data beats. + when(!co.messageHasData(c_rel.payload) || crel_data_done) { + release_count := release_count - UInt(1) } } - - // If we saw a voluntary writeback, we need to send an extra grant - // to acknowledge it. - when(crel_was_voluntary) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - crel_was_voluntary := Bool(false) - } - } - - when(release_count === UInt(0) && !crel_was_voluntary) { - state := Mux(xact_tag_match, - Mux(is_hit, - Mux(crel_had_data, s_data_write, s_data_read_hit), - s_outer_read), - Mux(crel_had_data, s_outer_write_wb, s_data_read_wb)) - } - } - is(s_data_read_wb) { - io.data.read.valid := Bool(true) - when(local_data_read_done) { state := s_data_resp_wb } - when(io.data.resp.valid) { - wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data - } - } - is(s_data_resp_wb) { - when(io.data.resp.valid) { wb_buffer(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write_wb } - } - is(s_outer_write_wb) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_wb - when(outer_data_write_done) { - state := s_outer_read + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) } } is(s_outer_read) { @@ -754,20 +847,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: set pending client state in xact_meta.coh when(outer_data_resp_done) { state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read_hit) + s_data_write, s_data_read) } } } - is(s_data_read_hit) { + is(s_data_read) { io.data.read.valid := Bool(true) when(io.data.resp.valid) { //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_read_done) { state := s_data_resp_hit } + when(local_data_read_done) { state := s_data_resp } } - is(s_data_resp_hit) { + is(s_data_resp) { when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index d7bbd25f..c7999f4b 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -117,6 +117,7 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool def requiresSelfProbe(a: Acquire): Bool def requiresProbes(a: Acquire, m: MasterMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool @@ -253,6 +254,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -391,6 +393,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -541,7 +544,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -697,7 +704,11 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } @@ -872,7 +883,11 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + Mux(dir.one(m.sharers), Bool(true), + Mux(a.uncached, a.a_type != Acquire.uncachedRead, + a.a_type != acquireReadShared)) + def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) } From 2ef4357ca8492fd08e00a94692e95ab7bbdd82bd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 19 Dec 2014 17:39:23 -0800 Subject: [PATCH 273/688] acquire allocation bugfix --- uncore/src/main/scala/cache.scala | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3cbc8967..932392e3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -293,14 +293,19 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Handle acquire transaction initiation val acquire = io.inner.acquire - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val acquireList = trackerList.map(_.io.inner.acquire) - acquireList zip alloc_arb.io.in map { case(acq, arb) => + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := arb.ready + acq.valid := acquire.valid && (acquire_idx === UInt(i)) } + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires alloc_arb.io.out.ready := acquire.valid && !block_acquires @@ -367,7 +372,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } @@ -500,6 +504,7 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) val tile_incoherent = Bits(INPUT, nClients) val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO val meta = new L2MetaRWIO @@ -539,6 +544,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou Counter(io.data.write.fire(), tlDataBeats) io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) io.has_release_match := co.isVoluntary(c_rel.payload) io.outer.grant.ready := Bool(false) @@ -689,8 +695,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && (state != s_idle) && !collect_cacq_data + io.has_acquire_match := co.messageHasData(xact) && + (xact.addr === c_acq.payload.addr) && + collect_cacq_data io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact.addr, c_rel.payload.addr) && + (xact.addr === c_rel.payload.addr) && (state === s_probe) val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) From e62c71203ed2b0e2deca8d860c9e52ad26c94e5e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 22 Dec 2014 18:50:37 -0800 Subject: [PATCH 274/688] disconnect unused outer network headers --- uncore/src/main/scala/cache.scala | 2 -- uncore/src/main/scala/uncore.scala | 2 -- 2 files changed, 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 932392e3..d871e652 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -413,7 +413,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str UInt(trackerId), xact_data(outer_data_write_cnt)), { case TLId => outerId }) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) @@ -712,7 +711,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(true) //grant.data -> xact.data io.inner.probe.valid := Bool(false) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d98bf980..e8e82d22 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -198,7 +198,6 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute xact_client_xact_id, UInt(trackerId)) - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.acquire.bits.payload := Bundle(UncachedWrite( xact_addr, UInt(trackerId), @@ -286,7 +285,6 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default - io.outer.acquire.bits.header.src := UInt(bankId) io.outer.grant.ready := io.inner.grant.ready io.inner.probe.valid := Bool(false) From c76b4bc21ddd1052195f593bf6071bc2e7151634 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:18 -0800 Subject: [PATCH 275/688] TileLink doc --- uncore/doc/TileLink3.0Specification.pdf | Bin 0 -> 123869 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink3.0Specification.pdf b/uncore/doc/TileLink3.0Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..363f9b03d037b340981ea2f27c752bbd2741a20c GIT binary patch literal 123869 zcmb5U1ymg0(&!u9Ew~Tv4uiY9LkJLp+u%cR3GS}Jf;$9vcb7nL2<}cG_#6J;Ip00^ z-uK?Rcdc1_Pghsfu3c5#J+t~pqb4Q8!okXoOtX1-{1cgj2fz++FtI@v5@OTxa5M+7 zNg9KVK@OJ4Y-+}q<}LsZ$axK95fNl_d$Tt^4&eW4@TT{s?QCuj{+BwNhPjJ_tFtM@ z9^zNc*}+uP9P9$nXM-#bVAC>p2LssT?I3m%|C%NLHOm_!vq`&yWi`RZVDmRyS#|&i z@4rJ}lW~A70g=i;q@!0@$=Q?pv-=H}+cYU$u$ z2{LCjb+BV|bq0w#3BR4A;S#6el!1J}kmb#tSxrq@Eg@+zwPH0lbA^Z<%`99Xld`;| zhAKC=k%GC4i@A}6xhpH=1g(p)uoSI}J=jy2gVxo>QdnF@-9}PHR$Ix<(ZmGgqNU=j z;r5Z!MZ);Q7k(o(NfT{#hPSGUh(H3DgrpTxJQo1x|61*T1pjIe8EX(2k`6W*5F{um zb5jR1NZFOl?JdDp0Dg8(4u~&+3)tD**bdn}GuiOC%zVV|Wkc2J6JW))OAOKMgz!~` zU>GK75z`Sc`6zi)`fIsdVR_cN?NZ4lrbvOtx)faf7x!RFRQf?S=kQqc^>*9O?>8L_ z?Mu(b>+z7E_cMm?-=k#X4L^#-zrmufFZ@R})BMqX8CeAy`MzF;ZuII#7)947rTpnnWwWfqK8sR7m6&$0+9PKA|;;ew6`xSWO_iEFSWXVO!g4RToLmf9{IN zB?+y77HW2x3AokQID-Qa#%`_iB96XLXxK%g=7Hk;Y1@CcnIPd2JcAxbFJ|I1kM{*! z{##w)o=4^w{ZON-+CbtK$QV7%awgNzX@b<&KIeL1k&wA~(H>OJq#)Dg8zE zm590ID70xhpl+y@1CD(qVVOI8sMHRiU?acI=;VtKuc3cro0Ro z^_Vg3mqhuIm1nN9Zy-}nY9Hd``~IyBDdmGdJp=9GQRxXFAwSt2OL0XkVzuNwn@z?e zqsFv+4G=HO5|_cyfv`dKCS}GFSH2Z%SLUP4p%im|LM!jj@TN;L1SiX|8nCOAqT&=990KkGig=SkJi$e4!GcPaQv zw47poWSRlu;!g!uX#Ii#+#I7}Gwk&{)kGw8A`yy+`^HD0aIS_(;_4V!?~&pct^oG- z88w<{DE+c$#PK7TG65f(lJRrfbL4!JOQ&VbQA5F{V!8XJGaaIMtIBb$Gqb!zwZIAM zM_UHC`r0&=Mv|%)w1TH{oGvK~w9+m#DPLr;%hfI<5fo=GhE>F2PtLNRB5YwiSg}N8 zo*A27uK&od;@vYKn_t(4G-PP5i4}+q$%MJpw)0}jTRuoIJ)6SZOHsVMki>Dw;fVKx zYE2qxEQ!PB6cT|)cB1j$U8;=01*il@@B3m#RcDrfbhe*7AEDHbLeFGtZy>oiP`Wc+ zOI4GiwNFWM!!*wlU&p{~7z=7oC0$VUWX}dD)W9y66g5{<{sAnxn^5hp$B|LzW=-bi zBDiEc%3S&SN9&GIFN!7n@Y!6yC{9&U6yANLYMP-DPJ3iU)IP}28k%j2kZyACO;Ng> z?`yj9AUV*#FBqeC)x?OSsk`yF!;JPlgj!k6{4)zv^qe9=D|z`&{Z2WGnqLjKNcx+J z{@o843GQ+8MpAs&K$R_z%*pWKPjuO-J6U1g?VbU1qMiJsTGd8oJ{O!D20kA8JC>*e z+{z)d^IZfs9~n8OB6ELg!8I?W>o-2@u=R~XS?_9`G`1}LEL^n9$CV*t`m%yf@?$Ah zyLQ pO-J;f%oH7lIvKd%zVF`ps9`E2dM)y7+}7a7o6ZImrk=cR_RNbe3eX)`+06 zj{NAbtuyW)mN=ZynjF1#5~RHssNBvVyd9#ed{Dh2`An{_jZ_i=`s^sSn}CY4p!NJI z7bYv}z=~CzR)lteKn{3EFR^cr#43NaoOadxp7ukh*%HpXmK;BH$-1Qm-mANmbC|rd z3q0MwM^L4s>uF5h4PQ$z@KfGpIjtN7)Th$mrY9DpEbQ;&736hKzl^w%aC6 z%*oqa=^Z1d?1>Cnl4>XG(stVF1XUtP!681HLz`P)uhY@lHUyc&+|!Xi>iHh+#5R=0 zxO%`XMOAck1b~(mL6srd9ombZ3l#-ce$hWFgXJw(ih_GQ0}v?_l(qf#;A!`b^>>QH z{-$Zo))kNv_qr^hSH$;%E|AGE?XIR25$(#$?{<4F2;aBGPRUKhq4v!cdmnO`4mRiD7NMCE2kxcHcpOg!8&J$jHTKXgfH zb^!!neu)4C2H5dzBP#_JKPe3A6%H1fHEIVGaamGCV8hE-3`UY}efLU-p_!xvAr769 zT75dI@j=n!g2F$Ffr zbm!K3VCA<>u?m0dKiCs<%Ip9cBlL8~*YpafbnR)*OX$h#s~0EW^SDWTj4ncXF=^(V zJZg!V$Jd3yL)1;@f6?c2>Ey@w zIwj>>l=9fePGabHra#4>S%!x1;B+A0*RIPrwSN>qr-tP%mN{~#-5)13r{rjTG^13^ znM{?V#W-wE>yVbS8|Mh2LT;HAftE5+BwGf6JGc_nvArhOWp?top(y#KJ{l*nlT^m3 zpGfivf*GihcO2;~^WUo#X$sd+FVUGFmbSb{TKR}!&M^L%@Ltc_T7Xh_xO1WsvL4** zti+`wg>`A^`GEqdOe)T%bAa6F3~qw_zGwL~DJN`LSXv61{BF?}R;oRs2Fbd%mYjj7 z!1<@}v4Ut7OzAaw;)ZJp?1O~s=Wn?lwQ8nbg`A{#+N}T$h8g`J*02a7Zhie(wCiuy z?z~urlhu@BJ2C~YUa{zQwuL@dPS_)C1@CjVJwk`65_JHP!U9)XBRcdnZL;tgvVU01 zk%2h?8!8hnhTTKVDy)ocB+9APVJoHU=Bb^^#e)D$V)=MmBzP~m$y{Q>X#S!jlGb=t zK^4rA0UfLSO$z;Q%Q7Y!he8eRHdWfa)P*N;UE(-aydCj}F!g=FSHjrb^x&dg<_%vM zb$io~ozF1u3msIK+k2Y)#FCm)j+|e4F_T$FPhK1_M~$16PZ0F2Nm5t(?eR3#Fj|yO zEOBuw!kBQ{H?MLn0yR}88FzdevGEOI@N}QY;qyTEjaw1KVG1t$$$)x{EQu~5=)@N} z57yXjGqDJlWS&K^d^k`dsYe8h`1*2HjINU~?IE|N6fFx1WNb@Kg%GG`U=P720-s%&=m&O7yzZmQv#`>@Za zq1hNmifwqmM9MVm;LYFTP1f-fi+_ltsOyST+VB<&85eP3^`Mynm71B7soP{QwV>PpVg~Y5G7AN?Y9NLB zSCk!io1b77NairwCRTJWslw5CTLAC!+Yr~D&`2?EU3DP6>){PKfX;ixO_1Sq3FCLEJgytZQB2dPk_B2%a*k9TuSVTche74U0(K|{VW zHKNIc55`nV7|~_3+mF9-+r%FKAf&ROl1Vo6leEF;EM??CAdLCC9f#(bPH*VG`4Wfm z`u6iUSCgX`JFSy`yD-7eMvxQc@I8lum*`A7To@mC>_WJGs-efPV!o~NH}>={KWS|} z(WgRfRj3+OOi!1CDG~AXpPMenj)5`Na(K|dSrlj7kv#c>$(?jPyZ89AqH=fT=T0|q zJmChfW*^G_UaE#jPYsC@ri_YzjKucE2y;gMoipwXy2<6DOyoNwX26Hn0d{U0UjlnU zEQscd&2gACjj^FLmBT&@F1V+C_4A)aEeN$du8d?|F}{hks7~H6vuO*E|cm z$9P~v5X$U*#A&eUl_-6oI!rTc$dah6pZHX=tGrKHrh*T&zJTSeQU3Og z3Mt%f7PBKd&X(Rf71pEf`|rx&!*J1JDo;0SMcy&y0c^i32Uf*ti|ck4aboEm`k4WSdv;;db)XyO0Rs4z2 zlBkN2>v<0A@SM#OB%Glk&(uLes-3=~Cu6-3J-~C}MtF+Fbh#3lGIDlOOKh0kt~G=qUw0Q97z1_2Y8Z?L77o_6wJRbqqy zvwN}Nyrrz_lHQ$hEvF1HOXrk)mX3#hnt#Pa>Jh6cE}w_GTez7z)j^@0`ai-kH=NL2HthIvAc$+y$y1jQCDKa&C5w z^B1OdPV-`Dv8@c4#WSv;?SQ#aK5;M_j$IJ!*Jyi*fQrz~Togs@WZ4xN@+;caiR}sv zq*S6ZD%8@RF8Mw|u8Xm;4x)FHdS4|4yQU36Ud zPq<)>Nk**6NIDe4sO)U|9n^(F_c9p0_@!@jDUJeLEfE43s1ul>R4#-%JD1{2vgNrd zWi5~0ETu~WcwcCdYe{etz7%~v^tgBYti3)xQxF2xx|4dU_(7lpuSm zOFVOtV?IIx6XX#X88y!^Z-za#=hyw6ZAgY(p{=(i;tY<#7Ig@o(y8Z0I%?BVMa%d) z)|6_VTONF9ZY1RzsFdAXBQCeFHJJedjE1&c(N4yXd|;qB89ow+3ZHu?H7~?DXc(j( z8HhrsqH!)5M&u?=C-6=y79LsO(OPI<-6PS7@VcntGu~WHbhbmnE)k=!rtzt+s)pJf zP27B;@sC?4pQYh1Nt>DE=?WSCAQyMy9vfu_Pv#)N4^sPW=yIMRpL|tU!^AUGgj7@} z>c%a04!UtvVex{}$z%b9Fvg;QM9r9UwSB2DS!k(0P6pQ4`pjGvm^+rB+1wl3?DB(g zWM|420ijV$cxoe%@n?Ma5q$l2E+vhc2*#CwYYy~F9r^ebbie6m?SS#*@*hiKtZ0zp zA;n2bQV#E_e~tbsgPzd}$Is2}ytsgR)TnWgeMFDj5^*G_C60XRuV#>)v?8@Jm;I_1FECNiia@ME8Hn% z6>0aRyhwc-jvwo?fUf#M-WO3NRYs=qiS-!U9nBdX@aa31h<-`KqgK{Dd*P~k#yX)3 zAr;Ki%x+{++$^xuXZyVa!)x3<+ITmZ@D11Zah0Vd2g@0+ax3Cq3-8jSM<*(VJb4)% z06O{3Tui0w(W4k)OC=}2op~8~>&lMpwd`vZYyv+(^q9%Kco_xL45CMgDP{6&>n7Gp zR>$+$qV#u4877!X`=~2$hj?xKa}eA0{O_fHpCUiI!LH*3;S82tZ#b%Fq3AaGE2ky} zEX(0ypjBNr?Y=V{78+U7gS8tj1fGitN&8BLzJRt(UXqjz#(NbGnp}0hPP{MclLFI> z&(={nuLgSuSZ&{w>2B!M5GC7zT2S~&6Uz0Y`XZ`U=Kr|m)Ur5(Hk}Pt?dkpNG$TSs z){5I%^0v;UaAP7fdV8Z$u5tKV6gmReZy8U~xmOMv1rO{L%+K7Im+j-56+=(hb_@l~ z%14D9@@Hl6LsSXZ|NKx0os8CMb1@PgY}^9C{I30UO5_uUqF?=mSUTV=J{lF!8MoLl>WvyaWAa>W_1%55V5dB5=B@D^@rB1)fsbU=BT8TVx)+X2MEAYl59g z^$+FsdpGMiWqg9s#LCZgM;!^C^6spIJ%!fbbDZ5=}g*tn}6xp`Pl8(5+F+ zNO?NxZU?{SNLju+2zXHroTBYBGES@Z0)VqVKNk`?8xw58fdXBgxPNHZIN-uC;cHiS z8O6z34bYY+b4S<@t*+OB8q#v~lSf0j+I}#e$lThQG5g=+oeeSCK0cG;`ynTqo!pY* zH!u9B^^Kh?sx67275DzvRBsGqKx~ zBj$(mvD=%X6u+}WT4BtQ!7jwhigxE#Cl+LFHT+(ks@JHUG_dhYXbDZK74PuK#jZBS z?i%?F&>{aSwP7hB4z_9^j;_~@(~&$4i9CUK7ev365!xzIjRjw@uyLOGJahSlrOs87qdntc^S?t~$9#w8^@*A0(bQ@^Ns<)BI3$Ik*R z?{w|SF4=tTfM&(2)yF@Ymp=KvQgyG6&U*epUVQj@BG&oG>B18|f|QK$L z^|Oc|tF=-q4W%h+5Bc~Dz=VLWG%kSq=J$=eEtm zT~pBaEipHC^3%azN2{)Y1M`V74CtGO@EpujtgZEQd_}bDF_&@w+b@DO6-!yT{XH_&gsCjA zRWD&US9${lE(S`q#j$h zAR#CVuCR_$>>bl$sm$!*T+IN%6=U#zLl^! zXnfrzBgA53h08F(>OpjbWDlM%i?tJzb4Ynl5$?OqptS%Mq^fIYvF-zG!?o>q8|QN@ zoG9;DZ+|mru~JIJjO}fC-6jU2+-+j0vbIbkD$@eiH0xA|l}Vrv z)A5`c-Rhhyup`s@`Y@WrkJ^3lo1~bE&A@t2@yZ?CB+iHG4~2@*-xD*z5IAd6-(qycdC9c(C4cDa@&m+ z3>CMQDKnVIk_OnOS%&U>x>M_v&t=sv2O$WQYC@KE>22_fy0!9x4zj~lHTyLm3;F1_ zdl)C4t<``W`WsY0p)0|E z-JhXOha0E0(?0FCUur3AHTs-JUun}=Tw9s56t$^$c-mx1Za*V{$@!YQACD&t;jB>_ zKPr=HyZeC}mT_O*H&8J?E=J^>=gJ(!|F{&oZyz{EmV_Zf}g1(n?y1 zSH%=&WT;p-KrlqL5f&HeIO{ey0lIpqF)7rYu)PcX_&C1)b=l@Zd*2B;U4)){0hO10 z_rrF0>-~@ZwulkEt!(wV3H(pl-EGeVtA+!R@BG=_kHkA|zmme#istRE4(crgXbX?JDAO!Es-J6uHOh@G{UaQnW?^n}W#;!VTv)S%J^Zv@I$7 z_9bqQJo@Pp?ST&G@UN`E!r)z&E#yaF3vm2u%WZ#|-F9N*zXW@~o!zR00Xh(wV(9r~ zgK-L6m*7H(-au4p$-Qy5af;$L&-E?Z@N>W(dt~HgrrSWG0C5$7FFOp34_6j*Ge@jEn)x zzmKlqme`_3>SiX~2?+2LIuv9qDJ_260ctlzKlL#}Zc@WAKU;VW%mgrMSlS6qQA}k( zOseP8gqdWaGU~*+RyY09!A4~gx-^Q!4^YL$GA&j(m&p1A%~o0!c_ExiwM|<$)1-F7 zP^Tc+RH7C`{ZmY1Be)jB3^0uWO$)N&!(5Z9Yx0bX>z_Xl?!Rf~ytaAQ{ES0Zg5|FEBVS7AT)Pe%9KT8ZM&>Fdh4&&9r1R8`S|)cHc-QNK`z&CYN?fXafrmjNxj#*S6~Y(iVkI;pI2C)}C&tgZ z3wAuz!;W~ z3+OvwP-MwvkhM?nPiEZSn-nsG(OmUJl%!KGQ*T9Fz$^@nW(e^IJK8*TP5S~eKZWx5 zp8C?!4!P08)-DIJLpgts(=0BrTiIKx=kWBk(C^O0LOx1I(^JnHj}|Deq00EdNHB_5 zb`+8Zd{Qlrbw!A`xocQ6fgd+|`WTO&2VW>sy5jb50v8 zgZe?}f607M?t(tIVk5GKr*7V3&XcqJ5w!uH1oumrV(s+wgmn(#q$3W%S%P9D;Da=; zo=QtZMJ09QZt~{DGWcts{%`CRy556itSQyj2RFFcHXGGfvR1A@Q8$cqd87JA7g*14 zBNK~`xNk4GBX|CowbhM=J()eiQ)nWZCK$p@oX(}k3)Sp39>)%+{7%WRE(%tc`lX6V z8qG2`_{{?9U90LMWS+&>HhYFuFN>>(CRUpaU)K%tq5ObN7Zy|c-t``@Mx01J)d8!m zxoQX(t-%}$!_|>{ZjajtD=b6+72tq>KWInJ1AEdbH9juP8Xta=>0{CUoGK(QmQL8U z(_x&~s(SQo#r7JLVbI=n_oa61ZYAl4n{6=) za#T849F#R)p4tN?eKlT$c7E}5reb6Qt`g|fG0RIZ38bJ{W^QG*u>SkGd9RO^7F9XnTEqz+&_aa(&y=VLVL@6qHYApcMxZ+kMUCa5%y%W)=N@HoCi zB5VpY#|5%On)gYCIQQeki_}p;v|6Rbc0i5(GY&p(ASx5LRepd1w=ept9voYB;Rzs8 zqCPyd-OeuDK2|P5rP?IbFBv0DaGLy#o$U*+=p5e@@$8NjCE;B03M z`cI_m4W;^@+`n;F|A9ljF-;I(62>m({}5PgTF&MsuBKMzU}a4RuVoKGfvoKK{Ad86t6m!$kR?)I zxPZ;=Kh#QR(AS-mmMP1baiwDncKaQfb4H| zWz&7b=k$5l-vB&z2r$RN$_eD>F$8b{AyPhmRz7a7x43!u`B))P9S;Q3<6`CHJUUBwRI=3{5&=ivqLKxQ5wklPRd zp}Lgha&83VhSyfdt9V!^HvM z;f5^82jqpM7GlK%d<%nzog3m35-2A>H!D9U1U=;AVrOON<@*nH9xfon69+`{?`0u7 z=i_4q@z=B8h+CN)||Z__-h;Bnu~yla-U5 z7r?>J$H~gh#lZ#O=jLSPHiO7$idFT@jpX`r(Pc3 zTGGD`N?Y%4J36R43KGGH^7h3O&``Bl31a>s)YR0pK|fsm6^Uv()v zS}oJGEZ3}j2lY<40tSk6d3nvQPv%w{N{S2`Gw0*4A9K&44~3TRuGrpLIT$SeC{y;7 zx_fztVK*9$U(~D-8daL?G9;xitNc^W?I-A%eG1X~`ex#&1;(McJA@#nA4@k6 zt_z5AqOt5n9wSH6>1#_4?-oy7j;h?_fsXh@ zVb|jwZgyeljULLM%9FBSsCqD5YstKI{84uSP;l4dtoZe?dhtH32uynl?K~9K@SSFbEVrhgy zL_wT`Q|r>6Oc|lZpkkX4$;3sJyv4xhYVl-u_)D!I&>~}lM6#B@x2bW?1<)+w?z11k zttgxh1D+H1RkhS=fIFQKz&q$K)(<(t;XCFa>-DefHQ2@|5?tJyo}q{8exh6a%`Xo9 z%&)N~c%_)K+g^s#R8QEn6s-8R#_yjIW+`tM#4%Wc86Q|aNU_Xb8<+1H9~eKI@b8}+ zpCd{dm$Q@6B2j%NiZP^mLPMR!_>99q5Oj{U?yqIQOSs_0<-aM?=>QMfXx^XBZhlfK z%7J1vJuU3sWeP^BpL-?gWrC?SM5kZ)jG1lLNLflhHimc{j9`qn-y{>E930KFBgd9&sT5tZYQ4HB=1+B$OG_@J_{Xmnk$k9DhkxS6XtO5OkL3hVo zue*ico>$MEvP4B!#PP26ndAbM5G|=}TW)L!c;JT04gk>+r1P0x(hR#=t5NB*SR>~ zS1?x-*0pd{eq4t{H>hVv=JVwxYOj7t+bNWAqBE!quRk{2*q7W8F@4-&#!ZHc+}t-{ zTb2Vos$}TW;N?q_DZ^3n>G3DN*dL^J4XSlytB60bsy1&W);QNLeJp#3$chh|ZkQ>o zFsLHw_I@^p$_Co1m~;;LFb1W+ni~9Wd$nlY^!(h3-)s3fqaMm5&09%y96^%6G;v^v z=5MaQk;}bA+F{H+Ris{22Ed*q4#_la2&+%DmzxaZpqfJHdoh%cIN20S`VJlkw!KO1 z?D-}O!$g-M_Wp*7-;G1+C)ve7N4%B$m#VLsrqcA1A7GVd`H~tC{v={_Y;`Q|LrHsb z#@Y4<{s>$7cIien9%8(epN0*TOrV7yRdaw>765Orvek*KY9?O@?R822Alz?L1c^Ty z=eSdW@C-3SZ<$TEds+UnUWK{z4BYSaoo=~@;j;Q`yAv_i7}BnzFXUIJY_^;CPVl** ztVZ46E6fDcJ|o%Ben=mGLHJ7)MLwiQUR9^+C_NzZ( zN{&6pN&SLtj0Mdi-Au?~NbsvejWzHGuWbsdgGRQXvtzx2T-2zA-fd%LaJN7iIz1}| z6%t|JSjkX6^Y19yViNPcN*w}b)7{axK5j|8!ZiYOigAjjM@rg8UZxI_Uv0m&eNHPv z_J|K1b4TBfu8eHmx(YGby;^J)`V_#2lPcN+*W-RDw&cVzE}CQND;oS)zJH4Hg=@={ zV9NHxdi?rVpDn>=yzJzsc9Z?Wl*j(l+%%Ffai(Re*ID;=W>yE(K~raTGA_2*zA+9# zhx_zU{>WXx1O0=G8ve=nXOEm5;V(db0epA4OGt72O?(=hNUE2P{%c$1%fC-06%jav_Hj?TiWxIxlU*3rQ*R_B2g zhry3z{(5cj&?7FM#9f;=e~4zhDU-PL-Z(lcqHzEH|h zRVNf45ZHpx(^SveX)`iwKDMG$?*{%-dGe{@;fTk!y5+Xjbz#GC`ys=3zHW0bx@pyO zrD#ImzH58Z=+)x5vF+^TRQT@Ytn-oF-SJTpL-gt(CM#C>r9V+s_&Lfl^=@gcYoo5o zZNtUS{U!LBt@GxJ{PkwE908ro^Yr4_fyCpwvT5V1+lJR`AV#e3&1H?M$n(kdHr=$2 zty8gXAolTKjU!7-%&Pd)4ZtVbm(hR;gkz6OhO>@?w#Dt#`GIbo%un-aLpzbEbg(?! zruJK|MY-ubv}?e}f=|UF6=!NW^@_KMUP;0;*&uRrqc}Nip722_MH92FxcMj#B6}QkHh`QyAJ!^=qAaGTzg?0O}3q`!{h1Z zurG7iUY<^|uP#k$&waYhT{+AL->G`KeOl*Ycb0wUYhgxeI2BH}N$~-LH-6>^?qqlMXY5tz zuE3$+#(W6G;n2LjS&n#}fh)hse|$ao&Gi@WEO2P^{6cs=wy{t1f`}xj*qDO57tbnS zp$9GWH+wsrLpPQ?c4v_E1#3Inc{kTI>TAf#CX74AdMMssVsA9wJ`=0p^3M#+U(GMx zF^A$#!=Fw=8BQZH1P(*yiGs1s(tX*gjx@Au2&GBpZ_u8`-#)ho6us(W63*~vr{>IkVs-z_TOd&!Wv4}Vd- z#X}PKqVNEh)L-8#l!{3+s-q=`E1dlPYDgW=S0d4{gPb6hAdft+E=&>BEB}{Co~Ccu zF+m26J1RtzibtKRVH%M$Ac{I)PM~2{jz~JcGJOT92AorN>$THl`Pe4?gF! zy&yVy&%POSx?HB&`*bB_V>$-P0W)LX4qZP= z|B}lSLyDsRqExCYJo1N(V@ds^Ktcm4+Tsj*N&b4i$wHE%hmei1r6$`hm~uaFj4umH zW&a%=M=6{dyK~UT6E$Jk2of8%biuXY8$0P3qWPaghsY|CBkforXlU3u5~Q_50oL^>;*%SA(+B=*J%Svf{~ zt1B&KH(dOIbR%F)0TeneYJ;}J#T)eONzc~V&iw=0DV&zu zVFuQ+a&umIcJxguX&aC~=`WdFSLm0-kbGU<_mlq{x?LtKS7o%g*y|YRm7iZi|Gg=l zHMR~B$x_m>dZ3MSHohyDjCW>qj(u%c-mTuFB4z{%{zNSu?W1M$U#6Y~pxmvgz^D)++EBVO%;*Gz_L=$m;Y2m^m7tdv zIoE+@;YlJbqv*cfDq|hzG@-tjZRanYlzu38UkKg8(oYRkG_>%eac$BXen@5oU<~a* zld$KuxF_m?oxdLS8sM4qX$O!*wVIKE5aojlL2{p~@lpntZBM>V`B7+q)Myz-@=?Ey z1(CvV!u`6Jo`GIf`fhAf(`FUtpQ^+N2ZuU&d+n@I6#2XNZ%FUB9ud*|1a{||*e&k% zNWGaN&+iJZ{HrlU2oG^}qXV~t;E7Xxkv?@1pQ=cGL4rhLT-p1m=!%7S`~zs$Z7q!N zg8UqgSIqe^9h&UdpjEzsT$QB{T{XE2UG}+>Jp8~jMz)Pb9fPS9*IZ<;=c%>#$LY?t z(L-7r5`(q(q?#E(0gGm2lNL=KG}!V`yh$(|+$f3wJ$W}Or$=faD6Ml$9yp$MylmO) zY}==lijPs*`!K@OR@Ke1AcD)qd0q(9QBhcnmxQf^@=0iHQSC&8u3JEa@d|z2{)DFc zF-eX5>)K5yFqe^c5Fa^*tcpi-DnYv*62@(UTrn?&@qLfdBWF+{q9SIM z#D}Qqi0Odos2_+u(bGZGVLvc)p>oM`(T$-bSbDvKJP=*5Y%mQFT`(=E#wF6i*)F^edZD3bt6;`B1~5(hLb>$}K#9P8$&yY8u++{1PB}DcYJsDCUn%=z9c1bKTEDAzSTr1Vl$!BWe|H@RE}4 z^QZ&v#aw3BIvhBsS%YdxO$oVj4sen>=gFwj?qh_t?qdX;Ud5bdS1FukH!18mXIZ^! zX-#Pqfj^B37WMbdXqzD$RjQs;O4CS6SBqk=>l2Ru8fzq15^bij4BJYR{shfwHEuf{CB!IWOa2W0ybA z^HW+V*b6H9isirNs;%{@>a`OZC#X$bp7$Cjph1*>_gCi{!+VROlqZ$7GmdI{QmQ_w$8S4Gb!p4FPxigiBn~27U@UX%G=;D#z0mFzyut?%k zn3(j?;*M2t-egS-x2hEp&w-8uFTQCeepBBUH|CjcMUQbF1wRhFH0CXCB;Y<~5DoZB zX`DEOMLfHT*N_)QJoDr&Vth4m3&N)(ia2dG(PmJY#u7qucsN+OaGHO8;w(H*Nrg|lnc~lt&fyypFjOKdp?f8g=y%w#^zVxXMnZ4IN3(wlT z0!5cI+?i28Ikn||c2;hQUe;QCtU1SAjc_aR`tfRl z@!Rm!Y2Ch;cn)p22G>L4M)*T0#!)e$8S1(%87+A?j!oa=5II!b3-MEQyJS`0iUjOB z-N#|X=IFgRBI7)2iVotGO?{MM;vKY0`t_}pO=7}RI{sm zxKcFYQ1w<&h&!7$DD?QW&YYvgdEu3DRn@51rkk37{rfKS1?ZELyW`jSdi|Vp{>o;Q z!fi~d2dKct_kYJpEtp^66}%CCH;ReYvIuw80h{se@cG$?x}yqD-9jd4$zG*4)Y87AsR(7H1!fHt{q0$e3nr$lEOGYeMJOx&`;W;B?AQir>5poUU8!NrVm(M)0hD1Y`R$F=CZTX`Qyu_ zqVN#LngYxpq^5{SGOCKhGH|lyXr>0nJWi_NcrGYa_F*5d^yfZ0t z?tazMWymRlki#B9cj0*y@Xoz zpfF$ofioH^qPnh*pt{6 z1g!svJ^^@*h+k3gD|3sf5F@I0!xM{r{&`0*P3ZaTAei3*tc0iQ3vo0q(-!10u*Fr& z@hKAd8PEMxZn`{NR?cIIfYawHsYG5-_-8Y}>Q+q6IGxa2Oyvu;M%{cuRx)vAP5Ecw zS7AjGqYtlpV@AG@y%}Sv!l#deZoN&RbBPBIg~T~&TPvr6Zj*P*q;8_pXH;Br43~Ku zoBYh{!93rf@fOdN8iTR$D|&qgntSA;2UxBc(>Zx=hpDw`YrmEvWE~<{QftOnFeS)y z%TquT&n6{)diAixBc{LtYDQWo%co)gB9$>t>9ITDM0A;ul8Z--=Els*O;W3D(V~tR zULBoFO^)v7op|{>X#2R)hhTeq8NIN4y2n(z^eLR5QEG zdb-xz7|xv)$vVf|*hI0UK70}(C6ony*vwBo=^Og{mMfe|ZrIHXXFEGYox zqu9@YHp>{w(E|p8^dLKwkXn{pirDkLFDHMx4(Dte1T(4l`SQY6rbX$@iNjVnhVbO~C=JwhhT*7_-EXdbpo!;H z&Q0U?d={}p){jm3F!XVh*@Zby;H1m*wka9GQFO*nuy2Ywt_=yUY zthSM^$=dbbT;!3`CBMn@#(pN0)40*9MTQ7TtH2?A4Qg!h08)`>TN?=sjfaVqplvWGZW9}VzVAIVwh31*5zz_ zmoDsAO`Q60ktA){HzufSXL$O!VClzrLYmT6RddpMwaUQuN5~tz{L{%)V1IWaCoa|K z>c}uQQ^HVhdeSYAe1v^HJt2UC;r+b|<+^-?20hehRx&N8sb-57bBC3fP;i>dZ5C%% z8QRD3Y}42RRkEw-_tmKdF8egrclI|0jIRs5@->(y6yen8fz+?Olb5R=lzCM2);ew4 z3YyV{GRR)h2+`z3s>IUVx&`W^4atjpi{I?l8p&3G86d^7X>Cv#b^Q2qLbQysN5R(# z>~xdr@kB&9p+vcMRRuC8ObVo|SDO7Bb~Cj8Ek+@nQWguV1{I`gohYDb83%^$7s7MS zGwxd=48mBVDFU?5^+^L$(Zv$pe^d%}b6JDCO?GX9k!eOr;IIiZcql0)G^^<0WqEkC z^i6KG9O9SMLu4#Evvly~1P>;PRMi!W?q>cLsoMUGfyHJ&maNaKJ{PMz#1-m*W^+W4 zQ=H{{KXah_eMUJ(ir*adn<(QfC^|+-2@hJupZ{An=yj#xQEs&uPB;7Vmu|F&(`4%T zMVHN1Hl{#Vn4{`S#WXZBG3b-+C+xE0=+qP}n)7G@7ZQEUQNOs^eI0ea|^ZSQ9}SZ=0*;KKxHHdoaDPP3gsa?lzM+U^E0WLI`iXUIr{{7dEuXLqQ&IB?Ei?x$r%W zA?J`NltYF;QG51Ac%;ih9yHuE9<<=KxHmL*R)WaTc77y<|7$1*3vZx~^(piZuUps1Be zayHYZGhO}7F=uq`(X>_Hb`>wBEzDkAW4d4GP)SYP-B%*FPdSZR?cPWp+PsU}=b z^g|mbXWl(k534WJO(*Oq08C&>6C@|p)LD?cOzRats@|I4GJP~YG!~{fkq!HI!T0G* zONfiUdrFnV3Y^F+0YYRL8_A6hk|Wu98r=+k%iaT(HO*5Rki4! zP!WqX0X<+ta-CFIKPYk|Q$&t|PM#leI@>pI?sPY_x`%gk9iZe9L#ou`)y$+urTon> zW~S1GIgJ12eFvyNcGKq*29FcH&)mqGg84cEGrE`@^Ionay z+MvQ^F`#orV5VCns7__@t;$X`cgqmzfy(I2C!_!^zzfoM(k>dlIqN67l;i~Ce!Vkz zEP4W^waM$zFB8VIo{$623<51JRwKRFsxbT4^&oA%_fxpVm+EFax8K=v^jTs>hwIg? z8RDhA_8SKjD-dk@T@OPdGg`XK%ZiMN{rKtQv3^)kFqw|J=)@uid7v$TKs5rpxO=4# zoxB=KRz!xomo8`cZPkXPvfkXP0aDw-c_i>GMHJw?JuO5o4+m|qV$fpH-w+-%qyY?( z6EQAZihD!D2j=9wJZ#nz(8s&6kDK0?+v4VPl3)w>7fa0$bd`u~{0@5+9&UWk2QXoM zazj5rI4*G&v3DaaBQ()Sgrv0Wu#4+=P|JwU>SGhB8<>tk=X9%b+BLE=tb5z%k!TRV=qB2^5?h_y zoXQN8oRz~wD3*wwD^N1EvWj^OGwOwxDr; zuwdp3m?}ubLrz)a^9o8nf!gpo7GuHv5Uk%olc)MA&N|H?=(vB`z10TWcD>wZGfB4pE<2wX&ysD*C%8B0ctv_q{02CSmz^f3 zk@I@z=j!!{)S$ui?Z7zO1j~Ob~fXHoMwZDnCxy7@>NF!*?tsai;9U zeNdMBWAKCPK;C1oou&iCiy>&VLYL(3tOm4UIuO zSeCt8OJ;y8F;rEHAa@o5UK+Lb69_t-Gl_xZTX`@eh*#UX%m{N-6f<*Vl+z{YY>|2- zvj%g+kq$7|%d(w}Xe zW4vj`Z;_Hq%pO4M07`N9@(NOyo~yFAqpNP%;o*C-ir0SklDT$MUnayRo}oNeGESVD z);j12u^pDE8I7OL#Nu8mU_*lGTuO=Fjcy7MgPN}kJt_T`mCt-?v`@K=%pgdpna;=C zbAf9KDr(NppYp2FKabIFmzx36@s{5v-tY&ZwOz%%(nEFb<09mFaqvQ)+i{ts(*g&dQgJvOxk7#gr3AzMfK_MNVk zZxJ{aSozN@eY84rta_9Np51`Lyk*b6K`A(Kl6_RJ#cy+Xyp~UTpWJbc{rb6wO zuabNUzoWD1M@XX4%fPD-^Q*RSQZW4f?CvmN7OPI3n3O&}mAZhnB^_C|L2RzJjgvln zDUCl2Lo{>*e+iu1fF3t^HWy0wS`^5RYRjBogcdzqTRQl_A<$iB#IF0oA&<&j2@$<&Lh%{^;iGccjIsaG!^)bDAvUA zk6mTH&5&LH`(7es_gS{hVVsLRVL#HePUB9PPQT4p6I9^MnF4>pjE&u-F8fXnkVm^d z?&Wp4*<%i5Rn7d@l$hkB#^ciJMgK^9+=!aYYa3~d|7el;a@;mUbK#(5?v3>HH#O?(%)(l!qijc)n7Kc+v{!BRO%25aH8 zLDd5&AEX7g$Q9cCKNH&IU*4{Kr+F2VCv@r^4m;;29QoSrs;wg~3okPWd}_=gNnV=z z%o1p8_;WDc`h(|VwiONmROA^{^DaZA_^_W0nW5GIwDd^ypHq6D>}^4UUp>!%3g$RC zHZ25ZrmceV<6T3y^1Ml@F?DICuiua;MTK_6XGu`VTbs@|H=Rh^^fZzl)}r6L&}MTy zrjeV|reG)&@jmv>RFG|;RN!ObOUh!D2EoICjNwQrVj#oD`e;c{%FB{93q>Zx%>2=t7>kJ;17%KvI0z39gtUS!NWfLI?S|3*Z8JTZ{-*}(C>o+Rv(mJm zjJsh5dJ1DXI!hbXiljDwY?yVG#)4W1Sm{%(;FNGPt%Bau%$%kqC2&D6y;!sq4l8|ozh*h{0sx}; zla{B19}=CLKzOXB{Gnak(X(|vRev%agFKMIr_~)kwB%ybnjtoB<4%8OW#i~xj>cTRKCUZ`y+Yr~%fPX^z7e2YnpRL>56(qxCO}3L=B5D+ z_vWvoA6*qvn;sm1gKqpP9UQgGoQrbJS!9&3V6*9CM>I@L3 zW-*RA`i7>IhZIUQ5$y98BUlb1juUUr=|V`eqZ8tj#P(tHCtIyBxtkIYB zh4yyW%H-rQE689*^P7$MX!G0mSuQ?v<4vYLmtAI-Nv1cpD@_$XFU1rR4}@89P$8J= zx?ihd_jOI&9u+~#cGcaY{A~}!bXAd6;JBLO&p6MKcAa|M1cqw*;g__6MbEoVe@#Q`=$TO{jCizAEXTozm0 z&>C%Nx79oE7t(JOTm9elJE$e-ciPVS++Kg&jX~S-;ZLbh+*G^~xG|Ka+)nGy#A2@* zODy6EMuWNi;aG{&3btpx!8hA45~OaJ2SvBoX@Jw~2#}rvQ{Q!mI<((DZF0^I%{|n= zNF)hNcY@%hDjY`RNX&SZ9wJ{~gHictkE|KnknMX;zU45gs5}6S&~9R%SwDgKOYDS5Gv@izZcm%UME#o|GBongQ7z~c3>+;ZPfj7TaWQ2A(uqgVNwH zIDyWM@E#@tm*;}-u2&Y8@G-2g8tukY?aDzFO}7@8dwi*o>U#Op`sqo}Lex3ODOIcJ zd$N6Lnl7$JlNu6(_UxZAJ)HE9{faub((x3CV55$qZgyWjuLOZB5!-_$WQF1hYrk`E z%t_zr`!yxvZof>O4h_=aE;B(3PID~lb%PDE6X3R^q3HvII%>K9f3op^@#_B=(*F%s zeo@vhdHP?m8+8>&t$G;{+fS(o8X%o7Wg@G>k%8uK>|9Qz0U8n*$G7i8|{Om zq}$H24(NB=ErMCLg3GrH{z$sx%9d0QjVx6`k0ffN36p*F!zk%`eXOU&Wwo4f$nB04 zupm&%L(T`|LuNtF1Dh67?_7}wf1S2@-)ms3PBQe%3+wb&+GaE2wW_&!#Sj@ux&MD0?Z0IC{}8+X zcC!EFcE1SpS2m^pj_Lm)?EhbhHaqLTSv3o0=-mBz@y^w(gE ze}V2VMbGfX=-Ix^IRgVT=l>_P1a*nT7^riCuJ1=KvWToTyg8u)TGXG-z zbgZll9E7yYOkalniH&mR8m%vo62|B~+O(* zk)7@Tx*?u*c={;MKd`qyILICBk&e%?*(MZA=9$`zDwMLof+cy{#Y?4qX`yfCR2W!a zlmSxiM}Gc~U?vSjFeD1pZs1>n0$}|Wkm^WDjl&HOl97MzA79rGJvR^Eci-cBA1iqg ztZRNhtQ-~sLjqyuf?%>_aCq+Z5iH8TfajLs2HPF<615*I)N_ppO3smJ2`AFg}nWFQZ`Enw<%dfg4>HGXv?*Pelikc)|* z(Ka_@_AcQAa9v+{o09OP7J%+vp>o#)R>@s%_s-E_PkiFv=MAtQ@H;qf^#kS z#kOLHIdABhkN&5vSQh}{qOvs;YLPud9`E-|!iq?x1lo?)?GX1P5JWZ)6o~!gG)`Sb zP9+a41u0MePn8ce2O_)W%od#`(62a7hVV|1N0?d`$XizitkKTE>j#OZs8)?m{+)06 zh>Sq-2@dS-81LltHe^BQ<}RWc;yj!;DuNB%~QSp?ibs^cqAoN;H9n1Bp#!?8DE;q{rB! zI>2b3e=RL?q$t_kn)TV76=2LvgjZ{jLs>%f;ojhnkzMv@_xcnvpuFag%pheU--+3n zqaR=xcF6pk>~nyQ5f-x&d_sad;-buTTM7VBP52x5Y=l1=08(%FIk%rsj9=XQ2AA%o z407xVK>&nq*ku=yb-Gb$i(BN$5NHyEL`p~<$9$NRZNP$lv?+3+C<4Q%K3p-)0kJ>u zF)27EI%mv0l=_g?Rb?XPuS*Qc+JLGUFZDdBCXBz`v~lhbvlQ_Q8goLkcP2uuAExq#Y26^H7fJ!n7)(?r;V(;S3`Aj*@1kE%IU9hUy^yKIIr=b9@Qq@y5`+^NO0rf;3VLW|xWlT6Zy$vT` z1FmMO`Lxrq`g#z%$X8KpA)SzE{dG2#_S#3k;?lgzM;y;H@N!8p%Z4kX^+La%S9B3OgTm6&CEpAD8F-;rJ{+y(KgP33>m zcZJfSOyQCm7wK|L{vMhPx`MrNLFSiWei9^>v^M6K5a5ejn1PtT>YBC|bnCzUhStn< z>CgWD`Ry(afNp!Rvcz8$REIfnsF;V6P9Vt~T$63&g_aKzUo}oN3fPtSUCd9(bYU9~ z#RF?w$N<=BbMVErI8?-7=;*p*Ju`vXGW26`OH~m7>C}ipD?FacHgT7*;zv?e{QtcSj?#P z*^KG)8<3s=SB)NDP5i9m;!w%8R!e(wxi`$pqZe<#BaaThzac?LGzy#25~rd0$*8_` zO{u=5z91*Mn+E}`)RC(g2>UK(m*f%OilNSBKwat@=R-m}XJhEHk@cPqtbatD5&d@i zWoLUya$kPLJHYT=4_h8W_?_?je)m%_uXfc+hW?hLD*;|^j68+?k7FgcYVH*Q*LG?Y zuJE@$$JpKdp<4<=z6p}FaY5fcmp+4T6jxfHH6wjQZ({90A9U|&kksLr{4T_q=yoecupb2?0-1*tLNpW>ArZr@V@@BkEuJWhYCQ4WFF zeYJ8P$X@Ako$%J(DS9D2i_jo&(?*D7r1L7J)8OvptwF0K3@hI|(O@4aJ%fyT(3jGW zzSvr!69iDjFg45X21#oKt`or4wb2;0dj5>)l_~ibi_%-@b2Tj%83*~}z^We*{Z>m_ znUm`li%69qo&fYUC9oTZOs+>c9y{Yh$aljU0g0jT_HXC4R}{$tWRn3#p13rB0AR=~ zl4tJt#k|2UtFW$*hLxAHOSXSH2?ksgnHUM#-6y@|Y@SI$rGepn^=w!hL_7h8+Vx-L1CI$|5 zi(Sb0_Xe86pjNE)fJ__lrriK*|8?Y*>TZ9X9=`cI=BJs@36E%-hWxK>dEjhKqd~-O zj_8MC7z0`(GF}ioP50)oN(59-srf_ zh$Y0>g3ASv7-BQSRePd!(g`pS04M+zbv9d+M%YIq$DK?>-#^8VtWc$ys}t<|x1kV? ztUtf^<0onBx{}|(_b5P29qWGZ;DK8fZ@-utdT}*+%HqBb?`s>JeX?YKLYeThD0^ax zKaFrM)Ica5`u{LiSIEhY=-B?8e)^FAamtOSSbXzn?m%iXBm*x9J_9TPQCsHF1T=tS z+z!aj`6l1>ec_L>)yw{UR7pYQRy?v@Gu!*h)9=$4{{CZfk8P>B(QZ>u4^({QEl~}8 zM_Zj&U>;J!t{~6LH}A}@JAMO1lLm@;#(?j}i+%gq&E~`wP*-NJm5Hj>1#uG8s%8|y}!)WjQ&0i+J(7|MGA7LdBRL5I*$VQ z@^;W{LU}tTjXlPUzV0{Il9kj@-KNOB72&-0>)k%uetbfea(g3P5^cP9t*6v7^@vgS z$l(g}7ec-d1s&zAfiMG~OeN2;iMEHkl@iR*Gr+lDS_lV2^{UC>gxq2M`0Cr?!J0)F zKq}xtQzL&HF_BqKFRhRTEVAo$9r*r`rc_K}>pNX+55$bx_9w#sbEPh^Dbxx(`iXMN z8|>oF(`5%aP(tKp@FxCqGlsJJ%BT>&Fd9)35@$jGJK{ICukhO-z>oJ6!s}eK!0-K&!mFP-$l4+k!NcqY{JnE1$vo%t;-|8_BX5Cv z;A8B3p1dWhAK=O~6|8nqd7B&NQ}t=0$6vG+`dCL(X>e9Q^R~ZP zU|mVYLnQ=Am*A(Ngg%JQq7qL5TC89mE008RqUfGteWb_`BXU8}9q$Y9dnD38Y7E#? zP-VindBxJ;+K~56w-2QMPyX(l?a5o&H&?&ZPeR+ojpPq1-MDC{^0TDXA>Td68xPNf zw_F}=J@T4l{bcqbf?iAFu)j)Mx99Ym0SdU(@ASd99_a@k`Ux8w$zQxGSqm zFH$UQE7{Odf_e~^jBnUWcwy`|$h0GZ_6`5EZkLFa7e=wN!4G@7{@mHCRyxp=^IC>A=9 zP4xY6$Wu@Ft1#B8c<1f(7}hLn@VPZ@WzB}#;>L%CR=1%R=G`46}gMq5VlsCe%QzK=bYqZhG^W}(^3#{Js&Pw z%WZ_zEh1W|%d2JkY@L=-3}iPS?mgJ!Wa)SqjYkv>rr`TuJn{V4rH^c-&>5t_6Cd?y zU%tM&Zr@ECY7pW?^#9#Nk z8!uQIopz7uba3gN-YLL~Y2;7$WZ5nKwD20K{v}PNPi>!H@>qNzI?kM4!YXf>Jx8-h zGwYM9S)*B`8SuJ@;KU6jUt-fdr8{$T@`!rZ#!(%aV?Oc1fdl+G9(FGvef!7>gEW~h zJ{R=3y)O}_n>K1Mx`saJf4|-F%_B#oa?qbW#@Z^>BWQGc?6a}(qigKgP4!pz?G@sf zQB#l76TDfkP%6aapV3-u-C!w`QyCA~iQX;)U>;y+Lw4LC=PR0akggtj2jJ`=*DJnu zm{UVc*sy*EkcS=#4+Op+vD?B=kPqFD4mds_uiHTF&;?uO9^d%FvanJ_SYf=&4ZJ7D zN3ir2oR4&SpZFVTP`mgB1wmZLp70IQ%2=%A50eB!sQC9ipuYz4@R8&cgvE(s$A$iD zY>X2kd^Z#atoRlsZW#R*wNp$c#b-~)3C0a|ivV51<}ksYyi>$p9JZ45iNr6K9$#3> zykATsWg@2Kd0OQKUNfoKrSa*dX z=EM(``Ns*zCF$Vec+w?dri+WPPq@Tvt8oe9)^Ru}PF>}u91H?bRt8eit<(b-tR)27Dx4J$+{7FnN zg!|U4GsJ$sW}B}e+J4a23&KCBYuNop_l>qQqig)}%KfSBjrW7^^T*{r!Au0F{sQmTp8}|nD(bE!xJw>@@ijtoIhd z75bQ<_FUqToluHGr$5+k#0OwSso|!l89-n5rxr;8HW0gERyiT93^#Heor}EIDg7?tFWr4X~MS z&6Qp4!C65IS#>{JDrb7wuv{fP64q0@O(M3z zW_6cZw`)h>KNuX5NG{*vd;(~@=setbz3|`aK1O5yV7$%y8d5pt+w8RQ1_pyc>$f*O zh_7FN7hyV@<8yLso~>KwH19lFpAab`|HLl!P)y&fiJ-&KTwX(%km)vg75YIHi>wwuj^Z%>*jUqrN_E^`vte& zBVNsG%%bn$Y3J@&GXE`J@A+z2)XKWTQlqTksT%Fgty2cO^-7^*LFQQ~T)j~Dm_X>} zp{)FEVKHf&u3Q-Ju$d~2S>kl=q9tp69~RkEDaTaH8U|F7xCf=fZgjLL4+APns9wAR z{zdB?{^w;wr(9WSwIbeNOthu&ty)ay0 zQoivGR?ThLdX=HFm=CXYfi+9Kt%>wPBfkCd6SHFK=qc_!2W;4Pc%XRVCpheZ(O#h7 zaF~(JASmp}>fHI@r4`NNV$fPB@ZS@Vk5q6a;a!kP!fOq#r;y_HI;11Hi(cM~qH#ne z2DbwYksw{dHJs*%rxef0iQRu%%-Q)E_t_X^+Zz|?Cf!z`VX9>t<04@)x;RJ%!_lJo zYB<+!${{Uwl&txS);VeqCUB}B6xWX!J!0mBtEo*~i>f`)wJM^;P@&F5zUUY9SLf)Jo^1Mx?6x_>qA26BqeZe)VU#K>n=8)sB%)RKg12dwi-; zQtDYXq97_-QeUni-HJsm0+$MNyRiWXo91}4k%jpVs~@u10Tk;7u-c|=B)|PErUoUU zKP)5q-3;k^?wr;j@{`g51zMz!eJzZH3Ti7;D{6IUL4mWK6}?xUs#_}_shQZAxHV=! zNeU;P?nJzh@FhH{E6hk4d6j-)8UZ^2vgvuRV#Go$&&2Q))S!XE{ zfX?1o`K03=1}EA*XG%p1ghB#)MG2p{2u|zUWIjh?qE zospYw_H7Ss-_a-SeL3loB1v6o49i%M^2=n#Me}fdT57|8VUqBKf10S4imO-yD3h$9 zuU%s(_t^(7i@dr%`72fMxaqCR_G+vm48>OVbnxuGRo}|Dv8b`&Q^PD4{er$z->dKE zmG*PDQwb{U4T?)o?Xd{1&jZBK=8oN(r6&)}M$xT>)K8-c$61G_%~S zXwKr2l#~V9IgzxVBwS&P)(gT%rqB|7QQSsNfu6DmBj;-K+CC?7_*^tpFCOc z;*f^D&Qi-F*Y+4QE63g+^*aK?I-5Y=ZiYy%0lUA9HdULbbmLcBMF)!RBPb*^%C+1P zek~%1p2jdPqM!mfg;LL0BNg+nHY1AN9~VF*N6B8sZJ1?Dn$D}ImOryWaXgME?`(L& zve!)-52`Kvnculu}KnA7i&^buKvLXvn90qz9Q+>~V>)kC;ER=JM{?|2=E zlpm)bRbP;N)r$yb+*MwHXIu)skK0s_^(^fF`2695RsyqB9!ojW&S0|L(D0^Sk2t&D z>7dGooJL{%RtHHmo5fOK6}%W4-45^l`v+OY7DQ4f~>W; z->a5ca+B($W8dP2(4`RFdWcRdOp6PKgI7xPmOS0SgNmvfo735_gIuqTeZQMhU69Hh z5!z&YWPIU{pb}${3Ti^6wxlS0*;QPmixLt!T211@P$V?+hL0+M@QfVITe9Cvi7`Ya zSVA;T#Uufmz!I&x_@g}IMn$PILH0aEkEkP0w;F8lNId36vGTq2f&03X07=={kY;Z* z979eHgT+|cO_67#Q1&ADfqkZ_ZfCU_*1i|&+aQ=1PXCdu&7Q5zpTNhzPCs2ArwA;O z@i8p!6Y;%Wx3{tgk`bg4*AA|bT9`AuprMdaIL=%Et1N75N%(Zr^~j%5`S)#>{(?P9 z_Igo#BS{M0S>s~>8b?3XrrEsKHK6g{eU_jrLts%)9Aic_E)=GYLl8stJ?)^1e6(Ds z#(R}}2qkVp@k9bRSOU?~IyHYl`-eES1)s-}R$+^3Hn7IH5}MWCy4+a)g4$(Hw(egbY-{Ta;SuJzTZb|^46Cx9ahs=sn;e+ zkAi}YUIbSz*D<%HSR}hxO>*5DOFQQ%*n1sFtEYQd2XOM;Y)xKKJWXCOyG2^zT-qJu z4v?JJ25j(+dE)0Ran)=~U8<~Dcy@)F>6^}#PT|8TP;!R1?=Bm zRch4MEy-4b)-^F%u!rFgf7~>YM10JD0IByvqq$#KYAmtl7dbGCBxtCw{I-XfVXe33 zaKSr>(mBH^CHGnrC=!4`u5%5ke&-U=I&`U8_zFp&weJ#b;kB+*A0HP|l!~3_Tmg0A zH&05QU{(aJ?uvtGl=YiXbqc%z{a13}DreP@u8Fld?a|t?z1@C<_k=#dqv|$bXr|g% znz*JgNzCbM^o!aG>rLvFY18)evyWqmt;a7hC9V4C-y~hAk4VRrx+OC+)$=AlwBUXA z^flng4)m=!ZMmLAIIk>cclWb@rob5a(xgy9CeI!dmL|iTxL9$@iL^v3GevRRj*Y2| zHv2G$(lYn&x7Ckr5GF%`&ofOxFI6NjpH?E85}(fz<0GI;K9-6~Kzn@6C*-4JU8%~ejb~fgk|E^m)Ra)b?@LR;8Ws9P{xjBu&v;g zVIJJt;_9Vy5C)Hg!4c3NO~uF0{27vq-wu%LjJAdNlzlLWUuT=LB!Q*7P_K7%@GvPU zZ`^L|56H`sVs|}G+B_-m#P3{F_p9l$tW2?>(Z8E7n%^{OLSEJt(XaZ~N4H0Ju2UfC zT8|9p_f#J0x?GZ7J~;1)wP$%3AV@iFNe*qJ(eE*+Y_;XdC2id|5LUbfqb1)m4U}^GVMm+ zm!Sr8(l)Dj4Qh%tVZF)vtH*0C8XK-SIf_bjWv&DZw*Sa(042k~2q%7MX+My-N}=%B zB_i;%^FtX0P%sq;{qjPNEJxwq^GgX`@^U3MpXw;(s=TS7jkGb0s)(bTiZo27%w8Ce zt*@8xKs%PZgj@)UZ!InNT1&%JikF+tzZO3IyCpX%iBe$-VOrQ9IK)?cN3p4+)FrHV zddhbS^lCXE1Gq{GV*}o^;PpM8a0>5L17^tB5)>Y|$KG8A2=_X{)EkDoZt0%8K}jbY zW|k1u=Gdx0Ukx`x9>~>h$+bEH1FkWn9K$M2gFsjdgbo7`8p3rWWkZI{E#^ElmmqEf z;ZIS~ zyP2=h8MkG8O`xA4I7K>(@Qx8*P~Ss&C-e_;tU`Ax?L)3>ffjl$`^2;oH}=q((%0k< z;>3rX>SJ*U%`LoFVq*#pmn_&KI=Z>H=2O0U8j6-@c%Eedq^%S$l6Q|9b~bExVxFl1 z1^OV9J7bOAejGm+p`Jn|`UJ*2B1$^nhkZSGCD7vz=gg-v>QX@WaBe^^1iIp}BXOvc zEXzGcBGrG$iG|%bzObyvE&{)xyeGD3fR7r|Y7kMIqBy8HFB!Er*4B7JZ%!Hy9qL$? z0fe+js*>EI9xc=Tw~{{;&u?9qT?Y#gvv72=t0L`UeBt1Agx6@UAXUQ)2qG6XbS{f1 z9dWjpjYKwz;%sJOT!bgS{*s1Co!Fgjh;B$d1@T@hOdy<&VVvLxZSwJ5$7C&FRstUS zPSv;)b>FTxGjdUhEc!I(_ z;4gc^uFBptKX^a&uxWfLvy$(>dO;g?Xg^`HQr*(^(l^q-BFT=cjUTQ!U8ABk{;^i? zJRzi5hE*GD{3Db;n0VcH!#Hw_h}qq${7a=ZbgT#=Zk$EARohN_H38oNKlUMZ*8LdT zE56ZmT~d7pV<56i1&v^NqF>(+y*QUlU+&+iFePo&C%a-GP06#fPnub0{qc^URs z7tKfZ7b_rN-X>BS$o@4s-OGUX;LM9_e(G@ddR$S-%};+%RG&B}S&tqH(;;ehE>t}VqM(@UEqV^(? z=}X)-oIAHuGH=yC!L^p5J~muQq3iJSjb9bq3H9H(|Lzj%Hgl>l-1VbV0X2hwJA7sx zi1y40GxXz_Dsz8wsW;jNV^7q3D!OjMhIr!f+g(ft>Mb(w&zN<1UPFlX+<1dha*FBh z^G}>7s6pljXrG{pDIi86-nh>}gYoL_g)YKvPVomIixnUXX2eq<7rtM)mPhoMNb~;P z;fKbv_&~QvKrl-C@I1tZX@!r#xIu^npd8e@4x2D>D_hV!zy8#5@JQiW=IpenfnTwL zT%~}TzN5EO76I+BFo&>M34)D`VETD{9vz5D5m+Z=QHKR-{K;AAH-7?M5}`k;7h%1%-X zDzp3Bq;XGF;9hY2qJ)~%(j}Cuf*C6OKy;B7%mXQX|qy~ zHGwFn#xgceyo~uwt7@+Ie?{hKSdvF_6C9BU*Sh`U(Mc56S5B0{nHo2akAmE0Yl*JH zgkgIk-j{^_h_{pCCsy#k)aZQQqf6OO+KT;+L01IwxW0nM6{C<$)fkBU`~yP0RE6HK zC^LR)O%<4&&9C5y70Vcd$Hj zzDTP?b#x)~%9-`>O$o=+dYL?GwWQxche1*o{yLH*I-=@@9I9gddM6ZOD$uMU8Qr*4 zZ@DQ)B#bGB9F;i?nn_bLwRuu&L!TA3#Rg4{Xm&51b-h-j&2Lxznt&EC+T~8)#DkDl z3WN@@S6lh=V+}5R^=3s5PvWd03-=kkHdr9pcyglv6F$cP?vgO4a+zoNbT|Ihb}X45 z$;K=0mD$ijuTk_mx7BMl2=PfmOgCl5rJg55b|C$TXX+NNktkEsp^}DcLEFtH`BCnsFe*tir7ap{Dj9J=NHO+vd~Gqc;c@3U3Cd~^Yo z(~2!|4MN1uxSqSc_7P#4T!jc zvmfn$Oau>!MsqyLBU z?UCD=fp^I5tXVKwn??_W8&=uOZ3#WIh;frE4EU^B3?jF>NX`_G%2XGl^m0vM`$Iq&)~$M?q%0W4f5hWT@{KGeULj&tI;|n`K4AF83UB zI*Q^LO+3uZ4CEZi<~`u`K9F#X zM4nwL~Z8R+AaAdldXa-a1F`8FMZRXkX&iVa*3jKGr!VW zqJ~HQskOn`n@C~XHJ5f4?v@`kG-K;dn;PcSfLe3`u~&mOLx^J*?CrS0$w26!Z4h}u zl*L1CTF#=#-p`<%ZW-5*$#pO>BIjOw>>}BCclOmi*qxXhqjOVSqAhp|Kl~m@WGqzx za3MfqQcs?Bt1YK02kTgRKMQFn2SckxW1&?!frd%ff^p~(L4!~NH}zEWub!m>=&X&i z2)wmaeO)2(1khkpD+i5M;=9{LEP#Vvs;21l-xdb_+P4L<@i=CH&KB!Ke;yoOsew)G z6OK~ZF0$K#)1vdDRk4j1BfW)~NJX5{j0QfnL)&%x^r>32qJbqZokPAy77Q=RpTm9WvU**eQo=9c2^Qc(9|)jz_PQQMN1wBFr-Xj`@pb1tBg89&*EbX&^>08270 zd##zb&jwvF@Iych@NhwqqvhlB!E(0A=(xPsLfpJd^nETT)LC5qQW7d1tdMFDh!Wl^ z+Ju!tuEvk6xk48bY1u(Rif1vq_nE{ zo$9&!Wle)NO4*Oo&Ky>UphFKd>`nd-!l&tQmWuTzSZ1egjYs9V5xU*EkYC#{_PryqY$K^!tv&kS zr57!rM;O=LtoOV|p!a+9c596bJXFBwYtn{wkX|B^KGv!d1N|8lapHOmK4sOH?un}1 zSxnHjT=JM!%GclL8lj1^x`C`ux&6IV+I3rH;x|a_YsmcN6~z_qWTkKN{v5sug5>I= zc+`TZ>}kKm!j^>DwuH+RYi3o9NqnddJ}ms6CBZ!wr7WY)h3=-?6>CS1kTAhz^Vv%F zVr`9;{H%m7{j_z)y=#*p%&sCtV;=ms?NJBEa^88t(T1U_6;dPPw6v|J77N5$cjC60 z@v$7bQ%ERNFy`rlU5)w?s)9u7dI>eHp%RLhy{RkhlcL5prO^B-ys#hX8!Ab<-LS0R zT?mNS+)<2|#N<}t^reWb5Xqt~H)-LMbE7JD=@vV-((9b2iNB&a%bk*RQK>6hp}|aLRpCfD zJ=Jt9M1U?U+-IfR+!u#u9$GPubEUJ6i(4<2p5$;>U~$oc2Ad!~^O4<}%I@KwzmJc` z&JK+400etE;J)p$(DKTyQ+I^&OjvSZoT*M{;~SeeinHeFd3w~af?Xn`gHz>1R(vH{emg;3UQ8CVH-7pih}({Xq?}+ zCDwmL($yvTcu-gnPNTUx=xs9X2t&_ZfV_CX$MWWn{En_*w;NLE6vl>r5okg-(#fWJj2M$W7?=1OxPBIH?a)N z4S-!I(TXB~ktdZ6WN_%xY9sMWQ9>b}G9^gbSsuvP35Fsb-yi*en?||7uheq0z z5A+(#3u^=p1r7~-I9D{N?0X85H93vB4ym(Bbh^uSy;L5FSTMyvMjw_)9|kMCb#E37 zCR^=q=K!`$9E1aO3#J5Bb$@FnaOYjYc7y3EckYh`mO=#39DC&$nwza+?71El;<;|c|NvXCb_l=Cz z&TXi{Zk(=RC8x0hwoT|PpNWp!-JD%cC*G&etv4tv^>N9{9^bBp2B+zd^q8`F8n|D2 z>QJVWv^OY%HhzBKugl*^o{$@P(0F0*WJA{U-m?ebfQM@T1jIDVR!gbB(vHx&*)bFs zKnITEZIaj$*+$ZbY;~bFQ5ljX)VN4cd|k`EHJh2LyJ=Sfy2nQxid~zL+oOJ9w*QB* zdybAQ2p2^^Gnv@Q#Gcr;lZiF4ZL`y{F|lpiwr$(C-O0SnIp?nP&bxQr^;ZAWYwxPs z)w{ZTtG@5|Z1%K9K>N=5D(FPlVmnl4yphWP!&7b5rZHlj>*K*pdxJ~#P>z6SPQN!M zu^(NMe=n6I`)f@g0UnA5a_-2b3WCM8UDfj8xsJQQWFmH(gOGKBkZRg*_L8)-!rv+L z{J&9B2{(o=6AtTMW>H*XD}ox%32~h=gXSvdE`LPDgRLlNh9pTuwI&521i4J7;Ee*& z18f|&_9-9HAzOnW%%sZS@%(xGXCYi5Um;AwV{9D5VK*=|j@2=_PWjL!owL1E3edC3 zJn%b-8hn^c!3xch9Jp_wLd5?32nmG}PdPb37i~y6!Pb_fI5A1x9jhEJT=_sMO(1I7 zS=gQCy-Y3bG&e8=qWQ4Oy$HMzUY7M5>3<`WLD-!P3y!C5G$3M1US%SpRw3f35K;#< zamK-BdLF(2H^VNMk%$Hepk7+?F56i5mU5SIBfW$bq9X7;#)eHt9S(o21IOr?o`?0B z4Bj+nFe0r&GS#N*&&?Hni`++vbp0{dIUZ8CjeU+4lI6-e*hBGm+Ji*%U zkm24OvtFpNWm{3Wjmg}Kk4dmayQwetAQhyA4t>$Jk%S$%t7(tT{>je+4kmavk(uV{ z3GdIl#~TL#rd`#H&rUV)=)0{bSaTYFZ006Mh-NJqG-=SPgvpyq6@2_C(rHvji<5Ua zbCOk+bZ8v||A14@-LMH8D3?J@$x0maNI_+$W>Pl^;wD=0ZM6Jw_-jWQE7uaZMQi0L z)Xj($rP)mQUDY(QMIzGarAKHN%s>_4tU+JOSH#?~p#ga@SXv>X9Ve5CEGCz0p z_U@g?<7RQr6oe_+&xEFqm9N%LS5 zh8&T_{SJxMD41(s?{6Vzrv70OvsG)1U(0X*=(wycd#Ru*Orc?RZCOZk#m}&l(Afde z6tTJKK3)vz?FxF~HFTycsns zHPYz%n5`0mQQ65P2U&1S!<-2xqUhwl3yfua{swmkGpE)pxbM;$=4j|8O=+-+e_OJ8 z@}9>49+7D8I76mZ#w5lvP&3eE4_Qh~(dAWx0T0FqJILFq`|%-tQY-mkH`cNngukW4 zGg)go8oE7KE`LB;QPXen{?xd88Jdwo1p?LocIQ;gKCVXSidbgpic9lXXErEvH^Hq{ z@I9e+;g_e7l_RDQT+Sm2Ee174w|&vjfVHCCU>C$VlWC;8M8Pzpt$}x<9;+XtiMMy6 zHipmqaSxFvZ_1=eBkCSILFT1D9?{rt7x&#JD_b;8dHR|V(v8b!2-XRB9^j)nYcYps z)7H1shy`eM3WoP*E1NO6p;T$hRkP;GTfB&-Q2}ouuEm}~h=d($humHCX z9ZSfqUy9k>1|#0Y`@AV2uS#hcQny-plPwzPXd9mt3y{<^mdK)BL3Z9`v%~hhz*3ri0%y`)X!N{oj zpd|?Mb?KdAH5?@~RD&@j(pnxd_KHo)=PGoSU?Y)mJx&jWG!Njx>A31ZO`)uK=zo}! zbBExG=yKeu=(t_Y!p<1Z7mjJj-z7 zA@l$0g$s=}Voa0%B`>X_sSx}1NK^*GM0ytIPx&-S1|Q`JBv-rxULM(J4(oGK;<+=Y zaQRdtCaj2T>r+C6_@7yfR{4V#O#7@^UNv4xnI~lu4~axh-~}$bB?xB+6Z!>7wYvH+ z=rCuV(N@6o$D};FgZpMeqq1>Nt{1D76(+diU1jG4kBOK{6jG#bD!=t=piU^~FI!^! z3rywK--#dEa#}C20;N&(>#bkb4| z`l(RzaN6&t8vCIPms8DX`^d2v{7U*1L=9Z4R2HDj<2Q&@Ac=a`!op1Y-HeEix1*R7 zV?}|j=x1+ES)<2G&vf3%Sy^mmsClWpsNfDohJH6$OlSsoq#q!aiaQ{fSv3b>zbMmz zEAMs4zKtKd>Ani(Sk%$;A4v5Q1G`g?lEu(*`Ec>_PCE9d`lNoXsn0MP)vY>vE%z1R zMwBz8g7uS0q2yFKpwD-6Z7A*z^aOU9Cp~L!a~20?Sw+fndL+~QiB$h-O8XV%jf)GS zDVV@lP;yy3wEqoX{19)gjNbip>w#fivK@4Aoa_w}PA9dA%-F%-M8FoHjz!lgWER z%i-y=PCGlGRE?cYn#X78cEe(C^xImo2xz7{ra4Pbvh3mRK@RfEcps{19;895rkWhK zIlMbRMF*i`&jw`{9{#-hc?aV-PE)~P%aFy)T({DvwQ!$MJ=f8Hd1IE%A(eHe9~2~< zkb`t@sxqh9E2)AhBsq*>fxe0qD>MPC)FOqr&rZFpETD=W{Y~{V9T{n+GJ}~zMe0KY{3tmikqI5~15{-n&=AR_D z4y;TbImoGBR|Zw8SCG)LPaB$q*#6QPG9uvOUI zR?6>0kmrnAVHkg0MF^8>?&#moF|KcbIK8b-|9)o_R-b?a}8 z`I$~Kmvu&c$7PcUFUCa%3NHrlJ)C-L-QRXk0(=+M_JuPv?!L*~`J(PlJB*AN7hb$L zuGUww2S5UOLR)68Tn)KQS;7LPXa%J@e`2J*VL{1YNFmHz*dT3d#Vf67-rVN}FRk%m zI8h>W;t9G5k&v3LMIqk>b#O>Erb@2_N7P#8fPui$^dY0!I=;=d+8@%n^qA1RVU~TD z8FIm>9kM85$F3k3RuSw{^UTyPu?EVx$z5xQT~m|B9KXFrPv_gfy126}d`;L?s$<-w zj8b4wh!hm1+MRVTc<1?7Sd*IG2=gNyUX*vwTSUYv{=_W+)y^)qevS^Cz*cLZ^46I2YNAgOx+V5I8Z0c+?Mt|-ofoTh9s?o31^5y!LEe*Y+ z6~I*{pv9x5teF8eoy(G9AIr$eO9zVI!aGf9{Oe)+b^K`y42V=isN|hCKp{_M2rI}y z^xQiH-ICW~>fT>*$cwMe!+cH#&Zm^KY*U@^a`u|IqlPLw-^fg*2NX6&J$61^ z#zZ|5PPOK;73LXnMw<@x5w~yKPk)q4Q_VfFha2tXL$}62RD6SxH%q*zb{g?w6q#FJ zBx`9Y>Zov+beFST+urBP{0!%%6y^nv0lDdB!)(CQ`wbjg+&BZvCe!i942%ne&;pCx z?y-^liio9Re>ZFC1FjZ~jgQ2n`{Q@f5R3hDJOum9dzsuqmQsq0EP7K`UR0}Ta>6(_ z&SW2clO619rN`&Fy#b1p1p6+m74&ztU3&$SSz2_KcM+S;vAf*96MjTnV60TE;k>tB z1Rrx6@H4laV>cm6eI>V5#3?22gWk-+B;<#|(e!mo+Nk=AiX zddZ*1N;LB-Z$mGEr}dsVlKgOMEF~+o@B?!&X?uZw`yh zWyZ~Jjzjzpm*Y6%1iN}=&uj7-fZ&`AuN3lGZZ;O1T)LjGB~LDQ7>2r2ZE?Wpi|ydB zUx>#V+B9P$J$0CuZ7$fnnq&wDfd+qh!!}!9BL^$hj59L9wR7KsKvxHIchZZlvFsrIjweO@p5|4HiIU?tf!8- zlMvH%-El^L2bSyUmQlNWrz8Jsj+Iy=G2Hd z9giuBLwDlsn4I<<+qeO?e(W@{p?PLhG&S)f5k3qXz~=Ob!G3;pwPY7=((LWhcPok( z5owfh^B0#R$r!+CEXKOg+~P|vk*C2h5DclV-$kW|f%}C>8!=!_3O`CjH+_fkG$XQI zFO%=snZOQt0L?j}jY?}BP!#x`s4uN$u6E0z7p2aws$h5)wZgzbHjn#Vh-gL=4aQXv z_lP`%_)_F%0KMicF$1NZ;pW;YR;qng=}GoRFMN=7E)YM;%?puKJ14Wrnql9${m!?= zhB*bD)9oWt4*k6OX&^>Swew}_fS%9pZ|cL-W=A1O0gwUMpRE$(ntxqp95Ak{BG3I8 z^$x5uUdcLNeVJLi>WnDkhxEyimz%+9@d+}=$*OT7 z{ZsEo9J@`>7VfmymG;h%t5!+D$C_G(_ea2zUw~s(sx&@I5bV2&C<_gC0SBh;|B}UuIwNEQ~!4=wPOHL2d~?r|fWj3Ah6WsLBhU5J96> z+VV8t_O(-&+c4OW)c(~Z1r!466n)K5WtUgY(Tox2DhWHe*TK6~#+FTyH9?C&&g@T%NOQTK69$J;Q{N%BD9;uS|L zg$qArD{2up|NWudg%5;OIGiq}Wm9nGr_;8srkmRpCbK}GzcreQVT=+joSNMzU{gWO z^uUsEtI+aGvDDpXi>DAM6r=#fYueW0>p z-2}y9{rAW9SXEjnlqP|T7(BIIhGN>d~mJ+s@7;r8mh}vdS zC9TNBIwX*}+mbVcJp)crnwquc*{i8($4l2ShGgA)-?leRbyt31uG{SHrAdi8rOgBd zC*ET+l^eYURRGlQQ`Um()oWWXp)~+c8y%S*`ImORi(bL&4!vSKBq1XbP7w+f9%G$y zTvK#pBU;B+jz-(OFzSW4c3DnC>+3}gX&16M*xEn2W!&Oc%qCfi6PE<8vvPtpwZ-*a zH;y)(a@`BKcQ+3XZA9;ICj)L8-qUUX3pkswfj4u6shiIdn(b{ZR_+v=K5e~rk!$$47>1t76V%b_Fu44Ozm8vH3VO~E3EJelLd=z)&c zSNl2pfHP^!83s|KxT~^iUX@)MyKt{~PePJ<|9gK{Q(wa8!F&O+%Q2qg>dk&LEi8%fFvWC@AW!QZu7;C-kF!2E9fnTp6x#1F<6&@Dk1?V zz7mH1R>Wie%+d@l|A5P3jw@g0p7zmvt$8(n^SqU_LCHPLzS`36RPBOe57W5aQtOm% z%k;5oUAMp^(QDL;>OG(S&=JBVGIr0bXN@A`Wnz@@pnhklzSy8Tb*2p+Qe!0qQiFlq z6n`33gkUl0jQU>7n_OZv&$W@Tdkf#Z!zG{88(0|81y9p_E1srQWh5J%0W(B4Hd?8- zkiKAzkH`Dey<9?miBn%3@-ScytnxgWq{PWH%x9-^f664#eDtU{(P?4>uv$HI@)F>K z?5;3|t3KB+OYA3o!*^kB9~W>mYiy&vki23X6^YEF2Ag)xD#l?k7mWmcDDV z&7b`xslBV*tKl}{+I!(6B8jtzUJ0xvWed$O-v7A_Bn34wZX_F%cdLBswL+=s>u6+B zcO)GKej#u@?#g^1{OIsCahT3<7Rq)3zU53vH7~yRmGl*X(nta4|LLKP)ejc9Np2VDhzPkZ&1g2YNEVs*W z=g`U{!+zm-4BN*&=s0aGw~xNXt_+;!9@TZHzu zC(h0BwFQwO0v!9wdOdfj`=Wxqah*qYT6UbC+4YasuZ#LTp3S|ejRpb^F=2RN^X14)3<1Jqx=+v28$9UiHkCaM`TG z7E_It*ijvEfM!pcsol1TEJoC!@`|Ehz3WW+OLRCZ_zz`hDg@OIxiV6Nu(rf9weL-w zL@wz^S-MiEM{n)D-%1iK&Cy9!%jy!xl}+X(`uL)Q*|oZLmfiZ`Sz@=Ogm<)iG9}l> z)6!Dch@R9JnGNYtKRz=^O9!d&5^cF%be1KtZNBvKdX^>8gRK5`@zfF9b*HgDT~wBz z7kAvOv^A#CgT|tCcPFdX;use*`MX+c%ASHIa0WFG8+Dv6>>d&`G~EuDCJ-dyrBtrg zTK1?3Ji|3K`sTA<0@0oX4m;{vTc<2bo1&Xd`&_xvSrD$+q2hm_QHXG;EHTvz1Cm&QLm?pL*URc z)yn=h%3HLs|nls=Q#ciy^ ztWZ6Y^cdy$-9Cr{z~R`@!BZ+Y#?od~WvzunY}`kd{$kg%AcBgPrbB@VrHDERhwfZq zACUNbe_S*z=fK2{fyI*vQR|l}pR=^175uL1*$vpNH#1cU}+uv3rxXh<$Y4W)=8DiD8>aHV?s{(;(2ZW6Es-IQz*_{l|j3Pt=4 z4|O~Q#GyC$%$><2=UOX3v%9!)y5$#iq?y^pb-;>;Et7~0E1dkS6_JO?}*J2XA3pD;Rf-NSJn@=h;ta?&}8Z z6;_PVh9LARJGpS$b|6KF5>*`x>}g%ZP~?MqfD6dW@8xNA96kLNRDLv(aWPoxuS`8< zm%B31MWiIt^&Jks^NU_S-%xJESda{LFBTkj=wwWJZkmg5EYtjZy>B$S!eYN?=cX6l zH85$X9iW5@7Eui9DgGJU95@M%;%&cu+kNp>%xATT1E|xaz`izVlc$@tHhtmbI*?ta z0#}A6yX)loD@yk_8j4fEIzXtxlw`Ksq%0?kG|VZ&?W%=-RCm@i8gYS^KSBp{n}qry~!gy%>l!w>iR z)l)&hflDcCNop!jv7wZS_lPWTQ^f<9_ecBI(KX%?ex#&ER)0ak8L1A}q-{i~=T|zs zWQ#3z)p1KV4 zHZN2s^DqKW8g9DRKgJN=SgHoS9Z%7Edk(U=T$7zfQ{6mH(QG*1Y~KdKHr}ISVHL>{ zTe&%Dl9#$}x>43)kqUJ%L6IeqvIrsp#&_MEm=kMy1{Kkq@Kj|CG zdYyesR1qXn6^X|#6u+F8&y{?Jw$+!o@LcLK_>F-lf8B#tvDi1;>!IQy-8OT!!y(V| zI-OCTB#q*GNhmlX@btL7o{OS;2FHBIkb~@;Dk8i5=s|qRdD1z2!h2bZI_bKF`#6@< zdxuHlCo4K5}BL6Ubv4}5Nhna$3 zzrfECelK=xb8kCh`m4RZF}smz2(rI=;@Nuo+jd*KXSyrr(WRslr?hb3u)Ov}hs=I$ zte*4sRC}%%n)VdBAM2{FphHOir+rp!z0^MRrL5;dABH4dx=sN zCb=yhDWaX!{J6JNwD{{ib&spF6On5bXW3LN*wkw4TcwDr4Cmcp_{8huqV~AY@zaJ~ zlTM=TApGmm1-D1D{aj|nE$*Z0ROGfvtEqFprY@R&w@6l2GKRceh`UNaSr!Iz9w2bf zTqz96ei|IB)lf%-R~b<|%AUoM3Ane<+vS_O;5%@=6S11UzP0PZ@WXbwKDCx z6Z=Hgfy%#2x5?KM=2qnzm&Ek}^;))L!$-1Y&vdZc(<&}ymIi9x|MQwl_`9N@PBVwS zNmYs3bHSpnXl3nO^HBRx^R$b6+gx7L4@22TwF$Cmib%I#X?$OJm3fvZa^y@Rb z7bl7mgDEXtIQ;&R=W#9&!?_kF=H0%u27N$+F&* zE#KD{RT1ULRM?#^*A{s>LB+GPzuUwzFRW6U$$@>Hj9ydjTkfFVi+80dS_WA;-^}-Q z+RvML>E80&%9T7-iJUa@Tw&B`9Z++^0=ly$AH*8)Fz zv)ABqIJ$)5T?AvXIq}J$GV`{~_5p4Iy`(36yOs}9AbVLpxVT?|VVTHg#FuVc9bS>o zc^_q@uP-$(of3v%fx;IG(o9%b(m~RXc^8^{nFN z!Tz)b@lZtz>--`EbSLOC&G^ z5#LTeK;TslQS;hep=gkdJPJQ89hdvSh24<~3~7j9?1M%4Z^y(#M#HD$F-TcD%1Ab(AbKq2{c3M$D9lYS>vr#R}kYg0*v}5 zYq=UDt(F&K;Mcfn^)Mi-_Xa;+$JWt z!+K&7?is~n8vQvYfylyk{ZaXq$?uxXE|IdCMn#I)a+?y2q2uhgjR8EoB+7IrUUq|8 zb&D=v7MOLF>JoAVPF#8IGLpGQ~FwGG~ zecnh6Sau=;8^of~6Iur-jB0`ztt@xmei-Cr#TT*M>eY6J|FjvEhie)A$@&_!7`0b4 z;f+vYE}oN4je4Vb%@;W3+Rxitj8krdZNE|vdjQ>U%^1?ZJ(4)S@<7vVo~zkEv#sw!5InpNg&}trCtR{>>9~j=)%ln{DbL0aq`Tv#xkr!K`+i0 z+~xfHcn4{I-Y?O(b`i1z-8ih4C&a%2=f^`6= zuZdKuu4G7FBacIJFsy@Oy71!eSw~F52os(w$cj>9xwG5;>l?-DZxl~6-=;?_FTemx zg}gU=_@i7yo1c~8Jxpif>*#^q65-}n0N3!gOZCur7RW0RfeJiX4bIcxLmJ7)5O`x z%0B9(i4DtqU^wM!9xVt_Vfft=<5lq*1n!RA-?aCeUo{_#`t`S_GpCj2ttSKy-n?)E zsNsTK`Y^3-90*5@o_FkCV%5ZKF&`Grn8mA>{$Q252q|8ur}>hL!LmHe7HUZD?=@=R zW%y_1y=7k+^Fql$6e}^dDOrMk@;q$yme5Tv?M$#RDwR)=_IQ4D;)0BnZt$(Il>^V=!co_&R z>~!Je!U(sHgUaHj(;cZ-hAo_T=@9{aS@2KMq;@kZW5Wu=GRGn{QF_u$F!x;B^ikdy z98GcrH;nflSL|}X&1fShJ!a_6)u)X%6L0ImdFy$DZA|P{Qbn$vJXV>_A-0l>eZ88K z4&g17S?Y4_d-(@}kc)EPDiE4AkBN#ax$G`mP%8Q($ydWz|F-)sRtKXO?5J1$(W&K&X^TNt#P)lusI=6DDnR}_c|AX zOFLrGL21^+fpVhty{XdK8t^m6nlV_cyu5K>7(0cLxRWfSHk?Ex7Hg@Ni^b2uQ9KMs z-H~IVuwpXVO(;x|b|WBZ#A$F^)IoyR*o2mrE!Xru863Vg5o^q1@lWRL_C|9e{;YI1 zQ!3-G==iY7TuOOi(QYcjvtteNgi-N$yvcz=P$GK;vYCy5IhU-aB1aAwyjc|(b08D7 zZ9dJM&nn{=hm;#`!b!|Lnv!E^VXUi|_1k(eR! z(c&67Y(sG6J{pS+-~R|pVH38+WBH%H9v6Q#kRE^|GzeSE@0|@V%VCxvW~>z4s?Fa2 zJ`lr#N}&_BB9k~1CKx6tJZ2oxOe)t-Qhf3qSr01XK1~e&&j> zJ4i4Q(Gv@Tn5BDjx-k26(dzY$kS4LxN`ZqU|2}!M}*Z0x}+e@5d?X5PG2LanH zlM368$CyplGq$IyIThRGTAPhRKpLb+tQkAo$3lfUR%zSF$OEbce>$=&31RhB{i&y9RAXEhFC>p^4eglYyGy?Eeh5A1d z0mYn}dNAQ#6rl!&7CEcQnZqakdI@gm<)!U_wno-KXFi={@cR%-$e@5qW3h;2PNPAS zM=dbQ!f7FMOud#1pHEAA^mxWAm>m<1f@#e6WgpWkSXfSXXbbFA# z;%#UMQd?c~+$RwzR?*@g6vb_X^#?5@;j^B!n4vDL9}B#0AP zTXXY>v!k@*R;aiU^O?gg^%zqn~Z+&KFY-{Y|fHUszo44ekmP(icF5OZHh z0)6T7ebN!h?l!JL5JBGppWZQPW~2W=(Ap1S;6ma}abyObNen4Q4vpHKU}FBr^bTu| zFs~0cUq;GujC8DV%yaa=^Jo^!Zj7WfNDm|?UjKT@2ib!ZMb~S@z>~)T^`hGb0ATzd zXbINO^!xvn$o~J9xt{qm$Nv9-JYv^u{2Abau3SU4oH5WOV0gu$N~TqI4Im>fzM2_u zL%UE<7Apu6MigOv) zWv_#xH5f}x_I{mA7f8lfhBrl1k#TJ`13HMNE+M7%?iVDFlsd4U8#a=;a)Yj2+BvObIFeSGCWDiU2!1D`V?_Gy}uu*9C@Nl~D7)K#PCI z1~UUA9TOWTt2QAs+b13I32iVle4-1V9>yo*z`@A&pR&h4=7WWkjggRzoso{2gNXu$!c(XmSJY1V`pUHAY|oWq+?}a{;bIOxq#!}HdxvJ zQ+D{LWPG9#te=DfE9YnNPf&u8iSrYC_$P$?>*c3eL&wR)$oa1hC&TA*CPG#=*3WJK zM=Sr!AYox*r(?I2`1Kp<|!AQu=$wtTWk0<$8>z_FB*#P@zakkH1|7+{>Nsi!OTI&$@Y(>_-``fKmCsXC1x-(eeTxJo$>!B zGZ_AFRt6&j105>|E9WP^!Ny7VS(EwGb7A77)s(~*uQ`ENfro4?EaLG&=z=2hNP>p-XxP8C?7v3P z68=PzNBT9LY}VML=ZwlahFND3E}>y4IxhWKY=4`YER7-)zGgH-)2U7t#yI<)QQEo9 zl=|e^dGvP2J9Yoyl%WYIJ)K`EQx^DA-3^1JFW@O`{9fcZX!GsQE6$fo?caP|w?m!G zDqrB%5%hTZy4nkLc*^SM)Ziu@78aG`*MKXJ-}*fe2shMtz2(Q|F&;aF_g21;6<1c$ zAbM})BVWjU=RE;Tu4SFT6+P(NDPP34lnq8sd^y&E-^OHa>a_SUMC4J1Md~CF_-Hk_ z`Kbz+>ZtraDYoqCUx*2TL~hbcJWwgtkIc-PFbAJ2(AfsQHzr&S2bm@#A30 z@(QjHh|7|);Q!0?;2tS8Z}SJH5Kkn0A*5K!xHj>i*mX7jYvy;VAAWG(j-w$I%`G}X ztRpa_gefy(<3C%m{jPRUcesb+Kn$_lg09g0V{$^gUf*e19x=gcpx0R*Id<4l$H`q( z!XYzWVMzId6x%|#W&Lpd$f0b=nlrLP^pNvR9ihDwwp~*VQEYvv3J~{4oTCfa^&;-U zrQ_Wd+_`(^szvDvp3v2Z;Q}EGaI2-Z6uoI5{(eJfNnV9#kY5$OK)2w}VBCN+)SXC!2L$f`arfQkUWZ$=pK9p%TE?KZ=_Zvq&<|^L3LL!&x=Z`+oewdytPWjnyEm*j@l2>GB)Z2~V0y~|d%@`*8^f{WM5ud#|oqYsd{ zGqa`d`HgMqD^b5?n}gn1=Xh5)*RqnrJ=OnW%A*Z~8TF(K)vt?xRP(brCc8GQDa`i< zfd5C+=RSxXnSLnWDVdt@oBBLHsTxg!P((W(3@1q+N^V=)F(yx$4(zuRW6Yg#6Z|W` zHa`L>V+Q|gECmR-1VcjQRk_C)e2_-DzdN+WKGw7xr=SC=SMo>7Mc^T%H`<=xNdP}B zFAgYU6~;vImNB`{s$Nk=nlBZmZ(e=7u!Y#nGwj& z)iY#20;o;{XW8n&>~uZUhl_$ofUF{{ypfz|$_CS#KHKPdz@S_lfzuwBA&5Q*-Nw|3 z?uLj%x;omaL8Aq$5M`VMvuA(OXw3NNz_c^#jdF#yAuERe!Qi9cDs{5D)>6*Io$;t) z{9}aPY!c)9TQlzPWvj-m+zXU#oBay|&bigf%Lu<3^{A*jua%+mgjH>sajl^#ah9={ zVLjmJW`Eo3SH-@+?>fP-4)+=zCtw8DAw;#-a9!XT-T|ebsbiEKe2Eg!12C|)ctbHl z8i%Sx4fSiOML>z-WR7qVwv=DjB<_Mk4J5io3S+H>z>Z>_gKFy4syqw%CB-PV`d!uG zL)IsMJfLQ_ZGQ9n5{$6lQZ?TRD@C+ZvK(lW@<$KYEJUb(dDK9^CXu=Z9yW$;Y+qbezx%1zc%;Q)2wTWD|a!sc0zkfM0RHm?Ftj=k((6oEFul^E>W4gMAK1 z350kwrL2Qe>P>^%GhcOxvpM5OP~wSLyhD)NLd_<1!__hcnQ~bZekKcDwYEo_IW?;k zsBB2TnFtm3n(>o8<1;O2zN8u!@Z2s z&@iX^G0{Z#h>^z%ss$E#J2%9?Ew}6k596EBhw{@acz8#x(#~JzJn~w|Mf0=QHRnke z@VEvYL36*n5Ao>+8a^8;74hqw2V*qRY~7{cz+}2CR?Pe`_IT4zf<`LJ4@EU>=0rZ)4eie|Zr|w#0@li)R#I zL~AUipZ!R%OF)L5gQ_HHmVHIKMCS8L!$j&#Sam}`Q;#uHl0i^GlsBG_5b$*rNgOt( z15IaBjD(h^jH6p7(`q$|{o=~a&0wtA*sVO?B;HJjKE6zur{*jGW_6alzZSt`CTB8I zINGT^M(3y^v{^huO%2O)wrsY8{ALVch9YtJ^>_1gM-&TXsp(Sq_D)ke)eDRd7}^YE z57tXhQ!Nh{CUt+Sc2(!t%g^l|CXz~3X*+K`jH#~M?a8oj-4-n-G&NSVUOaMNwx3

I;< zr0e?7u(ajkxOB;Dd-49Z_P60G1NhVh`S*FdFA4z{^c*Lgh0o`3zic0>Id+^c)guhp z``wJ6w8xA2;}Oa|uy6OlAk8K1`OkBtQ39DbbAiIiC4STWbCMKL_)Ll&QArE!AY>%uBxE2|74QLFhZ6XUl#z$=CnHsjra@GHX9#$RaY%wh z6bWy}{Zl?Lt`ns5tn;o71)&~@fMa&W+-tM?MtI@LeBl}VoE#7*^K8ElwcM_e2rHgJaJ9`9CIla)yrIs@z`C*fJ3!SmkeerOxh0a z9Sq#*(j#aG)9ulEhCB~``iy<@+uF8yhT`)<%)##TwcSx^hOp4X9}m`0`+}2=fG)(# zB0$X2^Yil@=bBZ*gp$VBa)(O|;^G>LCiv_G-4QH3Nb&k7et*=Ap9-8XPAJ6=xeFBk zKp#b(-%fEJo-&W0*bOHBkoT@Brf&5sXuui&jc z#Ao6USnu8}30BB!3=o>!o~;Xn_fQ>XgzS0umq9%m=RS^Sa*rTYJ>V^RH$EnPHX&3l zA#`mv6jdw3R|*CFpRM3B4El1?y*kM>$>6X*VXiqnd?}Z4-t=^Pr~;tbh_t4`+mMKU z8Bjg;eI%1{1W*uZ*ugLg(SXn(w*{n!;GR)rp*FS&af5}gDRF}guNiR%b~wKAa-fLm zBUtrn-5cN^8h(u&WK{cOXpb8?1kHg=u8%A`NXda#)YGRHPRfC2y1l@G>d65sRRcF> z&!)1CTi)NI7PwH2Zh4KJ4vA<$J2?cd7W~kRgj0=-p^r*8XkyhvS{+e(vnfRI414aw z9*A6xr{kk-N8FB3F--*L^S0Bq1Ml)}Ggx(pm8Iv(4!t{$v%qfr*bl^GYcp9KlZo6{ zh3$MWO|+u!&ph|~Bqei_Fh7@pNw8Oawxf?;C!vU@jshN_RK;C2esHCHnB>+13-ek7 z1abo6vzFu!f%2-Gfh=CCSmch{;wMC-Eg14{Lh+e&lJnYqdTL!tNlO>ES7U&-Rr|qw zlwR8@fE@5e&fWues6Yoq8m@zaViq)pqnL4aYnU>2Hl?YxaY#E>@X)9+&Uu+G>DZ;T zY|)5hK{TGVTe%fV*%w1B6bd|6`C8A=SfML@n>ckNm#(xs5g{d>Jo!c#Ty>!80HV?z zQ|@yKQm(8hscPIw4P)E4!X%rwosW6F!uH zl}lSoGGS=w0BavFP6=S!suzU8=BK?5>pmKRPb%>j3v)DbnUoCss^8az7~#YN8ag~l z`2s&#hW-OBPEzQKE*Y;0p2ys?=q-?m?bhN*d|rvIHrFxV(k+GCw`5e)1f;Hox2Mav zwm_edKlPVnh5#G*GN5BzM_sC}sk3r8IRQUnEg7;EC-tXjoudqR=EIUpPoF0|&%h{6 zbyL)G_KxsWjPw=IoQz;WmUK-y$kIhTglLR5NaLN#o^*3?+n8_=glS~iLY{CQ5WZbg z;jd*sW!g=cJo$Vrb`-7P=DH?%IQ}q{G%!K*mVQPRB6aOv824P*CPh`mOxptRzA~)W z{@!lJizy-!`;aQ<_o9So8|DwV zHa$Ey=wU!yo;m(sEFqA3g;f{^Xn^RT)i*vUu7&fsZXTmP8hVzipG6n4Wkb z=+%_iFj-XEShBvw1=cW5Oeb@}as~I3+-jYonqh4Y)MM`5_We=~X`AKUkYU?TQR|%? zG@5wOFc*bhYTSvlPHnSuz&Qh$2Df%Eu&p8*$>KYtpZ^ie+2jf3v9@-#QPt5@ z*K~Ea8?gF}cWAPvrLih|O02P#5)ZiO!B8D|x%+>Zd#mU;f?&bYvPBlNEM}I)%*@Qp z%*%b!Y=D#d7{#h zQhS@%OD@aXX{?^Z4o+w~86_ynW`O$X28~V3))hD|sm8DS90W-E;H$qIRVi;eG{*bOYZibA2FEX@)J} z%K^=m5uBrBGZ`|M+Tu4nZBFeiLVN?7Bgv3c5~qtWWEv0)%3kz6q68WRT7m<=lOsiw zjFMcT1lJ87FnW|-8s_DX>?rmf$hx!fZrth2)5u=3;n^zb!ngkhla5l*ztq(4eXqU8 zS?cZ7bF9Iy_ZqUAQ;u`jjk{m2X}242eg7M)OR@ZOve-HJ;LguBQ1`w$nj8Jyh%DW8-3ed!Y@Qsn%QiNAQFI#EW*JBnMzN^g!Dvh^ z=GoaSbj_mM?{TS`);@*??RC8L$PVwz{Y=@wm++dn3j5zXO_U2dzrBHlJ=Ja{Ub3a2DYv6ds0D)RoW-=T64>~ zXj-8{vK6Lmk&{-AAr;aVYYJk^1tKxF0Yg`lDnwluJ<}wVouLnMh^ou;jAt{suH^ZF%iXW!)=@z40@eryvwFoRY1 zY3{*TVua@l(aSf#*0&3KX+ThzC}Nf)l5~G9c6G__Z&o5;{RSnnhI%IX{UI2g*rgij zI8|0!07{g*HCG0!kNUD`aq=qE+$kWt4q7P!D}--c!8DplxI;{{vY85zG7DWfF`LT! zpu(wSgLIpCn}VMxYY@#kLUp#9aS6>jN)<(>FwhL^Mukzn(1&_)!phs>!%bHm<~wJ{ zrq#jd`#FX*^z~qdOzSZv}aI9`etYA6{8*dU*T#tOk~_E()?bfYZqlG}{5T@hvu zNk8xyph`L`pk|+Y#)K90$+sI*IDL;c%dvp~-@DH_!OGuio=X8o zgKhS0@SbkaTiKP~T>+-PLGN5GSbS}%)F3!2Pf*A!ExhVf1(~&`R>M+tixo}_?;6;7 zu-Tl+Xo|BcnI^ITqj3N!l|nvs=J?%AbBA+Fbnx`V((IW9|0t8 zuCL}px`3B}G6eU@RQKFp@n#1O02|zm!wg*Urd5`5qEGObou*zsCPrn zyuQY%GH(sL`+i1whH`_r81!}+N4zNqdC9}zJ|Fz;8*4b`0x0skQNLqR%r`|mF(lpD zJi*rspe=YfbyG(^aUF^mE(Z$*vv)}EG-8ByRPLZ9hR_n}uaMv(sUQ`KuAV|OuZm?& zAqN^a(+2azZA3-zk!@Rw0s@1VfS>E2PW%5GOKR;K7RzcZC>#X7iOn$faM39 z;t&|lHulwXcW&z|Y^&BR@kxhLEj9On38(aQnrS@nPGiWVDJr1|`Cw&^Jl{FPquKyD zz%hGnh^Y;0yCQByGc-978>L>Ii7{6$dmS$_(!xQ`3mF8Js^clUGh%VZDt!DQfNICv zRgu4JY)#@!;GD(t`lBabb?U$cZs7z3fa_!i%SyH{L>fh*!G?sjo9JHztMFlU5 z?KBF&2No(Vh?p0~bM&YRl6ACGq~7(RUcNQ*F4E6 zI|KL21G6)Re^6~-!aN9mI`h7NtxwxkT`qo`0Ha8u?^Urpsm->gV8N< zvV<*oT7_!ajSAMKt4j%bnPZ;y1Az=R#0Y$XtXQWWvCx2` zhr4cni#xY3p{h26^P{12J^;vpHt<31ce6Z3Zwtr6?yPPnFC%bf@BvP{|BZ|^PGawf zLf*ToQEjz6T%n3qIk;ku5+j(1oJKP;zKl=?UYtmuIu(3>ZE4{LtLT&(>;g)QmaU?C z<=+L_O0tC^Jrh3v3;Uhj`R=u-6z!Q$SXyERx)V_dd#JN^D9tjUmYFNi_au`?gJE;a$nXn2$_x;Jq z0Xx+6EAAs)816GnSI(Piz3a;}iI4XX-n@QB*tJZ&@Hc&eWBq4%NoHOKSfR(3&^xHu1wh`>`zMa;y~hto&Y#ijwHUJUZav>cg)ZcG|$ z3Y!Z=6)o95qdGE&x|}VI#q%9sH2_T9gB9mO`bVM4+-_Yuvu+SB5wI|uYDTLSHpqca zT})8$(}fm7(?^p|m2-*+OwIcyOx!TUL7Mq#8xaYywE}e}HN7VZYwP%DwXM+voOE@dxp1l~^N&d%lrzlY*yt($GNacE66S7O5Dq)JYtVNr`z$2E?{ zeHOB;Mwy{fwfXP!b>#3e31(3$CQ%GolWdCsEdA*TUD`&{70&s}O=yNWMK{NlM$Un{ zI9Tv;_Df#k4ehzad612QpQpwMBAGbIeSs;jK;Wk06=4zF?Z>I)f(>IUT9!qpTJw4} z5MixhGu9vuE~|WiNwh7~HH54rPNQI|RZl`|*W1`jB2%W5xK5+qVP&CW0Qz^^Uf#Yt zhm-@kZg%%22y}e69=Ht?;o=2fdu`$^YT{3dmNoTw>1P3RP@MwcvP>$LV6ekWTqiW} zzS541(Zwil7L!z^9SttJ1Qr(jKKZ-TL1qGbhnHbmw!bSkx#4DZJ-g19MCnJ+)wLXG z4{uwnd>8%;SDwE%#-~xob^{DmPl_m%Ycf)nb-_?#KPUu~ie$i&lBBDY^Dh@D;d1G37>M6nOe$;sW~e8Yi^T5$8LFDEYnnMIyM!<@u@$vos1wjRO&vMqhLI#% z0A8;lsOVhE0rBENG1Tl+64}3rAgWjmY$7BG{>*fWi2Dmw#YLdGW)w}6G?TW_YM~$p zuIXxzo1gZD*z>Bs2^>O_OJ2bxJ>iG@4Mj7AoG(au2P!#Gwhc0KE!d79nBVtDfB`xH z0R9*^-nlWwuIoka7TndJoT0#a?efg0K3A0cvWghcqNdg4_^Mt-uK_oygN0Kx52y^HnSD@8I zvXjJov`g>JI3MDf!lWcHJ*#mx6}MRLl}NO~TXNA~AU7 z$Td{#oT@%)D-XfiuS6Q4*=PZS2sjcijx{7z=1e-JPRps~pk%LRi&g%lZSwD$&oX$_?cp`vA+bu}q4(+fWpim3GwDW`HeZZ61Ttzv!B zcZr9=$US|13*!|wnqT2f>2?o3<{FTwkar{LWF$giW?&NbIy>$A435G4t)(1az!Wkx z?{$c1W4_^6WI0ao0p9X=_5>4?NZTMuCc~?0bDPxAK$}P=vJ4StjkbC|!@~5hKMA2# z7>%7dm=MvlNP|=E>>4x*LTWV5x~ikT6#zbthL9 z8mo4YLmF59+RGdE9rc?w75Re*`q0l?JANPcRXazK5%m6}*Gp!nK|bA&SvwAw_ag2G zf80=`GW z-d)Ss`JWkP{-!%9?K`@$RSR_mV|`>2!IvDhioNJB*zMrmv(t{Kjty{4aJWQ}dF&2a z;tW{r(BH1w1LiY@j&!h$g5kcY_mQDtw?pyF0*g8p86$TP`8et>j*rQaV%8lduD>z%JhcTl*R%?`@lTYO$I z4~{wO0_I`Hx6D2G>^tj3&aIqlT}2W|Mk3Ex;?Hr8r!xSES;$mS=0Z`Wlyv@&di4DtMj0_vVun3S%hLSstSoLEh1Vsmp-SOsb zRX!dQQ+|Bw6rBw=tpBBRcEjR)avt|E4z!P=lP9W0(?{7x(=WKJiT~!RfG8c_8p=_9 zs6xe3#NP7F5SlmW={NSNHQ}0AU`KZ3ludreSy#4~6}0x3H%k-J1ptuCeoE|JP12UE zCF<$>^T!`7a)6eMo-7Ntmiri)XZr$7kwx#aZ9=+9&Lyx>u%$_pj@X@gEB5gn_*_pt-IQdLrx*f@9pr-2xc=(&7?s83+^$M%&Yi*D^%WY7##(nh zPsFfVchWMEf*^(cVm7oZqZ@AMV%T%rE-Z{ekQe7s?eOINE3l+q9jX~W7p_w1>NdTC z*b>xg4>L;fLxr$;r(RFwt@Ui8KHm#CLX{nb6`ISj?8)4kf5*&dI`kjSHjW4!vuCW) z9q@g=X@2s#6vMa2aNK-VWP6LeV@C8VNa-c6oKThn+NTlF@2K^ri|6nca(@i^_hq2b z!>83II~@}z|MV=bDi*RFC8*{)@K@B_e?!bhZ$#Y@&^m~-SdrkBJc}#lN)^+jr7Vl; z`v4oCWf=Ki0Ez7X9p(Dpv0KcH|0BAko49Ee%zzYfd5hMup-1~JdL9TJbsmWMR?v#L zLk?t9_^>~_0~GN-Tl3fas0ES-0|>PlfZbZg!MkfVJ(NEXtUv{39edsVeBC{2+IjPcZ>HpjxeyxrisPuG&sWLNI z==DNLI_DX^+$!LKz2T4pdX=aUMg*4=j>j`89TrbI6(*OHqDbr^SByr;iNGuQiBL-y zh93ViY*Tsq34F@}d=O?)_4tHDC2c>C6Z+3m_&-ZYT^M>% z4;OJ|7eg1*f5(b55HkMLy3&8}od4Lvf3;$twf_Lge>fr|A=5u>^4~`$!vDr7exjfM z=ucTw6H7xOdyjt*=s#4GiIAO>>GK4_pS^rut25z0F7iJ*Hnsc2O08i23yl2#Khge! z|NM`u{XdfJAG`ZsW&1C#^Z%u6|BtR1|8EWcpDODg{`y}9`ltW@0bTw7#t#2yf&SUy z|4U^t{x=K${44&`692i;|0JCAKht&n^Z-^EdTC3OPk&=?|J}&R$imLb@;_j&DZb~Gl-+h)AI|8P@Fu3Ow!&1TvT}ZHxW^F5 zYzh(uCvU>{qq$hx*p}Uf17;1atv~&vtK-C#9)X4Y{+P#gYKA(8ksJ?$1YL)_rAQ32YUZWg7-K5j~md3)7y33N1WJq^$Pzxg(E~3 zz{g&}br#Bt>&IadZ7|AHT9MwRe>cP2YXJH4NblRh<%d-9L0s=!U2m{AQ4l0d%iDFE z|I37bXKnh&ErI{b_GNEq)w_;5KKE1#xNN(hP?Trcm@1$-kJ|E`+QLY|`QU@t@Gq3k5dP@EIfbrphG=vTgMzmY<8^U zJ^;tIAYbv-C9^8jZ&DR}s@-odC8#T;=p$LDiCz8oo;<9j$KS|+){ceaSk!U(=nAi8 z3b9OSz-R}`l>szv3V^=kX zpP0=QUfG;F*Z3bNv}MkR%3;a9%3(p;5e}Y+c)Hog*J6RcD6gYGW$s$y79v3@r6TGr zF}GYFLB%KhtdTR)K3ow!4}zf8oT^zJ;A=AaOvs@WiPuQ3)$56R&BeP4SCpGyAQ?=i znx7H}{~uz$g9ZA+@3rAlydV3AekIL?dHBLNtrYsZ6brYo$tcwN&AfqPjY;38M0vcW zXIxl?q$&7i0kaRY!CPtB%Fh)n8ScfL;$F~^T~nOwg&iBV(;*%go2YzPZAYYvGE3mngq42Kz@WAPiYcQP~pzx%hT$p{pU9R=$%fxGx}N{vQO^GtTu8PoyZR=Br9A` ztq{GBgh3F+OrVI5&7SK^RWTO-{pnE>$ug@&ty2ea58y9BpaFZrMvDzrI_c?P4tqahV1PQ zY13Ncg!MaF-g22l=7VtLvBR>cmI#cHeX97J1?*?fp^Ny7I#$_o)|YeW3a+zj#&%n4 zIJFwKTaDNm6jV6?^UY}g_bkJ!<{1!x)cmDf+nOwdij1#6Oe6yvQ-*-A-%dA8z{`ok z(K(vrJmHJ74K7e(Z1`}lnYHu2q~4R|96Bpc#h$xRTb;-xQ+jq(KBO9u$7TzkW!@N9 zYiYg0lN2n|-DXT@xjcI>%j@c%jqsV~9`qaXEIfC!=|}NL65H_wQSu2*jyW~DUG~fI z#hNkc5R3j4Uf6W=cfWm2k@Vj+Q1*z1FFG+~=h7s#KBNt*;4w@RQZJT)=1=1yN`ZB` zI;kfTl5@9h@#mR;H-s#OciXLOw$gA7Z{)Iueq%=rPBm(7l}Nwk#j@xOt>=J8wSBK3 zQuWUEvR&-Lir8xMYGClQO!gF^m5w01_xGya8(lqsI#k5$ub0G57ZTP!YY?+XTzx1=TS+qFoaP3Q>?kiZ_n%j+4>k86 zR;(X%&Ks7`I-YIoGW5Mms}xHa&6uil)S(@4)+NJp2RmTeW$Vsz>N>_waOTy6#4xkbyKP^tA0{LZw}O!EE7E}LrhdxCRHhMUB>IY%W316YrLdM z(Pex(#WqcU8s|pyA$7R`MRATj(V1Adj>84g>_H^-F-ol`09j)b9P!~a`~h>II5w?9 zqB8GtEgS&=jVxxrW42R^fQ~;!rSgkwUlEZhcbPFR-j6^qR{*%>LUZAvlJ|-QBE$?e z{>lhbW47aHD?M5E3vG+rAo}WY6Nn-7cXW>}ZUY?x92w}@K_~n?CFYQT6yAvS3g4Cm zfG}Y)pX^4fyDDGrE{(hOFzAFsJN>YL=%-?E97JHb;4)Z|>8*o|&48o1uH&S8hrrq@ zcJ>zFh_-^=a3lo=mU9>h&qU~F#W57eG$d@%p*raE?5HEC1J4%gQ;=}nP}$Mh#_3w`Bc~c#T(F7EVuefD zhda~o`sa@!*Xoz*UoaqWFb*4%J9hfDmU{-zYm@6pB*g>Uowz)q|^S{7SgFtEpq=ZT>?a587~g*#j$H&n+5jW`X#)3AE-sBhGxY$(^c3^*Z4Fayh zJc|>v-BW(gl4ZWLaMM9o z(mGH_u+~7=@q}BIz@p!f$u!+E;fU>l!NNobzIhKbb%1p!fp?@Skp$8Cm&JF|b?7f+ z++@ezeY*Ie&K=}i^N&Jq1WQWaOxC&QPLf~pW1D87jj~O{dP}I&o|eU?KPK?>oHXb% zwkrLiQB^gdUz3e-O>4kFgJPY$=+T;?gPdI(cF>v3a=kGT3(SO zX+?MEuuF4P$jK2%U))iF66x4`LPoeH@6I~M`R$_$qDf|=?({QhFwzJTY43>G-Lw5hrr5C2Z##fyI57*!$hTvG z<{=S9yl5=gsNxTqJDMXtUkqIcV1q- z@B~ySrLjA1yu_cgb?w}X?bl5$t(&<4N7*7_2e_48500M=j>C8->mF)?pQk42_)L7j z6+657GD(zUiJk?3g8t3ubzO^y)(gOiJ^|sLzph3KQ(pjF&hgTBUV`#*Q!*?qgt8*EY^NpuVtF zMSzCEqKE$#zO!FZF>cU&Qp7K1m zIm2F?N(iTVw`+aGK<53Yq3C4;JZ=s{f8aH?QvlZc4w8{2W!Wt;p zWa~^tb2?q?cxdJBHpP}E^822?zEi<835rJr7xe@{Wz9~oi8L^^0zUB$#EH-=j$+%LR3nm9= z($%Ogmz~4dx2}E8_(`qSVeX`8GuGpeBMxe%#b;ZR72Iz>IY006K#H1C&gukv@2TII z-&aj8D4E^%H|MZ8BgeR9V{sj+kdo zZ1cM8B5CHt`)RVDB*$}she|BI!&{TilcK>N zsI?bX1}W3sQrMmtzoa5t{zeyClSQl&Sf0~3PH@saFR33(^!!C2zL0i!9!}ET-61ad zOBk91$6~hjV^3^3`X}c-d@Z-w!Gh!$ zw0co@T?mCf#f^?H89*(VX`T97;wnxvH;R!wE8C0rFWj#1X?V#>7WX`KAuitn>)hS5 z61V=zgy{Hz{lwz6e5Hzi((gPd+lJB_$33S+F6CsaSCgke~o zW18RUu;T8EbxM2#h~jGlUtm`)^pV-g1$-7&))5Dj1J^-)x^#)F5Sx&?vxiu`+c;iU z9{7`IY|*B&<={JpWh3XZX&#dp?^7- z=8ydtw7xf;N9qryu7vU{-fZ+gyg%Le5BN454ZxkKcg2d7_`x_3jxeO$HHCgLc4$j{ z2sPG2UQQV9H|8rk?`BB6ZJ2Q)^pggP)! z0kIRh>ODJmNGt$P6r|dysr3Qe-C9cg2?4U1Y_2fK@_nX_GT8F{3~w|30M2bsyNAWm z;y7_Zl0U*%IAa|yYDYP}=KSZ;I&y*L9aGWRQ**j6zXtYv(ib{^AZzJWe_KzNtktx)laUAfny;tnzS4)(?+9cj(CDu3U>w+<uG+l%+!RO}g4>?|wZIw3;{=4s1vZ&~kvV>SgCIno}Gl}On zZ?JLs$l(mc$n5B%wl{=C_P`8IOm4V_v2t*ztFV~UEIDN1ZbuE*&BlA83by{Om zJSKIi3Dl59;)flHfnLY>0b80A?40Fy>~luw`(h;b zCuOe_(zlig4r~W(v3IOI@q1~9%*u(Itm_;AIf-9i^PH)?*~1QM9X;sk*~MI*s60EY z+5GqnMs#O=Qf8x<|HKLjg1G%I>v=97y9aK7fYn2#!{Kv4L)%r&dt*$|WXyVrvHi^7 zfAwblP6alk&9m@=DP~YD2LFettSIE^r)w zT=mQh^_{_F1&cm+`@fXDnHDS&m)Y7WSE!7)1K8M8`J+<>)>1(+cO=`9c!k~>hJ>Yk%Hlr)h6&X+Dvz~tLAr)Di$sEV^r{gS0##yw<8?Q%c_>T zC&`9Z*$*0Fr3R7`RlrwybAQPGg~c4J=sc$!cD*C*ubiH)_OP79TfPDDhO<)G*9RuJ zUHOU_U#@}ET!{g z)QgG_r3hMW5!@k7B0$ZTkVUa%F+ylzDIFbiaFWV* zI##%iY6l#N<{HCr_06C##gRZUO;UNTlDCYWljLcRa_&U=xx(qM<;K^|(!ZE%5z7R* z)`L1ph}uz-q=PUD8b>?g7qWBfD#SmZ>hK&@%c2xBr{(^(XPLYvxKDgMT4_*@HFfE{a z_FR^2`CPAGc2=COI|+=O1hH=Oou(3MI^M7NO77shDl=T?QmZ8-iF`4QIC z7dlttWjl?pUt-$~Rqio=@SX6`p-=Rv>8xxC$+ac3FiCvbOqp<4<6+Kh*B#;DB17%; zb*h6L8I&Ho(2>Z=%(e9PzgPl$uoT_zpVF9=CaEcMBS|~QndwNL7pZTn=O+)Gn(>eU z&&qQ=dR%=a<=6;9!cs~Rqa#!IU$s!lhhQbWbp)1xxQ2dL_O6T!v9gstf6PjF^l6I@ zzkki6W_rM2noOHO;#gNkoR`(@K+8maPFgBweSzXHAiO2bHrdwqqd4$ z0#mkS2T_D|RRh^##{+y+ypss%J^(Z437IVY))Wy;`S~)X6TCkK>?yu1zPGnOMtRe> z-$}<^l;wVg6*K`Un;-uyv*7feFKEm|tVj;l!EA~QyDOqQY@ePvY@#DC5;~s=4WcdR ziZl7HAXW9Umv1pa`(*ErN|vH}v%Sl?S@L{-YjkJJrOzHR`_sL`&m5PKRu}|AI@+4K zyrgkt96Y=<^#3aS!14ASF2g-~qQ=GkVD?Qp`oep1`~%K=1sKeIX2=u`Ep5T+i39yY z=-b=U`kqyXc%l2ouHSpyOCK;Cy!Cu3M#8%;uU#D@w&>EkZp7)lY1_bt5~B9?Hxt2` zMX$g1BcVBN^2c<115S(z`|dpAy-p`JS>O#mMNFk1G%bbtPan(XEvZ+B#r`xgqFMZr zk5;!7N_Xz}^EXet4>o?}gLh74KW}A@pX*SHN1NC_LUusHmkd?FCICo3C4P zQ6(*iGz(I0liLK*&N3l)%>q}W;fsWYj}(rjZ)s{D$BVCo1jo%x2pELC+6Ed5sB1CR zMN4hrE89=*mBoK->BTp;B3-Y-or^1Z4A=wX(Ro`W_%64 zdJ(~MHtxq)I?q;5yH~>065Sg#p;lE*`|(Xvw$+Kk(`-p@iV!i$F%sc9^x;(4VB*on zq?e@eh}_bzE9~fV{0*Gc5*WkO*T$C!)#veY0r}PHO57bE`Q4o1Z!!83qos*O<>h*s zx^U-pket;g?ATmRoUvLS6IEWdzTXt>EjpdgaKia~=W~FQqV$?Ii|arqU4BfckaI%? zJIZE2x2_o0BDJN;lX51_-A=4cThq?niG=@fc-$y_ICj1^+S0>>J0885An(dcjhFo*!%8n*dim(R|E zsK6sNpisaZIm}ztFo}XOMTmE4F>fwau6~SyYn2a%kfiK(G$X2Z*ad%89geXpL#g_) z;rv71wSQ^Qlgte5M@IrGVfxfD+nzvEiV)jCTL4^@>GnEw`;tBSX7qLAvu{fffvF2f z@aqv~rs5)p6`jS7CGP9-{-}6XE2EwXE&chT;23Y_jc`$3BE))4N+|*L4@bDp&yYTGiO73D~%L=h5h9S$O%3?a+e#4;V? z!}D|S9^~L+OHuQ7yw^y-^y%o~8`j5laYx&g=Q_^K7k>UVr$4{F`11s9M>!)i6sukM z9-K3_k{jx$(et)VvP@l-+>Z3^I%HwCkZpD7+7${@qPft0E9TAp(C1ulb)72(maopm z(i`5Pu5NjGV_w#{-Vw3Rek%9xea)%^=5uvY{@mzuy%n4ZGhOS`l8p{@23J#a2=nq^ z?WQB6UQPbb(DzkAY-3kA=Kf;ZS2$Vb0(%s%A0^QHNfQO!FEOvKRN;zv&6g|lkE(HVnIXTe zIzi+%eZOlsAWIJT+_Mcae1&E#>$bKf@;x(Lab}Zq%P+2 zbAiwFAbQy&L~duC9??W)86@g4JkcakWJnrh$3D=shNU-&lJb}$*BjO3JPFqXNX%eY zAUWY~qLgpck|gCxJ#RbVWe})|)?n4fn$Ls`SD`XoMx$EzgY$D0wC8h;jK!VtXYzTd z*)=(Ibxrb$$mdg9s!%i3SVI$05Z$Is3yso8q*K^3J1KIT!)j{{J>$A*=?-1noCFIw z3rF8-hqOOee0j?4F(Prmm)CD7V2&%ROLqufo^P8=&+W0Lo1a&=@t6XhI9uS$=P@q> zPu7PSd3}dM(|L0FvhtQcTP;S5+PGhfJ6$Zp^0L$D&L>3)7NV8yfj%l>xe^~n!k#-P zKn~Z2U0U?5wo^F>ae{Aec&XZX0w1>I2e7ovx+{i|f9&LLTf)HXN+qzr=Y|5N{b*kX zJkg!JRm*#rwR7x%l$cvJ@%AhzSUET2GcELpT~{j~`_*`V>?BEw9jRqwZnKDTtX@|j zw^0@COy=@#MR@krmGnY!sP6z4RPMqdY}l$fwVMC{e%L}N+0P=^?}{d8d%yG2#H8!n zpTk$33r1JYNFD&MSk1FG-*ibGtc$R!_WGoY_P$@qGjlisFN5LahXWG~31rhc;nKKh z^vdVclDW?%pUF-{!+xTbWe@ z0&u4hB~2&vCxW`?d|Wu~;EWtKO4+S2;{w*JMO$%_JGSWxBI5Ayab)H5YDkWvqt4lDsUl&k8rB+=7P-0Sb<$@N-Jt|~A=--=&YotctZM>&j!os(=K1I%I$*Ez(8g=O8h}ThnTU!>_~$Z`gv~)D|zC zF#lLzZ~(UZ$C(cP7U~XP(cfDaH}Z35b_d)5#ix9_^ULi_frt4Y2LI{K&zoA@oKc*x zSbsIf6@YRFqkoCDeIgL=Pr_)L*s7+BFFBp1vG^ib70rp2_LBHWl{7#CJ?&bS*Nca_ zCjyq;9!P0TGSffB#(f6jkaXeSv0g;2NAw*gnnBcblnLZ#yBy_I8f$UL>83IvZr+Xj z7OUc9zG;hVxwO#QdCBDr*3RNeeLIP@H?!H3?ffOR3nrp+93^dWVri8i9eYE6;+=G5 zVTTFOIx3*Pixx=JJ=Xt>OiSa6iG5gS4yjG{vbv2pnWCD9oTiG$2U`8AO`Aiwg*lr9!{t z_MiV^6S~dv#XAlbZdGqX{3~9Fw9SZ4@7Ef@^2>dc_wBIhneFH@*yS z~vA;O?0F|4d5|yq19D(=VBzRvB~P2Yo9auYst^=!*PLT`faHu1GNPL z%mx#?P><^q-PGm4rexAvY?0qaa3&7Je^e{}9Q-(5uOMA53vt+PX0ah;DdbWHbNaGn z>e{z5{1j*;3o!~Q#J?6ZCHH&I7$@fL?5hU%JtIyR{CQ{={wk*NUZa1}i4`uZy6;zG zw$^UG07m8T;s@|y8$YB8R2J-X_zzDC`1t8)00LR{zG<%VO<>z2a^viqrn0 zv)P-5Eaw8lqiTOse~vSW6qZV}Ss*`q9bzVXvc|@nq4{4IpR7+>R~^0X*Bvv-T*9u+ zw91(=|LUA+$c{08ufVw!{092sDsCdrXW#V)~jYZ`lbvW#Z`z89NBqb`2%8H+uHAv7M?AuEJ%x=f<;TQ){MMshXG8* ze^ta&K&Wz$j;G94)En6V-Nm+k#hFdjGxq%0-sP<+iLx3-Sxl-Sd{xyx zkehIT^!$TFe&PI0n8Oka@khU0!!K8cw@fz}XqUcEX4o+g?kUqx7BvofbUBbW!#d1$ zmNW*OZLoOw2|fFdUnM`{MG%&zy~0!;GYP>B&3G!o=-Iwm=;H7sdb*tYie=BZ#X&#{ z2e7Y04|%1W+{`XB^Lj^9x4y^c?OvWFp%fh}?xX1E)U)#M?n3ssM-T`G)6xMb?p>!q z=Y~O0$!79%Yh~Q3zgOb5Zo1>4&`7EZJ&GCgv>Tkxjcj6@%UfDKXS~;AQd}){`++VV8+<3?BwsyBJz=XE&p*X4U7H4E z=5!|JrMEypN4N1V9y@l(a7Wb53MT%dQQ6<^tpT`nysmd!LkN&N8cU~^jjmYMn=2zg zY?hLl8K8rUUmr5_Nhu%QV~0<#1r+XH{qof-r^@2w*{QDDAOa7|^9n;cyCep2Z(0^9 zrf%k3m`s2Rm^jsqz4ZJbM(y%OR2(pKt7@JpBxE5TNm#@5x^&v+|$*UW4MVG@_jxb@Q8W z%1%&w!}qS$95_^wOMd(Mm|U#LYRtI&V~bC{udL8R-gFota>b!)+BUC*O7cPXvcub4 z5v(E6zOC^x+?-82r-ga&w0JJW&ca`bw|72n7`16Ve89QrZLBOh(Y>d2o!s>yV=GES zF5}92H|PD{<~dZ8^;*FK!0`E~7aPLNf58RE;9bPK^82A@5%1z)azUak3Jdo3R~&m> z9vW>&LM10ac?)7@R1x9gpdQ#2-4WnnT_YXU`@Gt%4M^$2hHyF#?oW-<+7w>rtAo2? z`1`Gu7D5*cfi!!)np%a}Ct;(Cv4Ez#Q(NOgi{ta@co&9T3P+tRXRjk6{Z!qdEh2&T zg<4vTKUeAaf_=2g7NDv&-GnuVok)iUQd5=!3G0r=tv!as@|V?qy8-KF22rc3Mq#EO zOS2zQRSzz3lpfZ10V1Z>H%2dIcVAQT+g;`4pG^Y0zbKVm{1&zx=ZxOC&3+kfIVT42 z7XFkYic`!lQlZXEa;dK}p;e_(=-v8eOxfLZKVIjvlw}F5;Vj7pY+0P>0JmJ2KngcV&1eR#+4RgY)sS5l4Tr z;IA2R_D5^4P0Y!N0_Biz#O(U%ef~6{H%6=j^EA+!Q<@^g^i=GU0fs{z!((za$8B?| zvYR_bWBUK%>n(%gXuh!T0KqK;xN(QzzPKe2oZ#;6wk)m*8VHNK!$NS^#e%yN+}#}( zC(r)Bz4bm-&zJ71sgIRrHj=lgxeas%5i@nZC0D_Pi5A!6Vb}F)~!;uXRR0!sfC*m-g83>_a_@d zAy9H>nd=f`6g(F+=jUrBoxNet+^X1V11w?HY+x+}1W`-p^TJhrmQ(!6DmB=7xq4Fv zB}Htm7nSeKLSY!(rg-0~$FVbwj_MHO0bFfr4Oq?h;jz`MDipfPS&rfANv*~}?61SfIUwrOX?mi=$|Gqsj8yE_YDq%`g# zCKYzK&5;x=Lvk24zh2LX*`j8Jeoj~+s9JTor4)89Wysw3-{@+n`iz_jy5)~FMJvR+ zmf(&Fen^NN;q@(DWq3@d-}7u$!${D}p=@#qXJBux=|nAJV^?95P1M)yi!|%Q3qz(o74CZaeP`ZQ9{@ ziH@>e5>3(hFN?wI6pVAw{nAKvR3hNuK)?!ys)%AlB;RJ<5N&uTdeT&?D;ta zMkvJ{ruXKGFFr%Vw~ZLRoQtf$-{dw?L>qibFkX^xg!Car-~K!``j~Td#DY#uUe%UZ zt38nBF zW~eiWaRPUF<9B+p`DT4^s}K9Kg+O0$1`d_Ax^UcK))D;n8Q#P;#FinWW^eN1i+bOR zAl$$s795cd4x|1p8^iLcn);9(*d4z}()YoZh}|_TOZrKke=tLDFn$C(e~!d8(E6b_ z9ka7mcM_PiK@jybPc3k^!Pw)O?L%F9b!)xEW!ufK-RUkBy@GpW2li}0Q=MpCIuwGz0-Wt^ zx@jk02Ic@-kN7Kp{bCbx7&M?zwj*VjG~{Z7(%b3{6NYD(PNikus5b-Y3oAFohMm4d zDrFg}0P}N?Wk8SH!5#ToNoKz|Etd*iqcHbAh6WE@Qu+n8NK1-KfTR<*%dAhwkadLH zsG1{Sbh5fJH*@TcJ*KBBX-3&t#lV45TZO z<5+YiNQRMAT@?A(ZLf4jdCD#Ya~a`elqgIuPyec7=w@c9jB+R_ozUl0tfDX=+#SQl z2Yc%y1AAkuXzK5QX{}IOqvHL-IFn53%Rmm0)PWtS=#H=>Zpo-PbJ;$`b=S zcASW{4xv@PgAtah%|S90 z{r)?-E?X8LC{C;G1a=`>&2UE5eV!f~3@wFNd|l zG0>o(&gM#hl(;Tf#_kl2&sFQ4-(HROa!K!Ax-y^;&nS?14m9J>5HQxN20I&$K0fMw zZsFlxW(QEKbKH-dWkW^o?FYo@1}8As#kMpmcwOh4~=)B1AC zyAs%kMXG|CdIar~ppxN^6sc`tTz`t|JG_??|I{oU1cqmO0z;~a zTGxHrS1*!0m|=@K1nPpUfJ)CD@At*b_K=f0}|8PNlc82LbahNe^fKaO+NIjH`a00 zox_UzCI#S<7hyoP4dWnKh7oDdyXDy(+WtEB~O9KdHrL@wu~#7S-CCv2TF8 z*Ubp1U3Nw;&{T|Cch0nAd*KF zoZogT+hb#tjQ$O>`^$=#hj36c%QUIfw}3upK_;tE-S;jZ#F4xmTK9E4mle0p{zC{{ z7o5v%vX5{91r!R#vn<8}liaBUFG1vc|0H``Kdv@cj1`ILknjW7?cCWvCKww! zx$fxsG=EoT;+TwA0#6?g*WWr%`}9vg5Bj7xgXu2^KdzOp9$CMlmN;-#UxQtI>>T|^ zWh}N0j7hVnb#%t*ZL&1A*&kA6Am3d4NBLt~&s8bi+#Y}M^uq0QCA`6l$tRam^RpGA zt>Z=FnYGKGT>vvuYn);+FOIru=>2=B&gl*{-J$z0=T$kqIG5%U-UXzOjb0x*{#h5PJO}r?`PH8NB0a%& zTpLiMp01N{U;v)hi9_XcYx{1aHFtD_|7Ky>scd2^^}d> zBA4iIww~7d7RhB&oJtPEZRmTb@G^SUsk; zio=qH4;{_^Y*FJ#yoQGe4obUOSvM|;@+REsWc(h8S99`wP@wrlL&rTfas;a*b{a{Z zJ+sQMzOTq~cA+U-og6uuFq{g7wnPOOW>pNvNZ4e#i2$9%>`E#AANy$d&>`M(t+1*4NC`F_J%S1&y*?ur9j3>BJk^vnfHn=I%Zqdy|k!E zcNjgdG8Bp_kI?v9V~>8B)yBT|*DM~(fdR>I>?NLZy82{O*1kPjrF1(#2^3F@Cp)~( zM~@Vdvc5WrD83jO=3`et9sz)_v`4lgqB(qak}>(K&NX39)l}UpVfHZ7Wb<@f`S=yC+(!1Tq7R9LRMV{?Hu7A_OQzZnppWXdjzP?Gl z3m@G`P5LQN%Ic`qE?-?&%=f{v&0xKpSv|r{%{@!qJNlSrcI&{xk`@KTD*{rgfL}Hq z$~JrNWaLPX`cV2zrr^TxMlqoC{O5BP_+(NYba#SB1g!{sHS)cnSwQX?S!*~fny&rR zCO@qR87+Y5*XOAP>_%iL5_Ltf4zx~}w5&I>Nc{nr7+Wiwt;i*{omNd8u)UT}z-Mvk zZI65IIz1V1#iQXe+n=KQCg5L?Yw9}afyPJ>PVZo~k&!roN)JpcoMUk&OJc|FGP0p8 z%dEe5jifA)QtWD(Iv z*KP;lz}2^)Q;`8Dnuv$aUBN^9#EA<<1bQe8=sL*?I@|dxKCz~3*?Byu{Yv}iLT)lQ zX+yKj1^<+a&{TfM&#t!t2^L;?a9b>W6S5(fVdOD(xaN9>~Mqhxt}f6b~22I zp#FPk$K({4RTh_@w21tG(9Cty%nQ5=evl zJd2ps?ncCC<29vJ;U7i$OhI8XbC>FM3nO_O_WB#H)P7Ndo$kG%hN^$r4)J9A7a%*n z*aM4McKvxz(Fhs45V4ny+wl4u&F*+n5`yl%t|sF9)(OF3+4Pb9ri=Wd>PBw#bGr&t zT#50;fXF{nu>)fJ9TWrEn9kc*KMAH4+3@hIf)N|>(-ug`q>3$gM{_*0nXcC5PKav+ z#xAKePcupUUHdghf{LP~vaVK`M$`nd(3|g+1Dbu!F1>DKF<)9qoONd~~8Rv@=m@W|2Hv&UWB>=^SoQr>=={yFy=S;~<>#KeQijKF9lZ2rc_~i-p zA@enQMTq=!WsY$oPe&uX`0ol9HkjG59!MteT|fl&zTshfU!8oS^?J=_t}_hn0rd-H zRBh_bbtbtN)Ztg+47i5dn=rv+pkY<2dj#RFA|J;2OoB3Vue$58dU_@Izt%dP@ocJK z_}Ys3K8Iuw)<=|eaQLoR<7ycg7mT@ZN>UH`^BRJuSDTYoI<37vwGx9J0tK<8GDYNf zd&9=qg`#m_CK9*#?icvd+S+&k69#`9FLE1l=Of=4+o{h6ONvbeKYmy7`pG-xXLsXQ z;lhHZADo+?a{oS1a<)UR(t)wxc6fIvGY4#B8BM?!*t(0Pxg_ID0CACoJ9~f6!i%|w zQ`?2|t0lDwot=6*_v(RzuZ&dS$w*Q09DLrxdLQMn0GZud`5cCCCs8(gGUu_LcDcye z3(=y^C+b!EZ70zkp7TBQVE_5y&z&v2;{%hjzzLGQhDieBc#^%qgU~s8@z%=xf4e#V+^|SE> zNb&sqMYxt%ddrq?>oOPOJQ@?$CMkRuuj?|OLo8pGgr?uzrx<8b)!JE8Kzht_6M{e4 z&;#8I0*ug~n;yQPhDJ88*(SyC{pY-(=ciIAg7S~eR|~`(;ALok*6L0Sdrb8J=TAs_ zxwW_H!q3nMN!VAszx^2NCy1(eNR#B~>M9~#6oe`>z$PMUBUsl%({sb=`@YnOYDIqc zU-H{aP)iGs#{Zq5{r{=j^Z!isA^b^vz+tiJq=jT6RCbe%*uN#5C?iI{? z{wO~#T>QIQ`G*Ef33|AD{`Y!T{l{Rp$Z?nEHEcNdPt_i;Tbal==HTCZDyLSb01u1| zO7U+~NkIvKJ$%W@K|!^t=-XkdCX1{f0zl$^2Jjp}qe!Z+lRdcXAZ({TluKYPv0oPa zX+VYjF|FKFUPD4`V0)O0LEmqDEm1|`HBnYE$6x(7bVl&r0WCukbmw{=whR5O5-H@% zqtqC1j5coM2f?IkdgV;84D7*LvVz{L43|PrqaW;O$k}wDMj^U6d{EHzbQ+Y3w!*xP zCq^dsAf9N)Qkayz=+Mb72B5<}z9TgFm~Q16dyqB2IKF#kJ8?GaHk^Y7LeIr3@Z&jz z1@u8^a6wfK1`w+Y(_NXFQlFNcso}JGXav_R<*SrRIJxY& z7Ik-gM>bQkch&ar2NhjAz=G23Uh%9pTE?Y-ycDQWwVj1%zEBh}`*Cf>B8k;py zS@(Q=1*RZtaeEY)a+nl&I6&yK;Xp4$^e@p6$RGYdk14cZD1Iv+c`g_>KerF{yn%IB6tU1qu*c>C56CsO{UD z{ViqR=|-i@vDMM*w<7Mw<@s4m2MMO`etoA3RbjoFeqqn}P9LFr3-Y4P1fIX7f8O=C ze#2W59$hY$mOTeBiZ`@FhfbIP2r z{}I?Vce)??kb?+RMj~?tr;efSKyP(UdvYsj_XN|J^D3LUH0gIoi@Kx6wM)a6_l1Cta%6UB^V28Eo3P3k&$UM*RlxM$@ zSdC@*U51$3yKyjkMud#<_GiUZz%}>Jnwu3GN)HEGBX^#6I4Q;c-nM$d`*YzJ71?RD zmSp1lQli(q-TeMe!}uT;_gM|`ef2=p#mf_|+QA>#xi_^%*KZ6@^zmmq&dkvdq5+;L z>|(K}Q8t0qK>P(4NPbangkA=!P*<;7dGj^-=GM@G zX)DIwUYhgT1`pnLjVbe3M(;-V7H$2f==i$eA!F&^fUnWk7tQS6#?k>!bEYQ*bv&3yS0tlJLK zrCsIe?HDTT33PMp7a?Q7E{L4lBcTl0Yx5cKxgnM#9mdUyh8K0nIgpH<9D=l5(`2XL zuHBF;JWlOsEJUpBryJ)|+cpe--_PjY_$BXF72tbPR`|ioSGEJXiA`MT>B{cQ4pBio29*aeGstaPc+2F*d2AO=dU=+%$33b$^FMhn;fgv0+-;jitYm$>@96)U`-wZI zAq!t`Q>fLZf}tXy#l>aR@<-}sRmYz4#_yY)Px;i{4pVLARNZzTjzs-rAzB|N-BcHM z0K(=*{;frAp*diB+a$E3C2DSducOOPV7#@n0y=&NCr*{%vp(;+f*>x$@H@3%?nlSU zc^%}1P)^Rl8fO7N%9PZde6?jU$$*yx(J{GZM7HmHd&KjVmJ3qsjFUz)U3Z%}^?L7m zm%n&ZO2({ZVrWZd-DD|}_M=aC=^6!ojP$v*u((OgUmHBVPLzlK+#!~=TfHSNwvxv> zabwK-^}l|==`GT1?f2iOGeFJ3@tr4W+CkD=xaR%lML)HbkI;YNcZiNS7&k-3=+$2W z^;(^Va_QdOne+edUly1AF&<^Z6$ubFS40Ji{)kt$!@exAnSG`>N{R`d^ybfQj{p1P z!zg#^>Xlq>7>n&+!?69bvt8L3jZa3mi82z(w>URvqrZv7__xztO!zwvyNE42116T? zwTFH#xf>!yOg+Nj$&KF>8M)+1_e3a0f7t$t>l9B#nzkn43MHIrmQEXz)TMcK;@ImM z(vjOp|HY=|Nw72{Z&{+;R4G9E$RCf#y(5(D%c;N+#`4W&L1P^LDe=;P5eetMHCLASo!=af;S1DTuZd+7KjJt@zA1Vg*?_sEK2* zb96TlXMm}gVuiEkCX9s5*6bnu`+9QVz(i4dsq$5x?iIs5aEQ$@2kVC|adc13qlq2Z->(}lGJi%+1+&M|A-(7O={Wi7^@@`1i8gVZ{EM(V6vIpCV z@4+y$_k5{!l}F%_Z=#bkGK~GdQLjY*eVh3#e$Hfh#&HqKzNcV6eyzO7n4I*Y@)4ju zLdaO*dB3mOt_sQ#Rm4T>iv2hQ^Ivd%ga_2- zktkk2*%h`Shyts2V2A9a~8gy;ES zI&(CKjbHCHEe^!npDj3TT_!#|847(F_9oNuB&Zp~%-)I)+378QMfX$0gb?yYqx^TE zGUDM~jH(~x)GJ1!dc^4f&xRdq=Cg?6LJh@FY%3yJFS8n7G~?)NFgRZ49>b#XywnEo-YDVPFmHRJqgtY5g!g7n7=q`;eQ#s@QN9Dlw_`$F z&0|p|Sl$bN-~h#)8H?F)NxF`dKb}i#VY`uG%)x{~Drni^#q9Q-yM5p4W)<&UaQeEKj>x z0-E1KyOM~dOSgnxMKzqwKiP3es;MxBVa+FG7@s`)GVegv<4)jARbT^|n-kTaWtX8w zI{1hKflR*O*uzLVY$8H3Ito)Atjat*Iw_v1k_J{x-cXYVs{M?hT@IZ_o7NvbC3Lt$ zyUrHXGn+A2v3gvEm)%@euDEvbkyCz>-0GHE`*)qbQ~aJ1PMs`ve_>m1L%LSeVdzAEOW3t+(^@pvllUh#0(vK0N*6XTEL{qJ}U#9hlc@-hNE6n8#xH~) z5)}^0jYPuVqW@01ir?^H7RYQvvnJbT_-Zb0{COHEBy3pwgA{eT z%_1SY22uo5?~!#+n(t)D;A`=@XEIw8ig~<6hlc4A`m6StowICHe;<}k@6BC@GlJOF zC}2=Z6Hs$CIfY>`K3ConGvX*m)7y{M4}4m=spJ;enGKzJ^>+~+RoPsi+Z{_j_YW`- zKjqu@4|v{V-1TL__XP3mWf2wN>ah-;_^tKI(Xf3W+gqm+TCHv~<@dQ;B9&kLxp6k4KozwHc zwa=YQwv==$Ds7pkB5eO7z}~1U??BKfXXT8tHg}GZVNH){V&L)1^m@kUl$d|{{-(#c z;IZB^LDKPmOmvsa4xr>v6rD}c>b41eDk3&R3T~0l>FX(nC`_Zd7&a-XuGSApCig*1 zc@3`@Ml$GSS(LwGZW*je+(Vrt9z5P=n1*PVV312sF#wNlTHHfW3&jgJz*Bm7O}RPL2r*31)WJdAinDsflvwoe;n zPUOYXG$_9AeyU^~!pI*J^7VMS_EyVilV-Zsk2)w{)u~qweFlk)ksu#T<^w+bVU*q+ z<1XhzhV*66iQHrL`-_;JyE)Ie)_`k~6eru-;^9@+yH?AE4H-J59F}@K#yz7U8%mO#?NWKg&Fj z*=p)NxC%oC#0P|Gno@$YQx(m~KJ8!I#c@;&sA+Y}i97bM7+|=ZkEFvR8j&75zGrmd zhNd`{%NGdd=SDb?iW={3;MtjE#zpXs@U~q88$4L$Gi}Jwx)RpnPxAPo{`vvuvy*|_ z0`ddv!cAQ>X3p}+UArSUBuB^MrpuT}%Cg_MUDDD*e8a_#6roGE|4#kKyt3{P6^hex%oL?2#J@#q&e7U@5#V66migj1(qk@e` z$Kcak&YceuF9pd&U{T33&)2^J2)xZu%a`=d_l*Mg4kMQf>Z>+L%{E11yEm)TvwDT3{qr4 z?01Pd_eLu+mPW2uLeHI$V;#k{xp$@(cY5%-tA+6M^m+$K4x{Iwosh+XKX&qGX=^!c zBd0d7O;f)5Vfd)Mh#PJeG+g^UL$em=T;|tCdK6BVEuc3($2(1bZ?Wv^s}KAW>=dqo zJ)TOFP5Hz{s1i<5Y}40_+iVwcJ7tU$>C$AWk^fs|xy-t0ddRJVOaA?MxjfH^P-+9? z5r~k4>v`%>vsWG-fXS`IjvC^y={p^WjgFlROuq|T zKy+utjDx;gjE>HNku|ADZXxtF-r^)>of_d*8mq;qz%pnw_e~3^AV1INm&Tpms>MiT zIMl<7<0*&l3j|AUH_!9_Hwb5or-3}fvzw5dd)A@G9${&eg~p`u`>9njakXf#6M1P% zVgcRY^i`2aF_295i{7!;&d`5m%$q*$7iGOQkqI&+@TrQ2$0_HEe9Rxd_~4AcN-X!= zHCwD8PU2laBXV1c$HmJ0xSC4hx=n~*f1eFbaqom@-Y!-@NyCMtccu+rUG^O7xE?$q z@$nUiph`^@;l?gQ#dVo?y^8+@!Eo%Ij(Cy5%7c(n-`lX&co{Eab%w0?YB?|+ zh=$Hn%3AGC?Ud4wz1H%QTBWN27QyC84#`)6cx%{P*OnqC~`GHL9gB7&7zs*!+ zTls2Nm0yK@ga*`n-M*6G@lTa`2;w1)8gje;Pd0lxUYT1yO#AcX*@%M`!Jp#na-(E| z%9pxJ(hV+ZUIA$>(p5rFXz2vP+@c{sTQr3RDe()04{!QQFQ1KeB_(38@ZQA($|~fa zf#jQ0e+60INn!}3GBz%R)s>eYUh&6Zd^cp!q)d2_!plv%`BnJRHuH~(kB;z!!s5k3 zuAdW{_{qR(@w!GvC*xEK!s?X|6D&WN_4QP36X3#t-?yS>TF&G@i!i}LfhLbpO47&} z%)L&E+4a}k;b_U#2Q_SORn=ziZJ!x~X0zQN-)_n%8fxjzDmhi*-<5gm{M-+gk)9qw z=33oPm}k;2sW{w}g7}`~W&AgYdD;jG!$2ZDW8dpTM|l-FdE%ddi{#TrfBR)7N_%ky z7SM-9#LJ<|!uOKN5Of{mGFjeRLC#40^3by;4l9NNd6oN=7&!yW+q{ljLyivx_1C^T z>)B?<;NE6VX=g9v@AYm3-^Ljw%Es|bs|>y8#m9#-iKA(}_A}nS@?9Xhp;S<-0r@kS zZFGi^TZ&vel|K%{?P*}Vu_A2atfw{@(89oc0t!#n+H}_G#Z2X>$1rvG0R&x7J!639_-5)>H{r2?6#jSrjPL%gt#wCdgopmS54c76f*U zz#9tgsHnMd5l`EyERccD>C`IQgwx(O*M#9Chs()W^2K%2gyHv*s>OWfy}3M_7F5C% z7vi&|eEX?H2?j8TBOQ6g`6a*Jv(8@!tmC(UH|dZdL#-Xx!7}K>KD@H#r@~_)Ixwh(D1g6P8QT`k=4pj8<*r>$?NmW{NN1r2W2Id?)-|ny-uN)4pgQ}L= z*z|9{X`0=8o7Y6=>h_H?UiBN4e&bs7C3_YwAI{A5--Q&?z*{Jc0r zs*12n0bb8X0M_z;LcXx3kQuf&MD5OHAdJ%8TLu&xHvzN+(E{#GZ{g|N<61m4Bw~wW zQ|B6RPlg{CE>ql)&XYGRme@S zjxazD?)cQ&2QOZAe!;vPIt0}$BRU;D&yiBTV>w{uyh{_kNMUcv`oV3;2R?1CqVA1# zAaw$;SY)xa{%1r&EA4a^IkDRwYpX9&&nmVkH4!xRLtwzTBC=_k~&P_lK z*z%=QaHVcSvqWsdXKqE1WymwttWHIg(F7=&{aWT$OVJ&RlV=oO&r2AMT?KOHEZ=YI zEoYY`ricC zdKp_8a&K~97c+8{M$B*74Pm9D5dN zFpvHeu?!9!w`W$;!fI*_P<=i6e1=T#2oASO6kfn+IbsMVTQWOKp#F-I-=0Gl%ut$g z=OJWZL`VQTIxogk!~M!YJOB-E2&kE7XN(|b!|@OmRPY5~;0pwEqnMS;c9gLjcWkr8 zS|&1oquO+hKjyCqJ_cdm_I09KD-RW14#ERVqi(#%XYVr-`bdsL2F8kA-D7XIlE>dv z8+GC$RiJT;9>>EKIR80ls7KiONB`c5oE7}+ymlaWxzai*9>!#E!T_FWQFG#jn{!;+ zYD_W=egRVm#YE96iKX5qN{}}D@JzK=bBwR$GdClfZmYB?{LWKb6wnPc=b(`NTr{0% zV+28eDNMA{f}k%4c)0NBtYCGtyyOXB?Y!{^TAC9$tez{6wO#k|N6i%BO)`&@p<3Z< zGeE#HM>AJtmX_67inP%;Mrz7Fr^1X6%QI#H&o7#X+lm+4>8zZ17{3hGVX^ zmTlU0b*8qJ7brUzsGxUzR=5%&yjC2so>XpI+F>Avr7x|R4W!(AQ-lb=Z)$6eprasA z+Z&;PAv>0|bSH~s0S^N*@;xp>`GiTNrwk*O>{jk!eJ`f(MA4Xz);V3*Zj^OSUc~3d zGQjiC+Pn^*5~6tPGO(#nI4jz3`?o&?)L%K`TcdP6j}O%6a~ovMSwuKwUuigTWGH4c zh^PF%Y^nG9n3>laG+90cMSIr##1(y?*`G_?VqGWD>tc`H-Xf&L%LvHJ`q`4#l0Y2q>}yAyCy}YiQjt~WQJ6+cJ91zShLdh6{s z{E#<-8LtOBlZ~bhHjbv;_RNXoCBcosz+LUifGbiaBwCY4aoCX=lX_Z}eV;15^NtlK z1xlZ!zZjrSom}61Zh93LhXu2O1s9gm6RN94dCKl|STi*-S|x?{-OTFV3MI-GBO|xw z%~HMn=ZNe@Gui3C!Kr7H>o|fBhvht8Cs=jAB&g_JO6iIC4@hKu9^#p(Z3%EnMoR%$ zhZFdVw(G&~vKTL?7JU%Oz&!}@F|7^$en&$Lo7LmJF@1Y<$)~H%>0|(l(;)Hi@tv`6#H|AO7;crqeJf0Lo^Gut zA6DA>)8EUJdy^!7uU4Lsvm$Oy{tBgUidRN9A`ta_? zxN6RZd@f|`SERQs!STNHx4j{s0ai7C_&zon$g^vxAaY@UHd0CMq8rIfX^F2S6pexC zZ$^#_#z0)my@jLDl8tD2dcSr8XPBN&3ZAMLu$>DDiFwi;&V z8jxHW7*(~l6&nisFmLcru40~$%gQ2jDzu9&G+U(59l(%F7N75!uA2Ni*J{W!Tm;x` zIP>xu|F!B^W<=lcX)fP4ch&`P^0Tk|^T(;y9E+{kxRKv#0sF5#9vaOhHCVp2u#Ft#xM}-IoSW9pfPB z383P+;VQXP)}ZY1s;y*w&AF%X^+SQ#DRiHR&-_4rl9O~mISt!HpfURhV?lVKx$Yiw z`FZzA8^%4><;iXtDMsWv&G0CCz$vq)L?A_bS;hpnpmZX!vG^~D!(CjD*pU(c;nzL? zB#W2q(Lsmm{QJ(@e0rUUSInNKo5~;QsQ`=<$~i&(-ap)1xFQPOL1waidYP#zs}p2k zPV(}owiX{@hev++D)A%&M+L=3C_<@PKmM4hbU^R_ih)yag zDy#CyRZ-~+pB&c??pF@x(~plq4nd0FFQL?@`_4SR5eCr3dvcF}@I|Y(n4~;(V+h2$jVDS1p zCN!N-K$jS-XQjSOJ-Cue5=gc8SDlisNZO{~Aq)A%+9*Sj834>}_RRzv7M>>edK zK&J{?`MW9qQB5?KC{VML>r97`vtH>PV_WQLYjfL<&Y4!lql}~gDCz@}ZuZyK^yZpRA9bbn&HK^nm4F|0yQ;dZ%pIla~sGDW|B#i&qONxj;MFFmsRK zhgIa@3}MP1=JkfHB^eg|Ax4J4Pvy<8-<^Ng>7v(p6}CI9ZBIy zeb^$HgC|ezP5-LlcabI0>&?4z{&YoM>51@{;>dIXiWMhDr0FLETFiXYe5e(;ExojG z>C@O6>@+a!jNp3)%lt5U(n6hIXCL?*3*rb<|6y~N6ypg!82w!16})ymEEas4Y~vc) zq1V-0@v(YBxr;R@2XjpUVw|ie5My;2l1O_swyeQz87?8!!io_(S}_n|yU_C4AsI;f_2*LZ(HZ5( zHviaw*F|PdJIfk`sMNeiW=oU(m4hz!$%TKKTcdAbNQ+_}t&veb@$RHQXq|pyp`97k zjQyfHH|nXic=2hKX1@)im{$#2?&=J1%ktGZzWDTl=z*fs9!Mt3KN>A%ZIujA_oA+M^AKtBktx_Z_H?`n`Pw_CCD;s2!sqE8 z5|lH(#*&(R)3p7k*~6X85*--@GfeGC$COrpa)bq}=ZDAzCS6bM&%s91F~L|ZXAAJO zAisw4v($PRNtN$MnA*37vv1Lm{I)CJr30JEyN27kSlyV%YfQL~!)zS*#b82Zh4VS1 z2bCn1--7bMWrfo~6n;cjbfd{6#(V06nesg$(oh~|hJcgIL{Z9mNRHx{?}Np_XD(UM z6937w-Yn=8UW^Epa`LB!g z?&JH;@81n@&mGXxy6v^BseGzpOePt(u&rw(VH3-0N=7s7yQEcd9lf|wrJ$0v&Jn8t zR&tjM5!0?aQIfhIdcDu~zllD$GEs6Gnr#GRwJ@ip$qIGOOtK=PZDq+bvK|vMA2|LF zFi#5(xR3jsR_u*J*#3RqsQ9@Z$(U3+e6P zcSq5gWAt`TpODIF5B)=^oU4Mr|*i8G3ndW8)RZH zs<&@8bQMuVQY5-}EI8ZINOaL=s}m!bzOLw`bNbPM-&3nZ(T*?W*Ms~qEH#T9S#@+$ zw$n%L1IrkhI)vD=;p}62+Q>-B7tSW!L>Pe^=0<|HWlT&Y@_nGVe@d$SGOM z%ZuQ1+#JU8Rzwm_-42A6qG1zU+OiYJ19%ekQtY8PoYTET^pjm0C@b=FIN-3}uCIz_ zOXFjVWYf+`B$uWWdev5wbF}3IBTH=LYTnuSuD^(4L0n zEC8FMnoJnGc(p&~cj&0#^FAX({`}DwT{*FDI4JM05BT?PQkOmFOa$TZ&dPR?crTiN zYH0<7TCjjRejh<^-tCi7FdqAyjJuB3bJ#?aR{>u}Ipe)Zx6_1zs|rf9wY7Uw?~R4| z@Zbz{H#}kF?i(qYCRkfrq@CM{qCGrL$zVtqn)0m%x@dcP!@C z+P9(6|M+;eoK{uzVj#4VdgDcjI&C?M2s)i=beRW%_s%!ArlS2@Bd&$z@BIYwr9W3y zn{$wjE54@BX^-qj_+^Dz=(RQ}wF|2(8H`jWKyUuSd2Wf>>In^)b{7|E7RGiBR_`IX zjD>;Rf&Nc?KNepHZQxym42<6*GXh`Z>?C5D=gU<4kCjpAk;@`XYoc!nJrp0h4h@hu zyRLXTfgwnZKlF%IQIZ~2$c{e<``v#{1A4b8jY`q&%)#*mtzfGiS@W4$tKf8U2bZK=vwg=EN5M^RvUQ|OSoy8QH;g`D^ac*=04tp6`M_4USV>Vbo`i|&oruJck z(i){OeC$>eMba?CbRzq;Kz9U1y%Ghc(kGP{i#|X)` zUPJB1h!FJhG^R>QbGX3Zl(%p@M^6mBgDmFbnh+x*0I8C)My2WmZ_NAQX1J-w=<(i& zLZesk3Nu{fVI}qE4F6~A$|M^@w}*6r+#IQW4g9|CqMzwdjYZb>irtNcfA`qnylz`) zpX)re$aNL3g0`O@zi@<}jfG$T&|tW3YdjX;J-3++d?#zh1*OYHX%vG;OfK~zlmlnl z3A^=xcs5znRrZ%LSovyhrQri(!uW(hF&ZLw)>m?A)xs`w5U|)*7B5?j zy0I@)P!efz&e#dZ$84U(p39gGqYptq77H#JoM>WSQJTHJ8Be%U?lPD z{FA9Fi3QAwneIzUHzuq=Ta&((3L_?}X1Nm#QNo^S>+y+fEd3(6FW$0ZSXM)Sym$Pm zbvoUN&ra+4acxaf;aCCMAc1g5G#3`9#Y_^a+^40p^jo#57kb3=UP`CqF*iZ`W)9Vh zOr|k7N{7nOAp1)r30xf##>Fpa)n|(ur@uP12k1h3Xk*+5T6$=YPX&?f)(Jc|Lb)<& zoIZsv8JY6ZjQuA{ey6+tV+o7-x3ABg0I8K^7s1wK^*C+;rJ;U68U>o*aknv?r6MGu z$uSTaP$N8D8cX94so`m%D(l6`6DHsFTKw{7$Lh=C_Lq1FaH5FU_e3k!noKtN=9v7W>{(;V7L(Sv>K~l^_)=J~Kf@O=<3y3>pu%9>8RYjzorc5> zt6NW}V+Sk7e-qP6FtHa`2`!vg=;e0B!NtUipSP^#Hz!}fe_6xnn8P@VoE{=xa1Yp% zyHLyt%W4X}j`Hoo0Gy?n4*(4Qfxv6;>xyf;O{7UUbZkxKeN#nEM#nDWMQqCyP3g9X z@L#d!J>lpj$n89=p0LXhjkJ;QvmG#SK!%R)zb(%nE`X80Ek>Jg1i>=D8aex6d$>66 zEgs7I`;|U#BEU+gl@}+D|8zajHgHUE+=cdDA~y(wM~n@C$j-oW9Eq1gk~f8BF^A+W z_jdt3KtfN34E#b~vxD4vAO#Ue-Qy76naXQJ+5kIo+%dn`r_XwkJD_}utj&I$I)w2S z29JZtmx@1(AEb&{W-1EvtLeAICLW~LxwGdOIk|=X2!4%MYEBO>Ikj^ioSUTOLz6s7xWiF-^hT`>1{ zeq?*iYja42;yTa}f^L`*W|I}ZLI_W`*D18o4*!Lia)n>YVYvPLme(Owd1AqJs=4#L z>P>S(@E?k6SNVnhW4M$ME<4aJpPMkH&M7#JlU zlSq||=p8^t>vB;w-00g7dsi@QDTTapD{Al&%D-y0j~pJ}5O;VXApHfr``)#+pikKK zlj`?5XZ-40#|))H{WdDg z9ym716W(w}l>Q$iA~C^DXnb)E@V_$8n0pMTihq7Q-~Hj`rg%{~A(NqUs$~#{3RSzl zu({cvT9eLEfMdD8VnVJX5G}kr1Ann$*-jVc#{3F6MmXO=;67SAP zUVQF$voE0ha)}gH-pNBS+h>z~nU#B!1O!%CX-fxPQoQsgelPbK!)6^o{f)twuK#_N z7s!c6LWZWqTN%W~SLuzU^(gJG1ZO!VM~G+?)RV=@K%8oxCq$@| zW1*2J0+htJV=sLEWb8kl>CGa^dduIha$|Y7+%ns@tAYw@@lp2TjCA_KR+(HGkoR$2 zQ1?03;&xbZLe=d+_)}5}-cof4AADr{9cXbUZ(zgZKk~#;iY?)*X z9%aaiUL{4+X&ImRlg_~Jf8MAj7hbQQe_)m)_H|u+Ge}JIAwBAmCDgVb-I$ZY-Htf= zT4Mc71zl8M**tg4IB{`iOosX^vzD&qqzI)sc~GqwQ@Yg8@y;(;0;SO zrNlbC>VN*qwG~(GAoFk)n26;1Oh^MHBv9Q%DYqw4oVwN?l4#|=DMxOZl_}iU!#mHP zB{*YifoKFL8sZ-RbD(0j>eyPYA7-{OE?3W|a zz8*8KMl_LYY02Uw3?1+&2JpM1aZ%|p&Rm_V5@oAV`2`w6Epi%NNWh29b(Z~1*840+VeDbPc{UV+!*fG;LS`kq^UnhEd?d6JKa z;-oTZ0LEb|1K1lL_HiRjqpiy>hL0(uWla`wt@9Ga$pg59ZhW8pDxb|>N^kMR9d>+x z85s(A4#HxE6eSoX_-N?+q!Ar+54)Xhn__meYt0)ao-%?b8#3iqc0JEAy(@-E>-pg1 zOFQC;tzDZvHko2KUo)u=*{k=alHD|?=s)jdkKG^vTG(3Z6}|K$)K&oQv#FpK^dSze z!~21C8G(_T&MyuZP%>TaKsJcBL;2Bz@EmAW@+mxd^T3%uFtJjOvGE@4KU*RAjpjuN zV*v=S-7j)<2Mw{ue)0jGNDR3qpQX7>v5yU6TXX3pv>%d&LIAhhA?`R7>~C57AyMH{ z;imTc3*2c<*0)-Xt)48v$MwYhq_Mm?)EDH?P zK#fllzwYnkM6pZ_4+>CDkI~Gy$<&T!2>2qFasnI@y6lz)YiB`R9I&x1QUy73u4?I8 zU)d~K)Mc>%4v0rz^-RDf3=rap~a}Pgc`3IEt~x+hC1TrDN}0cT;6# zXzEH=*J`36k+V=LnqR#?1p~_yttAiVr(B!YytCn!E?g;F@0*l%f}Qn5S$Dz-4k)9u z)bR5y>rz6vC@%Q~-k<%0hIK#V{GX$x{!>=)|DrG+FE`Ksx29;b&_#t^yWPJdhAHHq zmM?2zsxou(eyu3?kh0kHMKkvw`Mu-unwY{6B_C z|BtsVzn3eZBelR5{O2cG&->bv|IT)05t{eE12P|af}IV#&zB_;|Ja^)aAmvq%ju;h z;6L~on6xu3*Ozev2C9mlpX89{W;jALfzi2>k;iKd;wVd)^*ioe$1h zd){|R6?MbNTAv@*zzm1aZf|nC{obY@YA?rXe>E1ev2)_58M2ptGsYU;oB!HWxAQ%v z52hD;jC+BMK&I31q{3A^iC5k;WtRtFS|ajMGSXX&x`)H<`0h@@p$!Xpfy^Y)KBpDJ zKi=hvuEy=LXOQpQSpCbF*cu&9gqF5FmWKxA%o*E|;!E?wkEOEx z?|5{rXVMK}##QeN18A>J97vxnz%mHD7@ki;Zy}+gc!$^=4q%Xp^FC>?^zRG*rRxRr zv5oiukLLDmB(Ts|CO^>hX(^F{9nR5~<|&^Y5`~0!SNsi~=2cNRj+ocvzecJIen*4- zUSmdm%aj7d;p3Ju-oS!7xDoR?HlZ^B9M2g@39w8VkZ4=~mjrm-uHsN4*{@)d)@ITn z!qA9s@;}0{eCf!f(TKs>Tv2U~f!ouw6Rf1M-T?o3tyrM2@IRyFf33~_+)Nvx9Fg<3 z2U1OUtjp2H!3^s&-G=Q$wxFh^1D8U!R-Ee9P8QoD=0dHMIJtud8M+d5nc?6CMHz$% z-|Oy9>+8sVk3*5?ZJrX6H%mv>wg9zy*gezNe4}?9!_Pz%TbblO_#(Aa+U#EK$_H2h zW4*g)c({3vI-naSJ0cu;w=@1B2HX)0>E$Ra9>ejXG0hM@!W?rO}^}Y(E zAHtAQNO+1kjk+&}hhctjmP2THSIlDSO3z8kE)W(Xb zW3M5Uw-yU*^rErJ8j6rk@2i`HwwOD!?#}e!ur)^mtpvr+iaFEj!CJ8l9#;Oro9UWP z;g$DJ(f#e(VuH+i^r&$}L}kFHx9H$v@@KSBsV$f~rLQ$95PYOc`+9uWNl2S~;(f1eag|XUk}0jK8un z+*~+q)+awDD<0K^Nj1m){!H2@#QiJVFo+`DcC#wS3;^oS6PW+vfzQY@k zjT(=&d|JBPa{;2fOWItZi^kHRJ{s~9syMa(98%=vT!9n)+3M1ihwR90vn{HaHITmt#REb7PrKF;OV@PkWbxAnVy3$&89`?Hc*|NMuUpHDHzNoG7 z62JZg>Q#n%^#Rl%I^HB2RgfM&7wdu!M~K?Hwck<+B?O$_UPZB;hesPnuWnJ(4kiz( zCUtYe6`7yb-sef6P%!DT2o13B7saOa<*HIc+q6qJw08(AizS;oVfYS@7wo3@^>i;M zbO=lsJX2pP0VJER!hFgVM0$4#qJga?%MU2@4t9LZrMuX|oywFMQ1|N80>%AN0q51q zm!0lI58fk0Q?r;HTOdC}Huo>IoDYKcY@OifH~V{}>$c$rTiTci&fW|+ z;xIiKieQ%E?b<-PApsFtBMDm=tIHcCDA3Coc`<1%Iop*9I2YemYtxXk@62*0z{Bm^ z%rarQM1#IOa}Z&VS^yjhFHF11d1%DZk~^-}OTGtbw9Uv7NNS0inoVdPwL(3rIkexk z>sPaXf9N5y0A9a(kifh3!fynHV&j`L7tbc(t}vc^t9OJEuuhG#xx^P+PVWSf3Ljt^ zL4IU1w=ND-dM7r)z}7CSNXy5$VqLdM1#{kp?1HpmXIW8CN_lOprourHO{>MWpxxP9 zG%~7girESmoLH_7Xz%a-vc3biFn)w(fl;>Urr~hY?OguqHF#_|tC93O!+t>$KgnN2 zW6Xq+<$##1Q2q$XZOAV9MC~D_$`w{>;y3Rb(ty5=+#}AiOEodAbGqA+2&!NkD=*5l z-YXqlIoa%&oZm%lQ10U8bdauyP;%S8{0}74_up_P1ST5rja4oMLKK$z}sBmMkA(%ejEheu`nHATp!jW)1 z2iro51N2O(Bp50*F5@YuG8%j#v_ZTb1>#e>p!Vtlii|aS*-K8T*q4}NG zhyjcnLNKNaSCe>+Z@WY~;w2Uw)dgjf=jjw4k3?_xuFLlh1e8P%!5c41%C@^7Eod8F zeM^7Emzqw|V=hXZ^8Q-7#<%tZM=>D;+s%V9ReBGqnee-DQdg-~&E75DUlajNIz7=oG%P26=9mnK`g%iYT(A z-|DF7`7&$%ZTX?0Mr~a=C0WpNs%!N3(S&VR+k}y$#FzQGejv&yE}Tq`X@dyUB;_e5 zdZIhMb8m9g$OES9h!SL8<2{hMjVZbKgT-hS6qF9Yiy859xLD9;rzDzeBql1WK4f;r zIu0gYp8Y`#Cthy9MtGUahoY&%ttMEP1Z3G)5ENApPsJeQACa0WxW zE<}uuXDn{XoL)+~_c+RZMbH{wn}zZM{je@~CDve-7D905 zD=jap@sr>~9nEDW%}25H&Uy_7@iNn$yDw@r+CGKbGG3A}+kcs;EG}$i0({dY zVYY3pcWtS2PfrRp=vc{Ph1FdS?EB4p?n{FP$DMm>W#oP)Zt1H7!+&OlsgEMHzCXLQ z7~Xw~39Z4xYxDH8GXE&$!Jv%^3JP>LPr#gPHeM(A@lzP?e$Za)plPvDHn}Rba9xHVsv@$ zU1+%u90-WN?FwYE9Bl_a;nx)0JK}d<$#FWg#+-F;$@SYtr0+2It(p*|bq!hX2EGlv z?aJR6acA`X&T-OjzZid@Esm-dF9P&nc7%DK6r!_TM;_|%ZK1Ov1eB3mxtILxb8K#? zfJRBzA^B$XY>Ezm_D+tQp)%C^`=$ZY2g;B=iQC6a7g@1cD6~R;f5W`4*UUu@WF@@E z;Kf9bcM487IFUA~2d~91{7?#`SAwI3NkI|qKj(Q5$fB+zQ_uLd7AZ}6fZ<4VR|z=N-#&FiTv@0=ckA*>|h%Sw(3%yag*@OfzOJ=r&~`{qEI z{l){_(l)_Qxtqz(br;cGqi{@TDZIFHewG|0wNPC77Q#A>>@F@{Psvs?_wIvnSeN1r zsRNwl!9KT|H>VyKsESF=_ff!0STxjOAN&XQa;0U;pC}uI7!Rr(Qh|v8(b_Isj}V(# zs)daRZI3n^cCtt!fN9YmJnDpYkNRmJYQ^i$uXEqmnzr0kGT(~)IS^+m(_9H^PtBn% zG+Cn`nkv)$sk4Tk@ZIcdjiiiy2OiRYUIPf6glJ>1PXg`IZj;p^s7(KgV1Hq{Tax?f za|@q{f$!sb_=JOVFsZYhnDae2+P_rV136ze91`t0sZ^V-ASX}g1udJ#2Nu^2pzp

wz6v7kMotpYxDL_oMQGbXPpIT2{nDKR!j5+>ixU35+ z9m1?&uC^i@V^3`RyV^_j*(iPvx89#kxGzTh>T0b|N7GcT1@pgiexVq&n(~Pruoc+9 zFpRR9MI69rAsl9!@QDgC-cP#zpxUXwju_~@OHeE&I77UY2$khr#7<7Up1Coq?scTn z$n;uyu5l|N25X&d&Zgkjn? z%uCTbt!$^|#AoM|0X)g}W*IC6-G(>h>D!e1@`^FEvzN!2tF1z(?_>E_1*7igL634zMdYA;%KA4 zr7(Ma=h8-iqk3o8zn`dam2zpn#>w%0C{%-*l4~~8o&j#OE?sT4QaVJLk4{@|g4E?rb^-jwG3JEcSue(#QwJFQHv7v__#D9bR^;L1K9Ccf;V zx7T^i1Fz5UoOO<<3ExGs^SW@<>yLL^M2a#V%l%qB@@dyuB;1j7qv;CIB-QUty1<=j z=EM0oSgN&nA-h0C(nRv;LM28~Dq=bRr|&dY&o-MsH+)qQNiTg049^VE5Kt^hSS!Lb z%*4(xGKktCSKce5J zYSB63JR{}DSE1HF`V9HE#y?`)SJf_+t|PQMl|;;-s8k)Fb!jwgYG%*dbam^VIE=~H z-on`Uu6%1Q_FU1>ieKJf_yv$fL;HBhSf+6fq zVZS+?t6`9w)s=TQD5`0zf39vmm!AR$`q^B$7XKRGQzlX!D#bEiDk+FS1D90%%GRgl zja3YDs6&u z1M#|%(niZBHNwL%Vbh@m|40>S+13SuIxKt=1A85(;cd09Y&eVwM@scw_dX_h^xvBm z-h6A}L=Nxx(k!ry+5p&x=W}M&Q|WDh44nOLryqWlsl&x5m>2P{oO0ZML^zG7J}0oU zFQ=Kl{1E}cI^SB}x2K8}+VA8fRdUPxV;Y}IWgD{Op$L-hjPU;@`^OqNUE33$b88oW zw+|0bh_Bd&#s?(uOD)9WQefc-r-xn3HwtgiO@!~O4NKy7)>IUu_-<9%{X~2#IH#$- zSZMaHxS@U171zkgM#$KANGvUwJH<`)k*@4(HV2Jrj+|>;H41mjqVrO*N67vLP=kY| zh5ge|G)lb3U(OELlkv?gPG`a4!MOD8*v%`@Og?K zxN~15*v)T|@Oy9LJgv8L6zd^P0=(vcTh*RLU$sht14(7ESrXYfRj*n3sCCxQ`cY;K zOA!^k)Md12TN3&CS~&DM5>-Bh5p&~OM)d9v52+p@J&(C9YumJYq}j^%vI_q&<20g55_Y?0c`if+a}#a(rcN z5OlOb)#Igd?XU%9x*+0?$b@w-XVgc`3Q39ofQ5l<4}=oIn(#$L8TENYBcD_8{>Osv z2`|W18AMjFRon5HBJ-jg!rndhZGc|5S=TTYgaCa1gb49#TI4cL5UfS~uzEm;xuF`X z#hO6BpBn*%Eo1q_s$EbhTGFM4-2ZE7D9M!gUWnCpvqc z6NY2h(GF~U)r3*a{@d)#Fofwo>o7*eDEzY|(A52ZWttXAr6J?{k41QKMgY`&NSf)G z+n~?hd&dG0H3}JgQJ>UdQ(9OL@=S|xGN!uF!!>j;797By&3p+-Y-MMS?Q|saM3%z$ z)YcY#7QXk~=J0daZ_gbo(P(h5V-5HhlXAp!Bs+rUI9$?bB827J)6By> z&UOnK=^sew&1w2CICKg4(VvPX7=r7)u%=F(_w1Sl$pXGMy??2%M+Ou+1Z2TIc>~$A zZU-QN0E5q|(PK4Z&X$}cOv{9HkN>vAE+ZBEvZz^4i(9KDS8C<9H~1GQtx_sbx-QflYIE^x zc|NrLDb6`GW`B(qwLBrgR|D+B&C4}Gy>>jLkfg|OMspjG;+Dk7i_Ya8Cb-#@GxJ+V zSocO;qW&Po>uagH?{icF^~PVk!6w+c?KAzX%(+V*o!!^@xAqI3oz~!?@-#Q!nR^yp zuiW%aY^i^^h<$~A(ILHMTi_Q)#ug>Tz0Z*zHS+r!?R&zlH1qTK5aT|j^hQ`<(}ZyS z_6tX6{kA`Krg0zoiqCQYa1PVqa(x3PqmH(s=9XlPMj*KKUH4yD6blGK%jtmQOX>?v zJWOlaPUeBM7`Nz@yOFX8H~t9t#1BqyAJJ&n7b0s#e~jz^8#O7R$8 z;;zipDfNgg+{ocHR{>8td-?G569GE^|tWCiooAl?!Y7LKK;`&X9Xd1fTh@GG@xyOJwroVrM zCh#e}cKCkrs6UP{E}Qj6M<2a}N1$~5zGle)Rr1;~tw7O5JB?!12=uO=f}VL@h~5xa znsptTO=6fG$_3FS5IQBGsR4`TQdU98{N~DMs1SF zxLH!cjm6^4`owc>M3rJ#;t)}4=-%EF2(NI}sV96z2vzJhNB$3}6TX^!bUIumj(oRQ zB8VLjN{7R97=#(=fx<-t2ReHa+U#ABfOgkF7KnH*aPV#pv-a}f>$5*`)_eSZV+7pko#9PKt7L;ni_mdtB zp91d)8Sb-8#d!Y*%M@zb1y8zLXxgAIU$OhBJ(_CEl2F^CsM*mhg3byfj>hUSKZm#v zyf4Kh3i6|$ed-FQf{%NIi4}t$MuYi{k7b%YlRau@B@Y4Kd3v7t6p!pqiU(sX&t!x- zSmxJlOh`8>S67VL{OO00niK=SgK(|;e7+(#_x4=%quuz-g&U|J3!ew^z1D>|EQHgr zZQU6A(`K!Olk~q-Vap8^TI7#xDmN;J_;Vpk8vSMZk|8%KhKbLGT$~%75lTsJjVoJ) z?PPkzl*~RqA$C$~W1f$ao)L;gm*7#Rgd4T3(x^NnJHeC;$=RqJ=3xPDnQc%GF#}(( z#MP+0i-Psgo(Wb>Ijv;3c34Gqe;6WO72TsvU&|Kh*})S=J5NeP`}4NM8; zIL`QgxQpfJ&_fk~hiPqrE35DChI=Xv_qVRA{@*V6?(*QP_Rj{P$G7a$$Bc;WNK4k6 z5)V?zJMg@2tKi>s=A&}P_Fpsoz4MEmyW>VqVR&v;cm^9nb+1DZ7J}&5=#X=wq#Cym zIMQ+7(Iq*Vo++~839r@pb#cd*%Q=xH-G|tb%|{$9^66tzLpf-{3PCC~&yO($EB8x$ zYG{E+6I1fwTTefj3YPa~`;_4gE+q$O=cAmTXCyXTFXt-$l* z&J_m<6x%;Dg!gc)ze{rZ;ffO#o^1kHVhm2aQodSLPkkTb($&`im=>-($(;h0*a)Yz zO@`1ljgzDMJL?zgNEjghbyEnmY)dn*d=E7qRZMTBsk|gH8NGG&FdhAr;clf72)Q#> zbE}`~k&47c776NcPG^cEPcyRH`(EiJ@7+eYMu=Z9^oUU7Woh?fe0?eXg5FOrMU}k6 z*K~E@f+#*g$DU?;qc^DEK&MIYQB&1!H2OB2O;7q+!bMx|hbH6~y1FfD#}>u;_}qLv z>IV?7U!;Tv29~qW?(uYm3Dnr5+Gc6g4cAI44nBL9(eh2kUo~r2@x;7XstHI>WTFTd zZd5$yzhO;~;2MX&kVs7IuN_yJuR||$otdyvqQ$>LnVt_xNmgb|)AU&EN#n7Ow!fe} zS^l}?mh7GR$Rkl*Xo*(pPVK*bdmdfF!f*bmv&(+l3^uIvP|YGf@5AzaXJZ*bbm=Cq z)U1+XkzwiN!P}3AaRDZzGe4hSl#N?##6`2;2p!q@OlMpoIXx`Qn)EFfFAv_d2)C<` zG#}>R$NrJKIkO48P<4{*&3xMRv|*87*nI+lRWRA844b1?Is7>(G$zm^#!hq8HLB@c zh!aR*(MBNs6$*)Jt{t4)f$b-x8^!G&cZIGKMSDFiii2683NaiN6*xyu4UZd~n;3(z zfLsv%NxSOgqU^m4x$5Q>yvakXF~TXt>2)tQO`lC${<^{_zK zJ9IO#+HCXmKd2`ibk+aaS%jVK|IS;4hmVT`Nm!WN<)^c`u^p1f?`ngKG4l?)PZ6z~ z&2O6TACScfE?%@fdT94hve*buLU&T&7a#T-T^AnjI>ky8ZnJMe=d`7BBJ%0CM|( ze5R2-E*8-~I{|<_|2~FA2=+ejr@^4zXC>DQ(xL-^9gC2tCRwyAlom#8?>(ifl88*s zN^qvA;VFe^5?nrXZp}!`d5_2W`Dr9&GEPMxy$Ay}0uO5n3C+j>Ij$8l`{OpB#~j~d zZeO2t{qzpB7&0L<|0zC2N@LHwxglb!&>O{_ZBD)}t|%R9MKp&=^Hj1Zx;Ml{sjwgZ zTPSKv7me^jAqXYe`&v3wn`lObrnO`f(s_??w&WxHw=XuC9vWd{Nh3;f2iO0xKcr*& zr3}MPoWEdy^Dn>EdYe>H6FfyI1Jf9@XcFA{s=#i6f$3n>4bc8EI0 zm*u9WrZXYLGKVF>Dfu@Jg71i83+>nS%R`#xQ1H5DLMkOJc`||JmEqm=v2*~)*9zmb z(&NvKus^?);927uC*5Ki;?v#+*aXP52z*u+oGABhB^ zu4!n-peI9z>d|Hu)^5S=uT7@Cc9mW%iT~ta(a?B;@h!GSFR7d9g?KF*|9zUGpuEJ( z`)6E*a8)a?TMQ|jOZW%lgcieEQC4npi9TDT;`~>9cfO;i7fP={8dXAQNM zZZR4Fgu6nVPS*VP91a2*Kd=v8+l02_7UsC}Pr)h91uUCR_7lcl%EVIQ-||J_3BeDfM!1S+OU#32 zi!G@3SnowJJ5u;bVNh(2Cfpa)1Kmu|rB)5VA08uU2BC0%bf|;D-TfgJ9S@RD=HTL_ zuJZ!FTy3d#$FjpnI0LfzkL5{vP#Q(8(kBRBPv44|IG^d@O{>vd0@+cz?E$O_nbm7e zK6oHE?02RJ-w@?*=i)AK^#vzP2$ZIE8k;%@tkosO^Qy zx!DAcwc_gQeVru^gzzf4cl7hT&g7!N;>OZ7D+)T*C$$g$qH7gv7_+a5FZ+hCxvz@g z9`v>lUeKx*jAzBhl3_6*@{J^_Qf>Q_Qk5{Fy`*3)sSlvaOYklR?d7@K6G3*rt>%MP z@k1SBg%ocbE<^4{5}OV`AT_`numIU_SR(E*%M6c9>5jmfX$^rrZrEH0fqUqNwTNBy zklr=V9S|@iZk2v$B*>CpMZgGUv5E&>hLCO-13xLriq{|@zs{p9^U4MT*~>%C{YB{8 zv8o>L&CE5nzXXNnkqhrlYr91XddTo>du&LKKE>F@SDQIy0Q6cc=j559lNF_DLHKC+ z^ux1aV&XIdyv)VN>#?F0)tdZGdX23nSY8h{6sR$OLiYB&QHGAiQte9Wir&-8y`H2 z^>6p+a%IJZw@AQayw7%(QsvjITlJ707u4AQ_4IZ`%2@{W=Ab|~KAFb=g zkNKYGz4^f32X`n>AJbRZLK-FC^&fzHmwp*vGl8$k6kaDMl3Aj($taqEQz#X(bYH(= zwG&>@Pm#vz7Df+}XP{s-c-U#dqxOV!2JLJUPaMxGoCGIgoH8%=;%mYVKA=83XPf|SAmt1D`E6GArrQeN;vV=QLL++kO z&-PZORq@wHSQ6L3*)bMGYk5fq(}&JngLnm>sYnBJ{3zt>E7(C$U5xq0;Z`1oc(My6 zUuj{iG+j;Dv%U^CM4mGi_uYw~aot6FC=xvlPE85k{BHR*sQ5y@dSaQft4_VgqGT0j znuz{TC4>Uv+5TDn>`x~7w~lwJrJfo{hbsP8>GmOC>kyOj=KU1NvU_N7*I&`pU$2P(#6|@L|Fa za&F>bVi%MjB^=_w8qdSgaIxW_4DeHXVu*DW-B4=Es*UL7ADeZXrZe&IENKszO!t} za2=USI(m5#`_?tqS}f2Ub%uAan2j3AVFh2W8JfQ=z))FKb~cCaV7mXpR~moeLUQ8v zX@g?&qC>fZp+H6zYViIxQe=Fji6N^aE8d5~G`n1(@kO{!0=3v!N0f(A7niUCFJ!kp zbQpQEl4*VOf=5&(w>sXWfFZbWK{+7un#m}O#p@pabF>xOJQVI-jnVb&OrvDI`1$af zoQ)84$EOdVxgI!~Qe|IplPXWQsBE+`#A}7n_Ycwi6+TdBnG?s_u}%-Y|9Y!xSW;`2a-4Zf2|t z4I`}*OabR-Z&a0k1Zp!_AfQSMrqAELsn^NYkG$^Ijl@>~be;mFi%VK*Ntjr)SmMUlnEOo3Z;+Lvs)wta`-NN1#+B(ZK&|;gX*&BM_Y*h>&OhaPlo?G zN{=d#(XZN&)SA4b!R~M=y|wNh_f%|hII5GuzfwAtMO>@198fWsH259#wRu8gtyhT< zZR{LE7Kv;**eGjtKfe<9t{$W0!pCoxtEtUg;r431S2oS&ZT3+7}0sXDO_!ZK#!22sG>kH?eMK4CMlx3ID_FW`&vskRgD~9y_hMKry89>SgjfG(LR@c#k#zJccj) z!$&fQE|yiGBx49@hCBa!+c^#BW}24Y8^4_ZbXUEBfkQ!XbA!j+b3>%i51-;vFuDUK zXKbz2XVHzhT=oy26)y|I+#f7yREzFHJOAKyEqgztsZWd=cmMA2{mXBa?qgwB<6>BT zM4k0SOMguVpeRCE)xZs_??5lA3te^~lSQ?VbKI7#j)iBdmQ*&;+wOoaX$Ho385)L4 zi;%?*%T7ZpMF#C)!abw+E+)89wjf_$5?r;)#?>067Q%STw@|YI=j6*?2!uC)pZUi1 z>V$7QMaJwH0ftIeWsUvN_N#dD#dEyb2K)JHIQ`(G6+Ib3C-~cBftk-r^sBm*WXLj2u(1a=`K0NHn!!6*cn5|^mm}Ey<&;HI@&H-YXwFQ-IlJ{HkFoC=y zF*&0zjXO@#&x}59YqZnfUdw+VU}0QQ`W!qe-(b-Fs0MwRT^!G(3bYQfdmSJ?H;$qM z^^T+G*tRqKriV8U8P1*ayy-O?QLAj|L&AU$a2b^=LkR{CXgE-~rLUF}H3a>~vFIJA z7vx4OOL!fp*?FvDpjxjtxzgc zX}Ntaf>)FsvxJZtd@-c7%Mp(q; z=^`}}+HnA<(0&ck!9J{@zY4~lMynAW{p9P;q{3B8H`59f_Tp;0uT80DdhbJsgg)G? zP+yDs#3cu1pbjFCfdUYN-j>sJ^-KOYf>e5=wGrY>1dZ)6J>5)&*+L&(r^CrTDj&{F z)XYpXtTq}qWh*ZT>JOJnBF?|N_1F(jMHz}e82vE?4VUsRY3m}o?Pg~4QYsQW1y zJRBuDITW_*WO?BcWSO7O9s+x29q!WL%6H^7TOqiI<~Cs1R1tywjsW^1jG?P#^UH#d zWoHkv#ya>MD@16j%xy=Nz<9aHSjp%>5b5iFhU0eqYJMR{&~}|zhf7tNA^TqX_#vxR zIR~YgX%%f85B?3GD}r7~O5_$rFEl4g+|2cy^YK?+*HACESXnD=vydy%A|xs783Um^HTC26%?Dp*_xC37Z9i2P6Cs!Z;2+D*_tN$#hGf;uvy> zeBGo!GQolA!&k@&B#V0Zq*sL{-rw-Mh>VM17x4`-xl3QmKJODKRZwxx3YqeE zc2bNh8u9uuBwOoL68SbL@+mockAvc2t!@KeAUxK2bTxU=IHVV9w}bV|F?#x~Zauob zW5PdD%RMW%MFtX{MW?EW=D|fYS=}1Bwkvhm2y3KSHon!VdGaCDDJ|u93c_uK8}n(l z41|-6jVr8wU~`c%4F_eLIUg-U3L06uE0tc8zugvM2#&TN-+MtN~R zW4E<>>$iw97A^Of2S(`3iyz^0g=}<`&ZaC`Y*0A>lnr9;>&SBadci7*zolvwzyo027 zk0JF$K2yPPt#UlV)KZQ;jvS6DnwW8qas0Vtn{q1n=6gx1Kbg{K);r?RSTVcaMw~A* z)f_!Fd8Dc0R|^p3hC}2Ot+6a+eNAx>|RXVJ~6n9QV$F zECkf(LJTH%BbF(xaq|~nR9(V{Loa9_ohNI2GTcaRYTJfGY%%fvS6zFjFU7{wWa2f4n}%)ZpqUh?j*;9S!i zCsgLioO${jCqEOA1J;`yf|vxE+#h8K%wRS`(EVGBdo2<*pyg=eGXXjeW)}w+-SlM> z5SPYO2pu;w-Nc7K8;}yPy2Ue3H}O#IuPJHrjUnL;i|)U3{l#rk5eBSS^2RZ?sh(x|v&cxDJJ$W&syFpk~uv09Hl#8#VmHkyz! zkSd|}@O85kufRcPUmJx!W7kV4nKAj$?)X+WGxG)8VsN$i>ATBBl9 zyG)XtIl%L7eZ8N1Uux|!my{nO!&}bAH-(=DNA$xoEF0weukuCZ?g0vPDOPKb z^WyLdkhXls5}7az2=3VNjP=}kg^lf84`_8KMOb!geqP6U3<1pW*)mDwgF_^Bb2>D0 zi^pzU-?80hvJ@FF?Dk8;(H!%eQ-c-tlIUS~F7H%m7Kg5ME)}|h-S}?v{(mH3chF`B zztb#QK&iRdoRS#Lf2Ub(`M94ERfEt&<4>d&jmj3ljm}Xou#=JgyPl;BUk#Sx$C^J_ ziVL)5=b0Z4zlxqOyL6UAR@o1wYp7bh8)3-}wicE(VBV4BJEGWrSnU$W8@xRPgoYUm z4=LrhRkOO?AA{)6aseo+&m4vuKr@T0V{!$ZapHU&_igbOWm6;&7-(Z1hT12Qb}3A z`)?oeX3W_zGV0e2mOWp|Vyb|>4_`@7X44IbZ1_`uC?+;*-gkCBf4mz7!pt(I6PDd9 z!2K~o|94ZpUpn`$+F!nrz$d{w6k+}Yz&B!WKK`&C^lg)A z_m<8ao=~vZUB1sUC(FrL2s*RU(CLM(dq#$!mC1#U!U{^gy^+~c$;5qcW#&!dw@{_( zk2{J6-sf(%GJj_~WCcRls9lrc!YY+K`ZyyfWN3IzKxBPOz%xqf*pu73NlGjmEZehR zC-q(5@P5<^kGq#O965z5VFmwl>FPX6V=eHZi@G>T&*y`ilEuuY?4@v~_Rl-ks{$dx zD>LBSqKxR$bFL1X?)}-WGByWrUPMnG=S+m?%>hfUR~c*g$3&^5?FN-Pboy?xDHOc} z-%BK;;X1!UWJ}!C_hd^vYo9eASZ#T$HRGT?e>9V83mNgrUzWg}N72#pY>wRfBxK^9 zppyd*D|oiev4!f+mJ2Jp{c-fG+=_O4t=+6S6LEV(Qv8P%Fh4H*J#}G(^s3A>?bq;3 zsbJ24&Z|lLx(Y>4LLKW$*m$cdyJWSAH2Wvz6mrke_*%&-+kM^N_#BF}){L8-EA*h~A5TiS!c|O* zDEMVDoD(8H&AEby%sxFkMPY%?#i%HU&w4SHm}M zb+

vQ0ml|AZ@^j^#j9@p%p|e+Qzip_^XQ`nY8SlyLBUD$bjtfqm~m(xw4P@%q&S zK|r&h4W*W~fNWs^vS*I_(10h-apHcqvi@#s0{}grhBcx@vUW zZLV&~@toeEde>X$S#moEPQ&VYqM=x{@_8{Sbfxa1{g-48C^#S)U8JdVi?VhXSw12F zJLO0cZS~L&e>0%K?o=c~;vnsA%pg{Ou(Yl&C{gib{OGR2U+V$QF7q%~lDnXqvVnad zXT=5S((ju={>>BgX+!lc9fOx=Z-01xE^iJv31^ z55HT&3y}VGhodLzf6(<7P;mx9f^cwm_rcxW-Q8UVC%C&i1b25QVUPg9-GheUPH-9A z$xn7~ckkZ)_ujnfuI{dy>aLpaebd!l^(^t-nQ#`Id){-f$1|5JvddQ&e9lKkIP9t- z(qb~`yaDFbx^(7L%3WYS1X^$u>0LhMa^22*PVD6&BB2Ok#C(+TLiPI_>($=1%4qDh zzyKfp*Eg{f2}#M%X(OF6@j_fB^Coh`8t+!M%Z#-^Vk(wEb&+kXRq@?{r~@mZ)a~QV zfhe!YqCVS<*SN7yew;fWHw-?G>;zk^9orvi9{vN`wGbMJbIZz|)SYPIQmQHqT+|1*WCF#JT>; zr{~Hx22eD^g`WB3X#q9au9@WjXtEJe@x@?xvgn*5DOSI@JGz<9 z?P8sEx|alTuhLH71X_EjrZYY(b%9RkP?S4>+fueZD4xV~O+rHK>vPxTCH?+x-YraM zXa@z?{X{J6-nK=4YCe|uBE2u9JOMB-Z8J$mPl zxx93y?>^mw;~Lr@tV#iLee05lLQ051)+MAjZJ7n}`kGA}2n1J@09hh7@Sz>K&Twge zKSp%TyJQm>3ItOF|44l=n)@~=nPOV|R_~1yP0?%gjI_asGD$yNm^Z&q|L7a~xeaLT zydDvB<#8TA!2bHqWP)$}q;?c-sy-NBIHAQy^nzM7_9f{L_P`LdQO zX4I2-P+X+&SQ1)N-0P-RH)U_-6K&|$=NnA0ro$dYP~G)MCG#5U^|9JzyGm&U>b0fE zX}f9;dAo|K0q)Vx;RCt)5n5FFjPjaJ{rIUjLk2QP!4wUw`Ro_6r9cTGI&3Aa@ zQX>Y71WF>KLW|I!k**1g1T&f@C@VB{L^K$r(+*WC>bbV$Dwy`S;CL)9L}MHd9>>OL zlB5vKSPWtbz>u#rl#6C zSCsC0ZH?DJ($;*-cwW+jbSn0H#RIkA#gR!6af}r z5Wj~}krKkle;oUg*E$Xih53LK?_pGyB3SZDzLr$#!-5x&86!c&ln)v9!M%!UKO%w# z%_@8&c}^;KiEPUV#^Cx#h$RRdw2tM&U>s(5d?9!?HM+x5ibR9Ha4f-WY+k1R(q>1* zx=8(%5w(j^bfLLmXA$kT;Dw;jsQp?Pbvatl%sNpO7|~QYc*LS+o<8<1VEN@E1Qbuf zHCsD-97ju!**S=NY+L5QKDnC}`y-VVlsKO={N(X}TOf=Ifr1ThJ{)VHw^zzSv z8`L!ue4^Af2BxGl(XZoeOfHnSXkCmJ_!!^3mMK^cI;lLZ2B+mt0)JD)Bg=7l$Wf@g z(8(Lq+a|cizU7^+e$ru@T*m_l(zC_L!~b+nuVfQ>4J2PuG|ZTjH^V%h$yUz$)YH1poqALzeEHbhnlpu5$-9ssMjn;p=BS;E=DQdtY}BW;3(v!$Jr4S-eN(#pxh&cl~k z4v|&L%H6`s$r9+~@sGwGV93t?@iiL8h^$)9+D>*KBvt@+j(<%bzWRSM{fh(fA7lPk zK#2cclz%M#F?6$Y@o;tnu>X@-bf{6i`$dPG)LH%lwGe^aFUuPJ^sYI?f3I9NG; zOc}uTZw{=w07Eu_Aul(ZF@T*HV93GC%fiRWVGQ8p`Do?fU}5LsGX`+60}OfC**<#s z`2f8CHH7 z4qfW%6KlE;dDuiUAWCsBkkx0qw77|(uDZT`7Y2A>xN)O|D&_qUxV0tPi^lm|b8$m^ zV^Mo?af4G58uAka2pM|J*1?TOgt$?N*sPkUPtEYM+P&xani$454kD(&=&-$2oebTL zFeF=D;hST%M_7;Z>}Nwy`QpoYl#kj3sb!=z_XRZigmQO0pnk12EBV=lB4(PWOaV? z6CPgV`!I?<0z*KCA;&6PPhO6n^aTa^9%PlvVV6%!X51|Zf}T?op=_pa^Et6U9{?I~ zLtS74YDfKO`hB4C`S)_n>9kJ#P3O&Y!e@fTS+v5Sv5}KepFY*bMj5o=Xwy@8q(d5( zDEeR#4qL-Z(`k%Kx?cd^)@o(vn;zdWJ}O!%Wd*-!{Yun_|( zBq#>7;dL3*zGPX@h+?sf-fNX#CR_V;pL{PK5fFPO{w=%x*v46aLA(3- z)4chCYJPnA0ouVd)(3X?IIhQ9C3xEM(P)KOmy__k8T!;&GCK|gyTj0emNp~CkCeGY zK@#F;hjEG#CKbZBKVFd)LgY%AC~A_!3-;X;@;i)pb}wK-o~nF$gYY!J5eNnI4D<(e z6rkC|iE(1X5B~VOoDYRDGTR0uUL77hpeA{EpZY|3!!~Wt8b+nl&xYdDR$AyguhLF5 znFIfWI5#wgs6?DEL1YpA^s(2mZ$8Z0XI*ytV(@>S#%0{~X8&d)@)rAIZ~BR1$j}y;sGVSGE;Q!zr9Z zdC5*(l?{<-w-e(2@PuQa9t3eO{m3UG`p5>^IwP4U5(y?AxKV-=1@mc!JbcH2?pS<* z+GUHNi)h1evbc&;3!RX!5=!HG$_=)Lae957Zsq%S8u)-;Oz+-#Q+~JAc z(0lpPJSx)V+d}sMx!$OG=5QVvXD3L{;xy$e0!-Fzo_%eTjD~s;dXncCY0>+&QarK{ z|Hhe|+BG(n-cb2nSvGkeBpUg6jIG3*wFFN-vMclh5O#L`aTNK{~^|yPzS~iL)GR*x9JwRwxLWTt7|X;`bD-@d<&M+RkRwEMOJbVi`@`NOV2q z0xBzT&=ZyPq@U?L1^v|>=LO-ud8P>v??z%}6S*=yLtLQ2*9Q+_y$sKVWDAOBzAlqB z99rOVGF?(+qRZ}ftc8Q{b|_)q6Tt){7Q&{z?YyAZsRy{JIe7cPs}tcBqN0XDnW47F zy_+sRWf4}9#lz7*Pub_pOuaYVFiZ3&jlV^MGs*N5E%;Oa@V7oZJ5EsPtqt>+WBoE@J2HUB8_VM6(0>UE#(4Ezi|}SDc;Gc>`o|Z4!2%)@#VD( zVjj4^1-)eN;nF~ZaI=tvUNK@cFuDq%`}u7Bf?}(ocl7QGWn>vykD8#n_xdq-;1p+i zoA4QS2@fjFaR;3~5b9yaz+m`%Z_s}E8KWoE1(%Yq_xM%lgXpnrwtN+qx2}Wz0(Hm4 z1^(?GW!g&Qi_|lZYu*FXHLf{X-^j=_{cFI$J$>f`z>xLCiLLIOPvn}7e*k4-5(k3bu_glxgri|G-n$Ancn{cn)QPe8=)auq z?zP`vga_=f&&DqL1U(XFE_mqCD-E;@#`;Hvxar|)0zodAagnBv@68(X;;4u5eSxox zbQqIrJw&xbsMVNVa(NiPErd$*f59}DD+t*bhriK@$rgXXRnW3eR(TNVe|*I6^`XR% zi6>8+yo35yN0vQ}cDLXwym}A&1+J2H{_W1H^g*PDK$lULIn2H~B+AJu@A!3@yLY4FpA~zB;~zCU}-SzXRT(VACs<^d>?o zcXQZqf+qidvW#6aC!H4ar+`N)i3nwYC;UZdTU-Pbo|He#xax7wTK*LNULf)pbqr!` zo98yP`!Dg^yBP5mS>Ifco;CiGF5z0y-!_5wwvcz@IJT^{UXrM5{Aa}T23Y15xTRzZ z{9vLTL|BC%VQV)KMh|gCpdg$E(V1X;M#_j9Gtpks<3Gr+e{SBMFDI5S@87qRphT-V zub9FBL1C}(hgq(oAKtFGVLb*`)0sZg3zCNiGXcwzf{&a8NsTuX3{Wi`#R+;nM0)v4 z>_^P1qwb3yrU4vcYaRH-D4`I)e_DmKLcxx8hKY-IW^#1yW-H;ozF&<#J-q|{Vp++z z@O-QbQPmIkHXn7aheNW9cH$R(mL@Lmp8nGIeZPSnsDp2K4Hm|o z$ z+C2TvjLUCR)`zIX_)C<1hPxG2CQ@M2ArIr*O2DXCXE{y!f}BFRwc! zqe*i`y$mjdJA$s?ABYR-7hj1JZ1jT;tPf>jZf@?$g9@o8Y~qNUn>hu0oBjcc&6r1- z<4-3G$hshZkp(14Q#C*oEBvg=Fs4R_2Ksl(Wt)~W3pM=Mmt_q%#vcVja_jqaSMwat z41onfOz7h1EfQTIDO)TjTvF_JhLhp(-NjvpE7U8NtDq}jqvkqe5M5!imm;fzsbZD3 zpLWYcDdE@L+T4a(O=dM_^^Nil!@HOh*F~>I!&=E&Q@9PjBVB$y)GWAhSGQ6jt?A_G z$+AC2zV$``-Z8~Lviqa^wDqanQ-r{TW*UQNgDS&@#&2y-v1P8c?c43M?cnxmKE?Iq z_Vo72U1Ha1A42Dtw$(lvN58y&>y{ix_Eu=aTIR`2`?~MU&0j%Zn>HLe2k(o|5e%ap zlN^iAsn_yap<2-#|H@4%x6mdk*2~o^)vM(H&A82Ys++4issY*@>pz{vH2<&}E&M5J z=;C#~($u_EWbE$srtdtH@9ul}{^$7W-lzKv%f{rHu5 z#yJdF2$-|zlQfuk(U>3D44h6sO)58=SQ#(b|3GgsvRnrYyWzNMue)7e>U}F;ayxZ6 zZ~fYx=?}&1jnB;6vE(*AmH;Il1xT#xryFQ~4_@|0o|=$|sw4 zR9p$>Uwt@it%rMsWx{a&n`ivLZ=7ZzMpoQaXEuaYZ-rtm1^qJb>yPv|_f7~|ZImi9+7{E0lZ_DOpKEewHsO1C)D-`(*NbnBGHvmTFqwzjCU3 zi3*3)Hk(g=p1dp)Ek4v-YHyYG7`}Bp%io2cD7Oa}s3m0lRsd))i`7w*j^{DO514;&5UPZ=Hwz0o!j0gO}Yyn6SHA6cs1cw6Ek<7v6^7@YS%lfqD?Uh)aF2k8tbIZGD98WTeru;F#Gc>o=rzh^lS>sd?{y z1|NvT0dMdphe)z*Qx`(gFxqWyADF@jel>*CV#Li)v|We|gQh#mK1iEUtR;k-p-?X9 zg<(Iys_iJ7(cU}4Uly^zDD_dk+hI>?(Q}4q=d+)XiZODF2@FbLEc`G6V3y21SffPy z@9+<2`ocljnSC7X5Is@OK;9Lx3yO4jDJsnTNK!Z6AJOZI7vb*P50$6YVQs;~q@)ruw9c zIZ&k#1K`F5_l>VfeaY^VsW`Hc$;ZJk#I(`dP>ec{)O63tJ`0^SAV>Z;7LRQ7?UabaOF^Ez!F%i zMxlgZ4bK_L7^wlwWg`iqGM#t2=SNzHqIIPv&JD5j)fLxd4bf? zsWS}97GaK{!oyI=OMoL__%wbF1_2tcWa-wCVcKQPG*~ow6?tW%+(LFHcORYmnEaG7 znWXVXcwKo0Ki}LbX0mhwpG(+j+cim?iE^Tg?8zC1cwf4co<)8Tcw(&_xSdB#)p z??60j_v=JmL-+ego^$ux5)|A{f%lgG-&wd|!PgyG*~FmN@hZHT#Q)dl@5;)YQ`=!e zHZBZ5Zf0((kfn=8DKojqG=m}R@Q9SGbgnYS&Io80UD8FZSVXL9 zv06KoP7Sgq@=6+~Nw9H%nP$Y)OP`Q&R1h>U*HAjZY@t0TM%rf7zp@{i@3mhqlC*HK z*Ye^y!>+~+*mV}v2{5^!Xs%4B=xzn{D0vm_-%6=8?0Ep@br=!HA`fyQSCkp{ zCA{Yba$5)DAkUm+L!1I$eX3ZPalDQolaa+H9OInV#g!}&Xsc^nZc!0)#k3|!a7@*X z62GvT5w^n^mXu8xb5yi|0IQ#w6%{?xt}?FN*U?3%N%9Bu&4kECz7k&Q+yE`{@=)qi zyvx1XHrA~|_;Ki;Yaba#7b~3l)kj9)<@;Tc)+0CkQ^hev!>!4 z9T}~>AJXUFzv|pGF%F8TWv<3Ctj4(<<;}$mFIW;MZ0hNp(D3dibL1i9WjF^-KT7Cm zva5R8?~M*Gwfw9TTaxfRueEn%0|OsxHn0s%LD!YF@&CEB5! zZ5?r}!y^MF-Jfp~N$u;aS2&JQbM=Meozz667;8a8zf3TKuh&AkQ7dGuIA7tvSh~&= zeO|u!hyzW3v2Te>adLH}`OB4w)uyU~<*bx)mk5P}f{QG@*pbAlY+>$IhtVCJ%OW_{ zALhT3u3@Sl#Xm=4rm9NszJU09{@fX%sNW#-$UIvcbFbc z^P<^>Bu6aeH?)^ccFzBu*gu~0BmnF(J7DUU09ca$hCfMEito||l~sBHwX1kSj@xPn z(F=c&p`m?dqgdkD=g<#(KoXFXHRH_LtU;MF%4!%@ZX^k%&=vM-j1@hX2!!*B*uz0& zM-O$piPVFaHKIa3fp7D;#JSjZXlpic>1rg&-9h$*g|+fJw2y3DJ{i#r=7l73G`86t zQyNpnhLwbclrCeD{!IUZ(LEcbMbuG)V!ROb%}$beRV?!GY;n1X(04xav{?OPG0=I) zna#6yNNoEz^5yXJe;CGKPuU_2nzWGguU4x#qAyIXV-Sw4OcgvAPj$UXa4N|ZR|hCo zjpwg`bma~-<KS5IH-qgcEed_Qwm%+Oc?Izvc7d$qPcxOD^?b4aDG3+ooI{ zZYd_4-$;&rWXr5WzmyRozA8jLwa;W(U5thI>}{ZA3scVQSx!m)hK`#jkzI;I!9~Hj zC1d(=w*=YiigyShrl@KIJ^GP<$a-j_1*U->_nGTw6nl&}!y|P^(lA$G)J0fIjG$mx zHsS>;x8$0j7P9anM307ihZR|1oKrZRWndE}JEDD?2Mvv0*n@d+gouKnfH`x?lfW~8 zp1LZfh6I%px#YGak@)v`M`B=;6|b@E(#lVMuRP(OJ^EuHt8yy`T+6(Cn!~ZYKLJqP zFjQQmEP5TVMf?EGFQm|d0|Ff#l2fjSMT7i}zUq2WGhQm%+s!GOA*zzjr}Qd+=7@kC zki#OG048a@C>N-h$1c&6L{ z*D98RQTPY4s>Nav)BgdM{@btr3iU8dYPx^Gs1%x7LG2%CmqJiTFP1{c0+mIOP@mBO z4>7&c|Fdp1kod3e9*dklCiV|(V-nD%g#H8l)EsIQP}Cgl5nnJl6vpVt-4O_=Ez^_L zB@X@zhf=^-8wkOKWdVvI{71cy8hKsxE4 z_!Dc{m^e0n3ifkUceqZ-almG1ApHObG|gH<(+1r?VAPryt;-0R z=2;d&M7>V8bAaiT?(!dXqk)8fb@y1WbX(bf;1KhiZZ7d3n5JI!-EO5|i~TD?_t$=D zmPoD!mRGmq3-XQ+eZb@uCg0T69VmQHbkI8SFZ3jT52_oV33M#6MKEz&h_s{t6Mz2` zMm8i3<&=!A*m4AjWQQM`y9yN$5Vu{ia?i)EC#>Hs-);PsZ?O7Jq_VQVcDz=(_Pq8` zYu;;-9Vg?a%nmS9mTj=D^KKM3bBXSP)o(*ub#05Am>22GL$syOHA(8G@y?P%EJrg# zopwIH|JyRPp&|=2#YTX@4gAfDyuvGRS8oy+kv~-4n4ROaa`$z7K80^!Ui$uN7Tq6l z*BQeL;2@)ym>ixQn;c~(2?BRmet5x0>5VTW3MIsc0+KQf87& zCaFIp@{$_}pj&bL}E&P`wAhabP>vtXL5 z>J(`-Xqr}#R*{w^>W168!Esu4RE-Q0$D!(%!)TdC?{azuwN{$BT~XySLiJ z!6wTz#d#Q+h(e&PUcUF`4>v6jP2YX?WCv`0S7!Ri-fg@1IG2mPij& zd*f@07qJqGp4nKn9g}9z$e9U`Y^FDpowkUbTz$5t{vc@Z*eSsqw>@hJZ%i<%+b8q@ zPyYt=7)Am%-?^P9VO=eNAsIXyMBF2Ru`2NWQSk&dVz42oV|M`~Flkkg(bRZ-%{rSu zlY6tQ_vBC?z@t+roRp+}WbSIJ(IE-7Hvxv6bfra^1tRN88BUH!;}ecHg=`9sd^W-C zmW6JDfdmUQB#|AWN|`C!J=s0gBPudSfTwS)xvYG^y)AUi{3c`?knK$mBYU1zPa z@``pz1t$YfGf?zmTM+|HDvRe?e)Ga%{Ocp*@IzMS^sNKg$P4`-$%osHxP%i)GmQcj~t4cy4#j&rrYdoq&-!Z$)P{rAI#`Gu^tEJlTO>(xS}3p$ksX-2AEU z{EM7Cy`mk>x=AlL&lOSk66mKw!_Tf4++N9myPw+@_uQ2&&yfZ=7@Nxz(#h%fV#v{> z02!qrBy68vaQNt-x3KT*O1&doB&Se+79mvow#LjTBx(7fXdWY_Qh{s`l9x6@S&q@} zAX3)T1t1i{Fp@s>R0x8UR3UbI&ibM8kuXG7-#Iw>fuL!<6KBmiQg<8g*EZHVth*WJ z^~@*QeDp-goClNHJ_Yzbq%|$fkC>&OwnR%&z55ElOi#@Zc*KkFla0MbW6=SZH=Cq# zQF+6j^T6{37osz22Iq>U$k8G`+cNr1w5}oviuC?LF!V{fM91)CAL7l9OnaUbVguki zDJcZn+``BItXtwlE(mK0VWu#iWkly>H4O@qOcme1`)`<%OvtchDlB~BP2*C})zuG) zcR|p){Z2k8hm)HdyV|=JAKBuCF5fosz@Oic=TZXqed#qj{;Z>H!gZ^LlHL(S;pRl_ zO@;RM-GBuG(EgSxb}R0D+(D)6RWgrN!kpVBo0aQ`V8Pf7FX3Br`_+oW6G|l91b06W z)bbO94|X&*MJjD_oY;-%32$+M!B4DRb(y}474yoq#an23R)!f&cecAVPXfTaFA`T6 z6Po|n?0QJE_@&%iAsSD3#k#A1m7jjZf53lQu>Xd^`b*sf#t2Svt`+U(lKverdfOBX z^(uvINmaAco5?5BQ(-_TOS>S`0nsp`ikKF$7I8T2;F`~88esS;*CB6E=2a~$oG0w< zQ+V687dY~QI--JpUj#30J4eLEKpJ_tEqwMS`G_xKaTWfSMBY5c&8NVrFMUC!nws^o zgJqvj4fZA`kwkL&1-lyIWuS6d?7e#gw>*k}OZm+H7WGaiCy~!4p$8D6Yma2vDF&m> z0cheRhk?76u9-*DGnNWj3Jc}T^E}5e$LS2Z+7+L*0xLdpv6uLs8eRWT722-3Y8NiR z?MoLav6flfmzZd}TZ`bRF`xLZ@|y2=I3 zLrV0RCLdHjly=z#Jy(%R8R|(nu~=Ecwh>9fFF2x%p)x_cTrYdQR21j4I-zF)p1 z(qHOf$Hz?uV)RX{9_$`WmC_}pT|lFDmr9pxfYLgB52Z2r;?4{CF@KGrpX^DBpOzL* zMVxv?-9b%jwTNcjFUnukCyOT!i=wuK^2qWsKmWX&VlT)Ug@+&Me8$d7P1D1O*NrpF z_eFdL5`K##1E_+FO~dpAPjFQzhuLUqn&O@AL!noYR`#v*YD*ytpNA}F#S`Z(cH?g` z`J@=+${(}USi8l)f>4_01Uix~sQ1ihe^ziF%Rr2|u3slwbeSS~r#_Vvw5Ft4a{VaW zPp$xqB%oKO;3S_0#>7b}(j;q!5cZ*%;~3qHZq=lzyI`yMkfK8ki&_MogrOYG(Qa#A z^W~GLJ{V3pL0VJ7S?A=jZ#qL~-c7g`Fc071#}bt5cz*I~+ES@;*RmKs-Nt@Jq%9Z5 zSKnhx3p#dr2u_?8}NtKaLe%?;ep+z#o}yC zQ8GfKt%#f#u&Zm)^waS(^Fe z1rAD9GEB8J+w>lBMA~@rc}|?bqsjNbaG<-tsMS}HA*U*j$6Mi#acElOf8A%daZHI2 zi}UuO-s(l*EoT?t#JNS_3lYum55tooLuUIw?WljkC|lW@`~i=*LtzvV?}EWHAl?3d zA+M-v_jQg4TYwP4k@NDQG7axJA%A?iQrS-h2Dj)jzpAOwaPCv2W-)o$!}*87nioIT z<)apUV9orIAf1mQ4M3g*Md?X8JOTg`NlYFO%PXSosl!h9M2UjnE9lsvM$EGv52__Zl=DzmIVSTo|ALurE?Q5L2ju^~r=Od0-8==rIvhWnk!9XP4^qpf}C zFKf~YzbdgdZT}2sV^eeX*f$UKkL5;K1G7V?4qMuRSd2w6-O7dDgtBD|B))5v< z!1owQSIa0u7w9n7evcR{=7Fr%Hc(He&QMR)A}kPQ@TyNl!9d5!t7ujiSBpKXlD`LJ zo_$0$im|NlW3C$15e6vQ4@K`f3+1UQ3^Gz{>T&Fo5u(uh+^C z<9OkQS$0dUYodna*g_RMRaK&l84^j+4H&>TF-7~5@xrPUsSHca$z0^0?zyVm<{j1r zQJ=X0F+2*@xLhYfidkKj>EvhqGfrYFEhrR)E8%`kLlb@PqoZ z?kvk-U1$UTyY=HdI@M6MrCH*4se~r8OFn8$jI_%6sMTV`9kj;erghlxh&Kl`Vn0z-Q8RCR^rO!(#W(Cq&C0HKDw+AYxsmeY2tl2)P%DKv$ zF3yESY(OY{_e)| z!hckL@eE;fYpkaGy(-vu+HYEAY&F~j7;JHkck99?HAvB^OnuIC!Gn~gVEOGfR7z6P zAyix(-O-E`6;@el-isTkSg%PG5+A?mkOcbJDA5_>8DWXfFh}WsDCKuz2qEe z)YM(ytFwi6;Q8u4!}wb=*gY7_6s(0@kY=iZRXrHWYv!~ZeS-kU_-bw{!r&tT&N%Uf zxVbg_o&@uK^b%2^ezAs&p+Fw1ga|_mTMN7XH+~EG2*3g~$3|f*1+M7-Djtt=c?|h& z(S+y+n^%UH_P%l>gJXnx9U5vb?k~a?ilgb`=(*z+1f26T0ER;{rWSPHLL_8VGkfICBe|g z_O&wi++QEc!*ovSqWtXs>*7ea%q0GIomS`$M(tP()nYQZX-xW-rbGX(YP%7 ze4we)t@0{d;j5F_O;YTXilp3p=_Up?+Vxa%cmk0jfi)=+?v(M$l}gw&vpC|53EfQZ zg`u=6ynH4bd!b?7=P<`OTJ~e}Ws7$ee&fPxm9{U>;zw2gwp*m-LyF4)8W)}6_=ear>w3% zTB~m#Z{grJl|H~;=q<_lQ~rmAwKdHF6<%rq|6^FQ$l4) zn_BA=Yndsotl`&s9jkZ?$k7NZS-iO6&|zTYoOkWz+{vvtG#ak@S(|$5>KGFVE;>50 zK$N^zPU|g>OW_aAB-XIg!f1O!nD0(S8iYo; z2Ds{2XNaECM$RV0zIfe=f8%y=ZnlugvQ?{9Y*iu(;QvbF@NhU-KK?1mlEu!Myk+V3 z^RQKsO4Qe*Vn#Hf-@xt?a8o?8!043pmbMM|d6l+dfAy$lXSDSJ)Ws5~54*a^>1>ly z?2cCtlM-Tn^0I^xyzU2k;z#L$)tTm{# zBk2pyMc@EQ3Y8w0{GdF3$7e^w2j{L@^0yT{ISP!}++|V=*R-ToMTs>6@+hG$B4m7C zX^66mxlN+>evq`EQk!lkFNjg0j6HSosG^o)V_)15(!0h>y1Uk2`ETe2=pFBFCNz}2 zpMB^4Xu00SUrkQO<<#3Ygytd_A35$Tb%(1Hs2@A~b}lyS|If zf1kOtQdUnh6Unxwe$SozjL;1dV?vPJr<<9RmyNjytg|J4L3)aKRcQH4FQHIeLC#H% zNzOHek4Z>_Pmf?t8iVu8pkNyRTl^>ZS?bR%Ui{?totZx0rHHJo-{WHx#|&(oco$@D z=sDk}zqDamm`m|_Gheqxr)lm`3C*WxolY_;(@*SR)fA;;ie`XN?)R(J$!QAHxS41z z6lj=gkV|*6jhh}Z)!9=W48LzRy1LaLD*8@POWzF&x7!yA_8d2jXtx!1CZPH|=y>&? zUL zuU|5}HTY%q;o8>;FOZ-cr*5nU($%pHG-i|Ma~~!y3mi(q3tbYK^yhXOV`7QC6C3oe zXb&E0vY{@Ab6|H--Ll@A8pg76iw1*G@k}dBcjg2%;(pC@3{HV^Fsu$bk2Y#MxxJ4& zTcVFJj;rsJ-xZG)7j;9UM!B5l+q76eo6ISG&0Va;AtaADuk84Q#hIe6MI>8O#!yOB zX6KQN16v-jHR7A98ts^i`ZW%z`a>W3G#)PL5V(Cx?r*UQP|uXX`WTje~5{ z7EzdYhhNcRYFv4R_~;jcOx58i49(hD{B$1h7e2A%y*!WwqFZCfj$6e6M0fx8vGDBN|;x&-&V~FA*RV7W%oi!5na|bklRIC{SXWvbo5~ zky7HJ>%}d0o8Qwy`|bn>b@aT@fY9~0MLRDh^i$(h$H_aV^uTm9ukik6(3U3C!dGZ? zw^>NoWplECEU6Z8%Umty?+OxrwsvaDY8KVi3$UlmetvbEbydF4<9@Z5rSWHyzdnC) z3$67C7q_3awk7&r%JbRos8UTp(8KQN{^OFrc{K!PY~KNy*fGa^qHNt*a@PhG>?>U5 zX7PKCklQYv*}=*-)yQMDk|~Gbr(vfBuKUS1{R%;k97=ZRdZ8wESf$UTC*`-N^A?`|*clU5wXvF;fjH)z0Iy^jFhqZfF z^Yht0A$|}K)lS(2r-$Nc6MQ!=xaTzj$3oeb>%_D55z(bdP#b6IPhhjV3x^400RH5t z030>wx=}?jtbv$dc=Fzs>Hsbk!&TbV8*Y=}L1X>JmBBt=L=C%4FVUY+6`L);fLTD< z&R1$Bz?XkIa~LGdgQ$v46}~KwfWbJ%;T8uUlAC21L-(hQv268GC^w(m z4uKc+B4J;*91T+TI{RfgGd<4HD40C%L==X0#=C5}vxG9oMLChE&&n5xst^Vnh__Im6u`_Wag1(z7!?HZ}Rx4FtEH~8N zmamZL$yXt7l>Or&+QEa4GE|)W$=RQ^-da*vCx{A z%#@w#7ncHmZ}}q&9}-yGWEzbVVRPwxHDtDHyZ=!PEufRpC4IA+YH+jqB? znvCS{(OZ9Ek)(ANkL4J9?EX{J{K8hCPhxqMQUN<3jWH^36n0X9uj*Ql;csLj!D&nk zQ97Z@=j+`(r{E=&vn0jl}N*d-JMQW6heHGPl(2l14I6koQ);Flq6 zg@hp{+eLxzn^VLj$dz7cYgAf*laWqyJPbiF{LjQ}^l1@O$40ngsFvdnj}Q+IB3&-G zWw^XZDb1rVV4*#lJ+Cq~ppmtajgYO7ReGN-+r$WRSxoec{n}gGw7ECxw`2C#LD8}{ ze@p!fse6L6<7=bAW4TONmFpR+lshCLk|<=f5xhw_Dj|`fF~`s5GJ|W@UUC8VU;st%byV_nVZ3i?&7IvoR`T zW82G|Tk^gk=MXiCW46@FX|w!9p|2b!H`U-4os5iuz$ixICG<4V&Sqy9BG%(Tiraqi zgT6A~e(&%3ws`AwMChg<$<-TNe&Yy#cz*cjw?k0Yh`d4I0x5e~%0T(ti}6jHi8uwb z<_(Lz0!>Pu_0$jRQ`prDb~5+Gob_jz;}J>B;Geh+P+O)La9}WP8M}L0Bcif6Wgz5x zFOSK@yQzFWZa-aj-7IEyMFX}0jCF>7$s#zFL9&E9x!R<1*1#5Ny>wRK3PyyZ49ZMJ zty1E&MBqpz5>@M5t3c-n(EbtA_Oj^Mh)@XY_3w-<{iAGTB_QhgZ&4Ctc~qjBZC7B_ z0euVf^Xw#!LF6rQq|RXetY74BF+IHq3HuT@3jGq#Ul;*$B~WI1QsEM(Kp&}22{^=N z2{Z-|bmRn|q&H75b+L^wrBFm>iS$^P+&3~h&q}IDT=D_zpLRIE>y)zSTHs46f%!U| zz1cL3T+ahsg$soBOL+`92GGsN3M%$Ef1hoOf&*J-dPP$?{zg@ z*LkkrU-$F9pYQ$L&-3r^^RRcNOEJT85Vl#S58DkkPlF;BR3cADkYo!K5t}-ZKI_Ru zWGdr&wR0ADTN?=>-?~I8d4KLmX1PZCwMt{i1|LDK_pEQLwu7<6F;OS@TA!-#P7X423`0TOk>zcA`^S_hpe0Vr3x(_=xjrgZ93}( zo$oYUTE{5)>PX(c6Pr5{T_duzi_9q}@C(i~ws?0_bJ#-na$k3zQXM+T1V1Hr_ zq-4B8V?pL+Qa_!-bdy+V9B?m(d?fR!rFmV0-CZT8oG}LTl|2D*@J~q1Sx)K_x*5H8 ztQ{7%zsgY&qZD-&JA5RqS!-lk>hgVMZ%O){a7Vx4`G^=n4QyVf=rf|Ly#M{sXm^L$ zYu#8ZOSOdhG)~QGjP!fUSd@5L|k+yucT`4I(WUG6ja5dr;IELXkE!PC5u z&5f5hnfF7-W;VYKqzP#E4G*jY=qi+Jv0o6QXdC#jiO-zkYk}Moef`*LLq$0IksPho zUC~*FsWzDKbDv0?r4#EDdh|L(BIgM~x0dFuw1N|a8eBxO2g&-hZ_uwCtHejB1M{Tb zV_&X4T3Z=hS(!j5SdP60`=gduR?vpuOS-Qh%SOo?!SZYR<8XFQ_hurU?6@(#LyQd+ zgA!Gbj{^tP1p3e1l#k#!*QXyT9+e>aV;|2UB+Y$8CYp?!{t?X1*uoIUZlg;zB&kQT zZVXTDI({try5H+e(D@qfSJD~9*yPw*({=sFv$JBTa3ywlU6mE6=z2NJv?gGr7eLF2j8dy1%Mb6+t^;D6uYFa7XN8_sYQdhXvx}A!>r>^tg3q)_ z?2hr`z}l6zahs8oR+7_W>vi%Tdkn=^L#Ug_b5{F1*-^~F+5)Z1fsQ9tsq>%b7iA0d zDza*`wib`h0W4ww7O_B-NzH&08ey;L{{p@NutW0CW%OA0x8PF8b}p&F{JILGb;2)5 zB`)CcV^7Pfp&IS;_E;LtH;aY~Hz7ySVuLzgI9(2Yvb*P1(U>ZhhQCZMN*@{}cC{on zjrn_cxtS!`mHC#Ex?ZKCM_Xjg5^|nz-I|JaO>8Hc$K<#-(iudP%BiBKT-m z%h9q^*_FO?)|9kdk1SgA4G!Lyh;-pSyT8+(KC&;Z?|eI;yHFXG^=Bm=c&1EE3d6mA z>m{$)B@Ml5pN0l7*iyM4;|8zV$3c@hu&t-IurU%+ z640ZYZ7MrjT^4D{s;`-Pi0_aiQl&4UW{XY*n|b9(w>x1b-x-CE2ZOzuS@;0$Al3(l zHmmd?)wT6|A*q|VKvxOiltiQM?yFbteoTkki{;B~6Z>mWxxpKm}W92E1sxKfic?ew4?{a)n|q(MUFc z=R$aJ==a;%J3lWJSfvPmf^q#F1NI0X&h?EXZsgfXNE`=RrXDVo=&~p`!meIwmuo;a zP7c3DzdRDQ^rn&vs%q|VqNT(&yv=mRu{=WqOwjv^Z=yZ5tHVmEiIq~4Ns#lr>>l&TuV())R2+yz9=xLN%?$e z9Ep(F7441)RB2pK?*UH9cKg)_^%9KFv`6HD)~{N0sI__8yMHOX6x8r(Pz zRd(S-d~L6xM0&!Aoa;WR>go>X$)aF~jo@)@mx>IgN0VI5;(gch=FGY=v=USut zB>-~w`n`;UhDM!O==VIo@rLvCl0e;VHONKV#qqXwD4?{`L8e&3jb&Q<`%2m*#9%R3 zdf~JK#&e3QSj6B2c$;U&^}UmO(9?ZjCaHKp?a2j+JViNXhjuEsEhR~-p!t@W&zNL9 z=}yeK)~1%G_c-i6FYiVnfy;Td?O=xDM0C^PM156j6sNNjj?^c2WSg4s*JI253i)-) z=|rOhuRXM_nnw2GS5rTLwxhoM-sS>G8L{g-c8Tclr`){GcFdvnm)MxLb!sgK@4~}z z_%1y8AlBla<9$*|$L~CE*|a_%PXpqqo9?$0JNKU?&TKi7d~W23zRsLliq{|J5-%S8 zF)(f=DST?3*|-5UZDCpnc()ziMhq(rBf@81e!5sT6dkhx1{e)G+3#S|H)sgY9HMzzzcsgKAPEFpR{*O@t$LZzuzZR!Vo9o`nGISIknh5 z%qYco@!4*#PpJdV01wk6jL<;HxjP<$dY2)ypoQ(RM#i5YM-Rujh?Vv{2@muWy{DJu+DCTMXYeJ1L8o#fTqx zTPJ&!`CMft2~pqs!Nb#m5xTsGM*1p&nO+LKeKM&g6foE;jUhwbMrWo5c152TV>`Q% z047t5ov*kNzhz|kedMLUJej#={=G!FP>01&HCs@@N>D8v9?ylmtIc~NR z&IHMIU75A3++BfWM@=AYY>p-V)0bxS2oe9g>cbZKPRG6S_hc7e2PUJ+`!6!YDAUc} z<{#$I_TDUMH51p}H{fCYfd89JZoE>nQaedYQ~Dv+!!@KN|r42l2=**kyDDqQ|w46Y4^ z{u~p+XdpsjkBANd))mo#pl}_C=z1`SkQpSJ3)h8;>fkV4knS%!xUP`x^JDy7A3_%b z74a3;2L;1_)`7LPVM3zLPq{)JLR-Wa41$88qQ1f`(H{I*NLM5m1VZRSMfV0mz;s|g z`wDe%A=l`qT0mf-2SOwU0zsi7H4AlMT~QBhunt_bFWO+Zj_7(iFp#$BdO9#2A>!%h zTA(nvwrCH)Al+Yj4+g=1_0@(#ME5J~{V%yXFoX~h^|P-I94rJ&{T&M=dI%X!L~$YF r%FZ{;`^VMQvLz4#6@DDXe>hyEKo4TzUpE`94H2HPxRR2o)#-l&X5~%^ literal 0 HcmV?d00001 From 9ef00d187f3b96312488166d1cb9edbaa0df2c6d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 29 Dec 2014 22:55:58 -0800 Subject: [PATCH 276/688] %s/master/manager/g + better comments --- uncore/src/main/scala/cache.scala | 20 +- uncore/src/main/scala/coherence.scala | 255 +++++++++++++------------- uncore/src/main/scala/htif.scala | 4 +- uncore/src/main/scala/network.scala | 2 +- uncore/src/main/scala/tilelink.scala | 142 ++++++++------ uncore/src/main/scala/uncore.scala | 12 +- 6 files changed, 226 insertions(+), 209 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d871e652..0e963f24 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -118,7 +118,7 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { } object L2Metadata { - def apply(tag: Bits, coh: MasterMetadata) = { + def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh @@ -126,7 +126,7 @@ object L2Metadata { } } class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new MasterMetadata + val coh = new ManagerMetadata } class L2MetaReadReq extends MetaReadReq with HasL2Id { @@ -156,7 +156,7 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.masterMetadataOnFlush))) + val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) meta.io.read <> io.read meta.io.write <> io.write @@ -322,7 +322,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire finished transaction acks val finish = io.inner.finish - val finish_idx = finish.bits.payload.master_xact_id + val finish_idx = finish.bits.payload.manager_xact_id trackerList.zipWithIndex.map { case (t, i) => t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) } @@ -355,7 +355,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { val addr = UInt(width = tlAddrBits) - val coh = new MasterMetadata + val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -385,7 +385,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val state = Reg(init=s_idle) val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) - val xact_coh = Reg{ new MasterMetadata } + val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_id = Reg{ UInt() } @@ -406,7 +406,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) - io.meta.write.bits.data.coh := co.masterMetadataOnRelease(xact, + io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) @@ -701,8 +701,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St (xact.addr === c_rel.payload.addr) && (state === s_probe) - val next_coh_on_rel = co.masterMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.masterMetadataOnGrant(c_gnt.payload, xact_meta.coh, + val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, c_gnt.header.dst) val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index c7999f4b..08d7cf20 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,6 +3,8 @@ package uncore import Chisel._ + +// Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters class ClientMetadata extends CoherenceMetadata { @@ -18,21 +20,23 @@ object ClientMetadata { } } -class MasterMetadata extends CoherenceMetadata { +class ManagerMetadata extends CoherenceMetadata { val state = UInt(width = co.masterStateWidth) val sharers = UInt(width = co.dir.width) - override def clone = new MasterMetadata().asInstanceOf[this.type] + override def clone = new ManagerMetadata().asInstanceOf[this.type] } -object MasterMetadata { +object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new MasterMetadata + val m = new ManagerMetadata m.state := state m.sharers := sharers m } - def apply(state: UInt): MasterMetadata = apply(state, new MasterMetadata().co.dir.flush) + def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } +// This class encapsulates transformations on different directory information +// storage formats abstract class DirectoryRepresentation(val width: Int) { def pop(prev: UInt, id: UInt): UInt def push(prev: UInt, id: UInt): UInt @@ -66,15 +70,16 @@ class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients def full(s: UInt) = s } +// Coherence policy inferface for clients and managers abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def nClientStates: Int - def nMasterStates: Int + def nManagerStates: Int def nAcquireTypes: Int def nProbeTypes: Int def nReleaseTypes: Int def nGrantTypes: Int def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nMasterStates) + def masterStateWidth = log2Up(nManagerStates) def acquireTypeWidth = log2Up(nAcquireTypes) def probeTypeWidth = log2Up(nProbeTypes) def releaseTypeWidth = log2Up(nReleaseTypes) @@ -82,49 +87,48 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def isHit (cmd: UInt, m: ClientMetadata): Bool def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: MasterMetadata): Bool - def isValid (m: MasterMetadata): Bool + def isHit (incoming: Acquire, m: ManagerMetadata): Bool + def isValid (m: ManagerMetadata): Bool def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: MasterMetadata): Bool + def needsWriteback(m: ManagerMetadata): Bool def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def masterMetadataOnFlush: MasterMetadata - def masterMetadataOnRelease(incoming: Release, m: MasterMetadata, src: UInt): MasterMetadata - def masterMetadataOnGrant(outgoing: Grant, m: MasterMetadata, dst: UInt): MasterMetadata + def managerMetadataOnFlush: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: MasterMetadata): UInt + def getProbeType(a: Acquire, m: ManagerMetadata): UInt def getProbeTypeOnVoluntaryWriteback: UInt def getReleaseTypeOnCacheControl(cmd: UInt): UInt def getReleaseTypeOnVoluntaryWriteback(): UInt def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: MasterMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt + def getGrantType(a: Acquire, m: ManagerMetadata): UInt + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt - def messageHasData(rel: SourcedMessage): Bool + def messageHasData(rel: TileLinkChannel): Bool def messageUpdatesDataArray(g: Grant): Bool def isVoluntary(rel: Release): Bool def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: MasterMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: MasterMetadata): Bool + def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: MasterMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata): Bool + def requiresProbes(a: Acquire, m: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool def requiresAckForGrant(g: Grant): Bool def requiresAckForRelease(r: Release): Bool - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - def getGrantTypeForUncached(a: Acquire, m: MasterMetadata): UInt = { + def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { MuxLookup(a.a_type, Grant.uncachedRead, Array( Acquire.uncachedRead -> Grant.uncachedRead, Acquire.uncachedWrite -> Grant.uncachedWrite, @@ -135,14 +139,14 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 2 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 def nReleaseTypes = 5 def nGrantTypes = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -154,8 +158,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata): Bool = isValid(m) - def isValid (m: MasterMetadata): Bool = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) + def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { @@ -167,7 +171,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( @@ -183,18 +187,18 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> clientInvalid, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers, src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -217,7 +221,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -230,12 +234,12 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -245,29 +249,28 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 2 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -279,8 +282,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -295,7 +298,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -316,18 +319,18 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientExclusiveClean, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -356,7 +359,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -369,12 +372,12 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -384,31 +387,29 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 3 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -425,8 +426,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -441,7 +442,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -461,18 +462,18 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -501,7 +502,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -514,15 +515,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -536,34 +537,32 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 4 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 def nReleaseTypes = 7 def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) @@ -580,8 +579,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -596,7 +595,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) @@ -621,18 +620,18 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> m.state ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -661,7 +660,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -674,15 +673,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), Mux(a.a_type === acquireReadShared, Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), grantReadExclusive)) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -696,34 +695,32 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nClientStates = 7 - def nMasterStates = 2 + def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 def nReleaseTypes = 11 def nGrantTypes = 5 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nMasterStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nMasterStates) + //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) @@ -739,8 +736,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (m: ClientMetadata): Bool = { m.state != clientInvalid } - def isHit (incoming: Acquire, m: MasterMetadata) = isValid(m) - def isValid (m: MasterMetadata) = m.state != masterInvalid + def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) + def isValid (m: ManagerMetadata) = m.state != masterInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { (isRead(cmd) && outstanding.uncached) || @@ -755,7 +752,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def needsWriteback (m: ClientMetadata): Bool = { needsTransactionOnCacheControl(M_INV, m) } - def needsWriteback(m: MasterMetadata) = isValid(m) + def needsWriteback(m: ManagerMetadata) = isValid(m) def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( @@ -792,9 +789,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientMigratoryClean -> clientSharedByTwo, clientMigratoryDirty -> clientInvalid)) ))) - def masterMetadataOnFlush = MasterMetadata(masterInvalid) - def masterMetadataOnRelease(r: Release, m: MasterMetadata, src: UInt) = { - val next = MasterMetadata(masterValid, dir.pop(m.sharers,src)) + def managerMetadataOnFlush = ManagerMetadata(masterInvalid) + def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) MuxBundle(m, Array( r.is(releaseVoluntaryInvalidateData) -> next, r.is(releaseInvalidateData) -> next, @@ -803,9 +800,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d r.is(releaseInvalidateAckMigratory) -> next )) } - def masterMetadataOnGrant(g: Grant, m: MasterMetadata, dst: UInt) = { - val cached = MasterMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = MasterMetadata(masterValid, m.sharers) + def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { + val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) + val uncached = ManagerMetadata(masterValid, m.sharers) Mux(g.uncached, uncached, cached) } @@ -837,7 +834,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(needsWriteback(m), with_data, without_data) } - def messageHasData(msg: SourcedMessage) = msg match { + def messageHasData(msg: TileLinkChannel) = msg match { case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) @@ -850,7 +847,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: MasterMetadata): UInt = { + def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, getGrantTypeForUncached(a, m), MuxLookup(a.a_type, grantReadShared, Array( acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), @@ -858,9 +855,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? ))) } - def getGrantTypeOnVoluntaryWriteback(m: MasterMetadata): UInt = grantVoluntaryAck + def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - def getProbeType(a: Acquire, m: MasterMetadata): UInt = { + def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedRead -> probeCopy, @@ -875,19 +872,17 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: MasterMetadata) = + def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: MasterMetadata) = + def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck def requiresAckForRelease(r: Release) = Bool(false) def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: MasterMetadata) = !dir.none(m.sharers) && + def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && Mux(dir.one(m.sharers), Bool(true), Mux(a.uncached, a.a_type != Acquire.uncachedRead, a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: MasterMetadata) = !dir.none(m.sharers) - - def pendingVoluntaryReleaseIsSufficient(r_type: UInt, p_type: UInt): Bool = (r_type === releaseVoluntaryInvalidateData) + def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 378ca3b1..4db133d6 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -134,7 +134,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_needs_ack = Reg(Bool()) when (io.mem.grant.valid) { mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.master_xact_id + mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) } @@ -201,7 +201,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.master_xact_id := mem_gxid + io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc io.mem.probe.ready := Bool(false) io.mem.release.valid := Bool(false) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index a34e91c1..161a9a0c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -case object LNMasters extends Field[Int] +case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 4e5ac961..af10d455 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,10 +4,13 @@ package uncore import Chisel._ import scala.math.max -case object TLId extends Field[String] +// Parameters exposed to the top-level design, set based on +// external requirements or design space exploration +// +case object TLId extends Field[String] // Unique name per network case object TLCoherence extends Field[CoherencePolicy] case object TLAddrBits extends Field[Int] -case object TLMasterXactIdBits extends Field[Int] +case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] @@ -15,7 +18,7 @@ case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { val tlAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) - val tlMasterXactIdBits = params(TLMasterXactIdBits) + val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits @@ -25,13 +28,23 @@ abstract trait TileLinkParameters extends UsesParameters { val tlSubblockUnionBits = max(tlWriteMaskBits, (tlSubblockAddrBits + tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) } abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters +// Directionality of message channel +// Used to hook up logical network ports to physical network ports +trait TileLinkChannel extends TLBundle +trait ClientToManagerChannel extends TileLinkChannel +trait ManagerToClientChannel extends TileLinkChannel +trait ClientToClientChannel extends TileLinkChannel // Unused for now + +// Common signals that are used in multiple channels. +// These traits are useful for type parameterization. +// trait HasPhysicalAddress extends TLBundle { val addr = UInt(width = tlAddrBits) } @@ -40,19 +53,17 @@ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } -trait HasMasterTransactionId extends TLBundle { - val master_xact_id = Bits(width = tlMasterXactIdBits) +trait HasManagerTransactionId extends TLBundle { + val manager_xact_id = Bits(width = tlManagerXactIdBits) } trait HasTileLinkData extends TLBundle { val data = UInt(width = tlDataBits) } -trait SourcedMessage extends TLBundle -trait ClientSourcedMessage extends SourcedMessage -trait MasterSourcedMessage extends SourcedMessage +// Actual TileLink channel bundle definitions -class Acquire extends ClientSourcedMessage +class Acquire extends ClientToManagerChannel with HasPhysicalAddress with HasClientTransactionId with HasTileLinkData { @@ -61,16 +72,17 @@ class Acquire extends ClientSourcedMessage val subblock = Bits(width = tlSubblockUnionBits) val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits val opSzOff = tlUncachedOperandSizeBits + sbAddrOff - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits-1, 0) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff-1, tlUncachedOperandSizeBits) - def atomic_op(dummy: Int = 0) = subblock(opSzOff-1, sbAddrOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits-1, 0) + def allocate(dummy: Int = 0) = subblock(0) + def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) + def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) + def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) def is(t: UInt) = a_type === t } object Acquire { val nUncachedAcquireTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) + //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) @@ -102,29 +114,29 @@ object Acquire { } object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz) + acq.subblock := Cat(subblock_addr, operand_sz, alloc) acq } def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB) + apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) } def apply(addr: UInt): Acquire = { - apply(addr, UInt(0), UInt(0), MT_CB) + apply(addr, UInt(0)) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, data: UInt): Acquire = { + def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := write_mask + acq.subblock := Cat(write_mask, alloc) acq } def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), data) + apply(addr, client_xact_id, SInt(-1), Bool(true), data) } def apply(addr: UInt, data: UInt): Acquire = { apply(addr, UInt(0), data) @@ -136,11 +148,17 @@ object UncachedAtomic { subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz) + acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) acq } } +class Probe extends ManagerToClientChannel + with HasPhysicalAddress { + val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t +} + object Probe { def apply(p_type: UInt, addr: UInt) = { val prb = new Probe @@ -150,10 +168,12 @@ object Probe { } } -class Probe extends MasterSourcedMessage - with HasPhysicalAddress { - val p_type = UInt(width = co.probeTypeWidth) - def is(t: UInt) = p_type === t +class Release extends ClientToManagerChannel + with HasPhysicalAddress + with HasClientTransactionId + with HasTileLinkData { + val r_type = UInt(width = co.releaseTypeWidth) + def is(t: UInt) = r_type === t } object Release { @@ -173,18 +193,10 @@ object Release { } } -class Release extends ClientSourcedMessage - with HasPhysicalAddress - with HasClientTransactionId - with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) - def is(t: UInt) = r_type === t -} - -class Grant extends MasterSourcedMessage +class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId - with HasMasterTransactionId { + with HasManagerTransactionId { val uncached = Bool() val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) def is(t: UInt) = g_type === t @@ -192,29 +204,30 @@ class Grant extends MasterSourcedMessage object Grant { val nUncachedGrantTypes = 3 - //val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) + //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) def uncachedRead = UInt(0) def uncachedWrite = UInt(1) def uncachedAtomic = UInt(2) def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt, data: UInt): Grant = { + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id - gnt.master_xact_id := master_xact_id + gnt.manager_xact_id := manager_xact_id gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, master_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, master_xact_id, UInt(0)) + def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { + apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) } } -class Finish extends ClientSourcedMessage with HasMasterTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId +// Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends Bundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -226,6 +239,8 @@ class TileLinkIO extends UncachedTileLinkIO { val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } +// Converts UncachedTileLinkIO to regular TileLinkIO by pinning +// probe.ready and release.valid low class TileLinkIOWrapper extends TLModule { val io = new Bundle { val in = new UncachedTileLinkIO().flip @@ -245,13 +260,16 @@ object TileLinkIOWrapper { } } +// Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { - type MasterSourcedWithId = MasterSourcedMessage with HasClientTransactionId - type ClientSourcedWithId = ClientSourcedMessage with HasClientTransactionId + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + // These are filled in depending on whether the arbiter mucks with the + // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits - def masterSourcedClientXactId(in: MasterSourcedWithId): Bits - def arbIdx(in: MasterSourcedWithId): UInt + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt def hookupClientSource[M <: ClientSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], @@ -267,7 +285,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { }} } - def hookupMasterSource[M <: MasterSourcedWithId] + def hookupManagerSource[M <: ManagerSourcedWithId] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { out.ready := Bool(false) @@ -278,7 +296,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { out.ready := ins(i).ready } ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := masterSourcedClientXactId(out.bits.payload) + ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) } } } @@ -291,7 +309,7 @@ abstract class UncachedTileLinkIOArbiter(n: Int) } hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) io.out.finish <> finish_arb.io.out @@ -306,7 +324,7 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupMasterSource(io.in.map(_.grant), io.out.grant) + hookupManagerSource(io.in.map(_.grant), io.out.grant) io.in.map{ _.probe.valid := io.out.probe.valid } io.in.map{ _.probe.bits := io.out.probe.bits } @@ -317,35 +335,39 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } } +// Appends the port index of the arbiter to the client_xact_id abstract trait AppendsArbiterId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } +// Uses the client_xact_id as is (assumes it has been set to port index) abstract trait PassesId { - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = in.client_xact_id - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Overwrites some default client_xact_id with the port idx abstract trait UsesNewId { val arbN: Int - def clientSourcedClientXactId(in: ClientSourcedMessage with HasClientTransactionId, id: Int) = + def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = UInt(id, log2Up(arbN)) - def masterSourcedClientXactId(in: MasterSourcedMessage with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = UInt(0) - def arbIdx(in: MasterSourcedMessage with HasClientTransactionId) = + def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = in.client_xact_id } +// Mix-in id generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e8e82d22..d2531a9e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -194,7 +194,7 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.masterMetadataOnFlush), + co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId)) @@ -261,7 +261,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val curr_p_id = PriorityEncoder(probe_flags) val pending_outer_write = co.messageHasData(xact) - val pending_outer_read = co.requiresOuterRead(xact, co.masterMetadataOnFlush) + val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) @@ -290,13 +290,13 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.masterMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.masterMetadataOnFlush), + co.getGrantType(xact, co.managerMetadataOnFlush), xact_client_xact_id, UInt(trackerId), UInt(0)) // Data bypassed in parent @@ -316,7 +316,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri is(s_idle) { io.inner.acquire.ready := Bool(true) val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.masterMetadataOnFlush) + val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) when( io.inner.acquire.valid ) { xact_uncached := c_acq.payload.uncached xact_a_type := c_acq.payload.a_type @@ -389,7 +389,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.master_xact_id === UInt(trackerId)) { + when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } From f58f8bf3859ec09ddff6c33b488c5c23bbdd850a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 25 Jan 2015 15:37:04 -0800 Subject: [PATCH 277/688] Make L2 data array use a single Mem --- uncore/src/main/scala/cache.scala | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0e963f24..af3a53c7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -214,24 +214,22 @@ class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataW class L2DataArray extends L2HellaCacheModule { val io = new L2DataRWIO().flip - val waddr = io.write.bits.addr - val raddr = io.read.bits.addr val wmask = FillInterleaved(8, io.write.bits.wmask) - val resp = (0 until nWays).map { w => - val array = Mem(Bits(width=rowBits), nSets*refillCycles, seqRead = true) - val reg_raddr = Reg(UInt()) - when (io.write.bits.way_en(w) && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - }.elsewhen (io.read.bits.way_en(w) && io.read.valid) { - reg_raddr := raddr - } - array(reg_raddr) + val reg_raddr = Reg(UInt()) + val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + + when (io.write.bits.way_en.orR && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en.orR && io.read.valid) { + reg_raddr := raddr } + io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.data := Mux1H(ShiftRegister(io.read.bits.way_en, 1), resp) - - io.read.ready := !io.write.valid // TODO 1R/W vs 1R1W? + io.resp.bits.data := array(reg_raddr) + io.read.ready := !io.write.valid io.write.ready := Bool(true) } From 973eb43128e1294030f0527e530e0f71150d6023 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Jan 2015 19:48:49 -0800 Subject: [PATCH 278/688] state machine bug on uncached write hits --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index af3a53c7..468b6587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -870,7 +870,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } - when(local_data_resp_done) { state := s_meta_write } + when(local_data_resp_done) { + state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + } } is(s_data_write) { io.data.write.valid := Bool(true) From 7b4e9dd137cd907760c82ce984de7d37a67ec939 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Jan 2015 20:30:52 -0800 Subject: [PATCH 279/688] Block L2 transactions on the same set from proceeding in parallel --- uncore/src/main/scala/cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 468b6587..e7f0ce8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -688,8 +688,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := co.isCoherenceConflict(xact.addr, c_acq.payload.addr) && - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || + xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data io.has_acquire_match := co.messageHasData(xact) && @@ -857,9 +857,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } } is(s_data_read) { - io.data.read.valid := Bool(true) + io.data.read.valid := (if(tlDataBeats == 1) Bool(true) + else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) when(io.data.resp.valid) { - //TODO make sure cacq data is actually present before merging xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), io.data.resp.bits.data) } From 3aa030f9605e0dbd87be440cd747e98d0320470b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 1 Feb 2015 19:57:53 -0800 Subject: [PATCH 280/688] Support for uncached sub-block reads and writes, major TileLink and CoherencePolicy refactor. --- uncore/src/main/scala/cache.scala | 536 ++++++++---- uncore/src/main/scala/coherence.scala | 1165 ++++++++++++------------- uncore/src/main/scala/consts.scala | 6 +- uncore/src/main/scala/htif.scala | 26 +- uncore/src/main/scala/memserdes.scala | 14 +- uncore/src/main/scala/tilelink.scala | 407 ++++++--- uncore/src/main/scala/uncore.scala | 197 +++-- uncore/src/main/scala/util.scala | 6 +- 8 files changed, 1343 insertions(+), 1014 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e7f0ce8a..6651056f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,32 +2,26 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] -case object WordBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +case object AmoAluOperandBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { - val paddrBits = params(PAddrBits) - val vaddrBits = params(VAddrBits) - val pgIdxBits = params(PgIdxBits) val nSets = params(NSets) val blockOffBits = params(BlockOffBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits - val tagBits = paddrBits - untagBits + val tagBits = params(PAddrBits) - untagBits val nWays = params(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 - val wordBits = params(WordBits) - val wordBytes = wordBits/8 - val wordOffBits = log2Up(wordBytes) val rowBits = params(RowBits) - val rowWords = rowBits/wordBits val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) } @@ -35,6 +29,79 @@ abstract trait CacheParameters extends UsesParameters { abstract class CacheBundle extends Bundle with CacheParameters abstract class CacheModule extends Module with CacheParameters +class StoreGen(typ: Bits, addr: Bits, dat: Bits) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + lazy val wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { + val t = new StoreGen(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class AMOALU extends CacheModule { + val operandBits = params(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = new StoreGen(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} + abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -96,16 +163,35 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.write.ready := !rst } -abstract trait L2HellaCacheParameters extends CacheParameters - with CoherenceAgentParameters { +abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 val refillCyclesPerBeat = params(TLDataBits)/rowBits val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + require(refillCyclesPerBeat == 1) } -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { + def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { + val (cnt, cnt_done) = + Counter(inc && data.hasMultibeatData(), tlDataBeats) + val done = (inc && !data.hasMultibeatData()) || cnt_done + (cnt, done) + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload)._2 + } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { + connectDataBeatCounter(in.fire(), in.bits) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { + connectDataBeatCounter(in.valid, in.bits)._2 + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -117,6 +203,11 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } +trait HasL2Data extends HasTileLinkData { + def hasData(dummy: Int = 0) = Bool(true) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) +} + object L2Metadata { def apply(tag: Bits, coh: ManagerMetadata) = { val meta = new L2Metadata @@ -186,19 +277,19 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends L2HellaCacheBundle with HasL2Id { +class L2DataReadReq extends L2HellaCacheBundle + with HasCacheBlockAddress + with HasTileLinkBeatId + with HasL2Id { val way_en = Bits(width = nWays) - val addr = Bits(width = tlAddrBits) } -class L2DataWriteReq extends L2DataReadReq { +class L2DataWriteReq extends L2DataReadReq + with HasL2Data { val wmask = Bits(width = tlWriteMaskBits) - val data = Bits(width = tlDataBits) } -class L2DataResp extends Bundle with HasL2Id with TileLinkParameters { - val data = Bits(width = tlDataBits) -} +class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data trait HasL2DataReadIO extends L2HellaCacheBundle { val read = Decoupled(new L2DataReadReq) @@ -217,8 +308,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -228,6 +319,7 @@ class L2DataArray extends L2HellaCacheModule { io.resp.valid := ShiftRegister(io.read.fire(), 1) io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) io.resp.bits.data := array(reg_raddr) io.read.ready := !io.write.valid io.write.ready := Bool(true) @@ -261,10 +353,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } // Wiring helper funcs - def doOutputArbitration[T <: Data](out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { + def doOutputArbitration[T <: Data]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]], + count: Int = 1, + lock: T => Bool = (a: T) => Bool(true)) { val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } @@ -331,8 +424,11 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant), tlDataBeats, hasData _) + doOutputArbitration( + io.inner.grant, + trackerList.map(_.io.inner.grant), + tlDataBeats, + (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer @@ -344,15 +440,15 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac // Wire local memories doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read, tlDataBeats) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write), tlDataBeats) + doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr = UInt(width = tlAddrBits) + val addr_block = UInt(width = tlBlockAddrBits) val coh = new ManagerMetadata val way_en = Bits(width = nWays) } @@ -373,16 +469,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val has_release_match = Bool(OUTPUT) val data = new L2DataRWIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_coh = Reg{ new ManagerMetadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } @@ -393,30 +489,30 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val pending_probes = Reg(init = co.dir.flush) val curr_p_id = co.dir.next(pending_probes) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) + val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) - io.has_release_match := !co.isVoluntary(c_rel.payload) && - co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_coh, c_rel.header.src) + val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWrite(xact_addr, - UInt(trackerId), - xact_data(outer_data_write_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), { case TLId => outerId }) io.outer.grant.ready := Bool(false) // Never gets mgnts io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeTypeOnVoluntaryWriteback, xact_addr) + io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -426,7 +522,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.wb.req.ready := Bool(false) @@ -437,7 +534,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str is(s_idle) { io.wb.req.ready := Bool(true) when(io.wb.req.valid) { - xact_addr := io.wb.req.bits.addr + xact_addr_block := io.wb.req.bits.addr_block xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id @@ -463,12 +560,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str when(io.inner.release.valid) { xact_coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - xact_data(crel_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -478,16 +575,16 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_read_done) { state := s_data_resp } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(local_data_resp_cnt) := io.data.resp.bits.data } - when(local_data_resp_done) { state := s_outer_write } + when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_write } } is(s_outer_write) { io.outer.acquire.valid := Bool(true) - when(outer_data_write_done) { + when(macq_data_done) { io.wb.resp.valid := Bool(true) state := s_idle } @@ -508,41 +605,40 @@ abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCa val wb = new L2WritebackIO } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits - - def mergeData(acq: Acquire, old_data: UInt, new_data: UInt): UInt = { - //TODO apply acq's write mask - Mux(co.messageHasData(acq), old_data, new_data) - } + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cack = io.inner.finish.bits + val mgnt = io.outer.grant.bits } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release(xact_r_type, xact_addr, xact_client_xact_id) + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) - val collect_inner_data = Reg(init=Bool(false)) - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (local_data_cnt, local_data_done) = - Counter(io.data.write.fire(), tlDataBeats) + val collect_crel_data = Reg(init=Bool(false)) + val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -554,50 +650,51 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(xact_meta.coh), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, xact_meta.coh, xact_src) io.wb.req.valid := Bool(false) - when(collect_inner_data) { + when(collect_crel_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(crel.payload.addr_beat) := crel.payload.data + crel_data_valid(crel.payload.addr_beat) := Bool(true) } - when(inner_data_done) { collect_inner_data := Bool(false) } + when(crel_data_done) { collect_crel_data := Bool(false) } } switch (state) { is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_addr_beat := crel.payload.addr_beat + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_crel_data := crel.payload.hasMultibeatData() + crel_data_valid := Bits(1) state := s_meta_read } } @@ -611,27 +708,30 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou xact_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, - Mux(co.messageHasData(xact), s_data_write, s_meta_write), - s_grant) + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_grant, s_idle)) } } is(s_data_write) { - io.data.write.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (local_data_cnt < inner_data_cnt)) - when(local_data_done) { state := s_meta_write } + io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } } is(s_meta_write) { io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? + } } is(s_grant) { io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } @@ -640,72 +740,109 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_busy :: Nil = Enum(UInt(), 14) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) + val collect_cacq_data = Reg(init=Bool(false)) + val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) val crel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) val curr_p_id = co.dir.next(pending_probes) val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(co.requiresSelfProbe(xact), + val mask_self = Mux(xact.requiresSelfProbe(), full_sharers | (UInt(1) << xact_src), full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) val mask_incoherent = mask_self & ~io.tile_incoherent - val collect_cacq_data = Reg(init=Bool(false)) - //TODO: zero width wires - val (cacq_data_cnt, cacq_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) - val (crel_data_cnt, crel_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && co.messageHasData(io.inner.grant.bits.payload), tlDataBeats) - val (outer_data_write_cnt, outer_data_write_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) - val (outer_data_resp_cnt, outer_data_resp_done) = - Counter(io.outer.grant.fire() && co.messageHasData(io.outer.grant.bits.payload), tlDataBeats) - val (local_data_read_cnt, local_data_read_done) = Counter(io.data.read.fire(), tlDataBeats) - val (local_data_write_cnt, local_data_write_done) = Counter(io.data.write.fire(), tlDataBeats) - val (local_data_resp_cnt, local_data_resp_done) = Counter(io.data.resp.valid, tlDataBeats) + val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) + val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) + val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val needs_writeback = !xact_tag_match && co.needsWriteback(xact_meta.coh) + val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //TODO: uncached does or does not allocate + //val do_allocate = !xact_uncached || xact.allocate() + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := xact.data(0) // default + + def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + amoalu.io.lhs := old_data + amoalu.io.rhs := new_data + val wmask = FillInterleaved(8, xact.write_mask()) + buffer(incoming.addr_beat) := + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), + wmask & new_data | ~wmask & old_data, old_data)) + } - //TODO: Are there any races between lines with the same idx? //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr, c_acq.payload.addr) || - xact.addr(idxMSB,idxLSB) === c_acq.payload.addr(idxMSB,idxLSB)) && + io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || + xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := co.messageHasData(xact) && - (xact.addr === c_acq.payload.addr) && + io.has_acquire_match := xact.hasMultibeatData() && + (xact.addr_block === cacq.payload.addr_block) && collect_cacq_data - io.has_release_match := !co.isVoluntary(c_rel.payload) && - (xact.addr === c_rel.payload.addr) && + io.has_release_match := !crel.payload.isVoluntary() && + (xact.addr_block === crel.payload.addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(c_rel.payload, xact_meta.coh, c_rel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant(c_gnt.payload, xact_meta.coh, - c_gnt.header.dst) + val next_coh_on_rel = co.managerMetadataOnRelease( + incoming = crel.payload, + meta = xact_meta.coh, + src = crel.header.src) + val next_coh_on_gnt = co.managerMetadataOnGrant( + outgoing = cgnt.payload, + meta = xact_meta.coh, + dst = cgnt.header.dst) - val outer_write = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_write_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedRead( xact_addr, UInt(trackerId)), { case TLId => outerId }) + val outer_write = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = macq_data_cnt, + data = xact_data(macq_data_cnt)), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), + { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -714,15 +851,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, xact_meta.coh), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, co.getGrantType(xact, xact_meta.coh), - xact_client_xact_id, - UInt(trackerId), - xact_data(cgnt_data_cnt)) + io.inner.grant.bits.payload := xact.makeGrant( + manager_xact_id = UInt(trackerId), + meta = xact_meta.coh, + addr_beat = cgnt_data_cnt, + data = xact_data(cgnt_data_cnt)) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -731,26 +869,28 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr := Cat(xact_addr, local_data_read_cnt) + io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr := Cat(xact_addr, local_data_write_cnt) + io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(local_data_write_cnt) + io.data.write.bits.data := xact_data(write_data_cnt) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) io.meta.write.valid := Bool(false) io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := next_coh_on_gnt io.wb.req.valid := Bool(false) - io.wb.req.bits.addr := Cat(xact_meta.tag, xact_addr(idxMSB,idxLSB)) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) @@ -758,7 +898,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St when(collect_cacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(cacq_data_cnt) := c_acq.payload.data + xact_data(cacq.payload.addr_beat) := cacq.payload.data + cacq_data_valid(cacq.payload.addr_beat) := Bool(true) } when(cacq_data_done) { collect_cacq_data := Bool(false) } } @@ -767,14 +908,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_cacq_data := co.messageHasData(c_acq.payload) + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_cacq_data := cacq.payload.hasMultibeatData() state := s_meta_read } } @@ -789,9 +931,20 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.needsWriteback(coh) + val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) + val full_block = !xact.uncached || + xact.is(Acquire.uncachedReadBlock) || + xact.is(Acquire.uncachedWriteBlock) + read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) + write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) + resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) + cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) + cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) when(_needs_probes) { pending_probes := mask_incoherent(nCoherentClients-1,0) release_count := co.dir.count(mask_incoherent) @@ -816,19 +969,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St pending_probes := co.dir.pop(pending_probes, curr_p_id) } // Handle releases, which may have data being written back + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! io.inner.release.ready := Bool(true) when(io.inner.release.valid) { xact_meta.coh := next_coh_on_rel // Handle released dirty data - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { crel_had_data := Bool(true) - //TODO make sure cacq data is actually present before merging - xact_data(crel_data_cnt) := mergeData(xact, - xact_data(crel_data_cnt), - c_rel.payload.data) + mergeData(xact_data, crel.payload) } // We don't decrement release_count until we've received all the data beats. - when(!co.messageHasData(c_rel.payload) || crel_data_done) { + when(!crel.payload.hasMultibeatData() || crel_data_done) { release_count := release_count - UInt(1) } } @@ -847,37 +999,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { //TODO make sure cacq data is actually present before merging - xact_data(outer_data_resp_cnt) := mergeData(xact, xact_data(outer_data_resp_cnt), - io.outer.grant.bits.payload.data) - //TODO: set pending client state in xact_meta.coh - when(outer_data_resp_done) { - state := Mux(co.messageHasData(io.outer.grant.bits.payload), - s_data_write, s_data_read) + mergeData(xact_data, mgnt.payload) + when(mgnt_data_done) { + state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) } } } is(s_data_read) { - io.data.read.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_cacq_data || (local_data_resp_cnt < cacq_data_cnt)) + io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) + } + when(io.data.read.ready) { + read_data_cnt := read_data_cnt + UInt(1) + when(read_data_cnt === read_data_max) { state := s_data_resp } } - when(local_data_read_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - xact_data(local_data_resp_cnt) := mergeData(xact, xact_data(local_data_resp_cnt), - io.data.resp.bits.data) + mergeData(xact_data, io.data.resp.bits) + resp_data_cnt := resp_data_cnt + UInt(1) } - when(local_data_resp_done) { - state := Mux(co.messageHasData(xact), s_data_write, s_meta_write) + when(resp_data_cnt === resp_data_max) { + state := Mux(xact.hasData(), s_data_write, s_meta_write) } } is(s_data_write) { io.data.write.valid := Bool(true) - when(local_data_write_done) { - state := s_meta_write + when(io.data.write.ready) { + write_data_cnt := write_data_cnt + UInt(1) + when(write_data_cnt === write_data_max) { + state := s_meta_write + } } } is(s_meta_write) { @@ -886,12 +1040,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } is(s_grant) { io.inner.grant.valid := Bool(true) - when(!co.messageHasData(c_gnt.payload) || cgnt_data_done) { - state := Mux(co.requiresAckForGrant(c_gnt.payload), - s_busy, s_idle) + when(io.inner.grant.ready) { + cgnt_data_cnt := cgnt_data_cnt + UInt(1) + when(cgnt_data_cnt === cgnt_data_max) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } } - is(s_busy) { + is(s_ack) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { state := s_idle } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 08d7cf20..6367b44d 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,7 +3,6 @@ package uncore import Chisel._ - // Classes to represent coherence information in clients and managers abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters @@ -14,9 +13,9 @@ class ClientMetadata extends CoherenceMetadata { } object ClientMetadata { def apply(state: UInt) = { - val m = new ClientMetadata - m.state := state - m + val meta = new ClientMetadata + meta.state := state + meta } } @@ -27,10 +26,10 @@ class ManagerMetadata extends CoherenceMetadata { } object ManagerMetadata { def apply(state: UInt, sharers: UInt) = { - val m = new ManagerMetadata - m.state := state - m.sharers := sharers - m + val meta = new ManagerMetadata + meta.state := state + meta.sharers := sharers + meta } def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) } @@ -85,56 +84,59 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def releaseTypeWidth = log2Up(nReleaseTypes) def grantTypeWidth = log2Up(nGrantTypes) - def isHit (cmd: UInt, m: ClientMetadata): Bool - def isValid (m: ClientMetadata): Bool - def isHit (incoming: Acquire, m: ManagerMetadata): Bool - def isValid (m: ManagerMetadata): Bool + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + val acquireTypesWithData = Nil // Only built-in Acquire types have data for now + val releaseTypesWithData: Vec[UInt] + val grantTypesWithData: Vec[UInt] + + def isValid(meta: ClientMetadata): Bool + def isValid(meta: ManagerMetadata): Bool + + def isHit(cmd: UInt, meta: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), + clientStatesWithWritePermission.contains(meta.state), + clientStatesWithReadPermission.contains(meta.state)) + } + //TODO: Use outer protocol's clientState instead, remove this function: + def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool - def needsWriteback(m: ClientMetadata): Bool - def needsWriteback(m: ManagerMetadata): Bool + //TODO: Assumes all cache ctrl ops writeback dirty data, and + // doesn't issue transaction when e.g. downgrading Exclusive to Shared: + def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + clientStatesWithDirtyData.contains(meta.state) + def needsWriteback(meta: ClientMetadata): Bool = + needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata): ClientMetadata - def clientMetadataOnCacheControl(cmd: UInt): ClientMetadata - def clientMetadataOnFlush: ClientMetadata def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata): ClientMetadata - def managerMetadataOnFlush: ManagerMetadata - def managerMetadataOnRelease(incoming: Release, m: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, m: ManagerMetadata, dst: UInt): ManagerMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnFlush: ClientMetadata - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt - def getProbeType(a: Acquire, m: ManagerMetadata): UInt - def getProbeTypeOnVoluntaryWriteback: UInt - def getReleaseTypeOnCacheControl(cmd: UInt): UInt - def getReleaseTypeOnVoluntaryWriteback(): UInt - def getReleaseTypeOnProbe(p: Probe, m: ClientMetadata): UInt - def getGrantType(a: Acquire, m: ManagerMetadata): UInt - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt + def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnFlush: ManagerMetadata - def messageHasData(rel: TileLinkChannel): Bool - def messageUpdatesDataArray(g: Grant): Bool - def isVoluntary(rel: Release): Bool - def isVoluntary(gnt: Grant): Bool - def requiresOuterRead(acq: Acquire, m: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata): Bool - def requiresSelfProbe(a: Acquire): Bool - def requiresProbes(a: Acquire, m: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata): Bool - def requiresAckForGrant(g: Grant): Bool - def requiresAckForRelease(r: Release): Bool + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool - - def getGrantTypeForUncached(a: Acquire, m: ManagerMetadata): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -142,123 +144,105 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 1 def nProbeTypes = 2 - def nReleaseTypes = 5 - def nGrantTypes = 2 + def nReleaseTypes = 4 + def nGrantTypes = 1 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = isValid(m) - def isValid (m: ClientMetadata): Bool = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata): Bool = isValid(m) - def isValid (m: ManagerMetadata): Bool = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (outstanding.a_type != acquireReadExclusive) - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientValid), Array( - M_INV -> (m.state === clientValid), - M_CLN -> (m.state === clientValid) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (outstanding.a_type != acquireExclusive) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = m - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientValid - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers, src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = acquireReadExclusive - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = acquireReadExclusive - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck - - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, Acquire.uncachedAtomic -> probeInvalidate )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -266,137 +250,123 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 2 + def nReleaseTypes = 6 + def nGrantTypes = 1 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadExclusiveClean :: acquireReadExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) + val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata) = isValid(m) - def isValid (m: ClientMetadata) = m.state != clientInvalid - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata) = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusiveDirty)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientExclusiveClean - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(outstanding.a_type === acquireReadExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, + clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, acquireReadExclusiveClean) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusiveDirty, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), grantReadExclusive) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), Bool(true)))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -404,152 +374,133 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 + def nReleaseTypes = 6 def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { - (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = + (isRead(cmd) && outstanding.uncached) || + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush() = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - Mux(incoming.g_type === grantReadShared, clientShared, - clientExclusiveDirty))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> clientExclusiveDirty, + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), - MuxLookup(a.a_type, probeInvalidate, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + Acquire.uncachedAtomic -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -557,157 +508,135 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def nManagerStates = 2 def nAcquireTypes = 2 def nProbeTypes = 3 - def nReleaseTypes = 7 - def nGrantTypes = 4 + def nReleaseTypes = 6 + def nGrantTypes = 3 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), (m.state === clientExclusiveClean || m.state === clientExclusiveDirty), - (m.state === clientShared || m.state === clientExclusiveClean || m.state === clientExclusiveDirty)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> (m.state === clientExclusiveDirty), - M_CLN -> (m.state === clientExclusiveDirty) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, m.state)) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> Mux(outstanding.a_type === acquireReadExclusive, - clientExclusiveDirty, clientExclusiveClean), - grantReadExclusiveAck -> clientExclusiveDirty - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientShared, - probeCopy -> m.state - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(outstanding.a_type === acquireExclusive, + clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next )) } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, acquireReadShared) - } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), acquireReadExclusive, outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { - val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> releaseInvalidateData, - probeDowngrade -> releaseDowngradeData, - probeCopy -> releaseCopyData - )) - val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> releaseInvalidateAck, - probeDowngrade -> releaseDowngradeAck, - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - (g.g_type === grantReadShared || g.g_type === grantReadExclusive)) - } + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - Mux(a.a_type === acquireReadShared, - Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - grantReadExclusive)) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { @@ -715,174 +644,170 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def nManagerStates = 2 def nAcquireTypes = 3 def nProbeTypes = 4 - def nReleaseTypes = 11 - def nGrantTypes = 5 + def nReleaseTypes = 10 + def nGrantTypes = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val masterInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val masterInvalid :: masterValid :: Nil = Enum(UInt(), nManagerStates) + //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) + val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - val acquireReadShared :: acquireReadExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) - val releaseVoluntaryInvalidateData :: releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) - val grantVoluntaryAck :: grantReadShared :: grantReadExclusive :: grantReadExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val hasDataGrantTypeVec = Vec(grantReadShared, grantReadExclusive, grantReadMigratory) - val hasDataReleaseTypeVec = Vec(releaseVoluntaryInvalidateData, releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) - def isHit (cmd: UInt, m: ClientMetadata): Bool = { - Mux(isWriteIntent(cmd), Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty).contains(m.state), (m.state != clientInvalid)) - } - def isValid (m: ClientMetadata): Bool = { - m.state != clientInvalid - } - def isHit (incoming: Acquire, m: ManagerMetadata) = isValid(m) - def isValid (m: ManagerMetadata) = m.state != masterInvalid + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = { + def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = (isRead(cmd) && outstanding.uncached) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireReadExclusive && outstanding.a_type != acquireInvalidateOthers)) - } - def needsTransactionOnCacheControl(cmd: UInt, m: ClientMetadata): Bool = { - MuxLookup(cmd, (m.state === clientExclusiveDirty), Array( - M_INV -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state), - M_CLN -> Vec(clientExclusiveDirty,clientMigratoryDirty).contains(m.state) - )) - } - def needsWriteback (m: ClientMetadata): Bool = { - needsTransactionOnCacheControl(M_INV, m) - } - def needsWriteback(m: ManagerMetadata) = isValid(m) + (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - def clientMetadataOnHit(cmd: UInt, m: ClientMetadata) = ClientMetadata( - Mux(isWrite(cmd), MuxLookup(m.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), m.state)) - def clientMetadataOnCacheControl(cmd: UInt) = ClientMetadata( - MuxLookup(cmd, clientInvalid, Array( - M_INV -> clientInvalid, - M_CLN -> clientShared - ))) - def clientMetadataOnFlush = clientMetadataOnCacheControl(M_INV) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantReadShared -> clientShared, - grantReadExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireReadExclusive -> clientExclusiveDirty, - acquireReadShared -> clientExclusiveClean)), - grantReadExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireReadExclusive -> clientMigratoryDirty, - acquireReadShared -> clientMigratoryClean)) - )))) - def clientMetadataOnProbe(incoming: Probe, m: ClientMetadata) = ClientMetadata( - MuxLookup(incoming.p_type, m.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> m.state, - probeDowngrade -> MuxLookup(m.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid)) - ))) - def managerMetadataOnFlush = ManagerMetadata(masterInvalid) - def managerMetadataOnRelease(r: Release, m: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(masterValid, dir.pop(m.sharers,src)) - MuxBundle(m, Array( - r.is(releaseVoluntaryInvalidateData) -> next, + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) + + def clientMetadataOnFlush = ClientMetadata(clientInvalid) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + ClientMetadata( + Mux(incoming.uncached, clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( + acquireExclusive -> clientExclusiveDirty, + acquireShared -> clientExclusiveClean)), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( + acquireInvalidateOthers -> clientMigratoryDirty, + acquireExclusive -> clientMigratoryDirty, + acquireShared -> clientMigratoryClean)))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { + val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + MuxBundle(meta, Array( r.is(releaseInvalidateData) -> next, r.is(releaseInvalidateAck) -> next, r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next - )) - } - def managerMetadataOnGrant(g: Grant, m: ManagerMetadata, dst: UInt) = { - val cached = ManagerMetadata(masterValid, dir.push(m.sharers, dst)) - val uncached = ManagerMetadata(masterValid, m.sharers) - Mux(g.uncached, uncached, cached) + r.is(releaseInvalidateAckMigratory) -> next)) } + def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = + Mux(g.uncached, + ManagerMetadata(managerValid, meta.sharers), + ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - def isVoluntary(rel: Release) = rel.r_type === releaseVoluntaryInvalidateData - def isVoluntary(gnt: Grant) = !gnt.uncached && gnt.g_type === grantVoluntaryAck + def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, m: ClientMetadata): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), acquireReadShared) + def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = + ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + + def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + acquireShared) + + def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + outstanding.a_type) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getAcquireTypeOnSecondaryMiss(cmd: UInt, m: ClientMetadata, outstanding: Acquire): UInt = { - Mux(isWriteIntent(cmd), Mux(m.state === clientInvalid, acquireReadExclusive, acquireInvalidateOthers), outstanding.a_type) - } - def getReleaseTypeOnCacheControl(cmd: UInt): UInt = releaseVoluntaryInvalidateData // TODO - def getReleaseTypeOnVoluntaryWriteback(): UInt = getReleaseTypeOnCacheControl(M_INV) - def getReleaseTypeOnProbe(incoming: Probe, m: ClientMetadata): UInt = { + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(m.state), - releaseInvalidateDataMigratory, releaseInvalidateData), - probeDowngrade -> Mux(m.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), - probeCopy -> releaseCopyData - )) + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(meta.state === clientMigratoryDirty, + releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData)) val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> Mux(clientExclusiveClean === m.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeInvalidateOthers -> Mux(m.state === clientSharedByTwo, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(m.state != clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), - probeCopy -> releaseCopyAck - )) - Mux(needsWriteback(m), with_data, without_data) - } - - def messageHasData(msg: TileLinkChannel) = msg match { - case acq: Acquire => Mux(acq.uncached, Acquire.hasData(acq.a_type), Bool(false)) - case gnt: Grant => Mux(gnt.uncached, Grant.hasData(gnt.g_type), hasDataGrantTypeVec.contains(gnt.g_type)) - case rel: Release => hasDataReleaseTypeVec.contains(rel.r_type) - case _ => Bool(false) - } - def messageUpdatesDataArray(g: Grant): Bool = { - Mux(g.uncached, Bool(false), - Vec(grantReadShared, grantReadExclusive, grantReadMigratory).contains(g.g_type)) + probeInvalidate -> Mux(clientExclusiveClean === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(meta.state != clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck)) + Mux(dirty, with_data, without_data) } def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - def getGrantType(a: Acquire, m: ManagerMetadata): UInt = { - Mux(a.uncached, getGrantTypeForUncached(a, m), - MuxLookup(a.a_type, grantReadShared, Array( - acquireReadShared -> Mux(!dir.none(m.sharers), grantReadShared, grantReadExclusive), - acquireReadExclusive -> grantReadExclusive, - acquireInvalidateOthers -> grantReadExclusiveAck //TODO: add this to MESI for broadcast? - ))) - } - def getGrantTypeOnVoluntaryWriteback(m: ManagerMetadata): UInt = grantVoluntaryAck + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.uncached, Grant.getGrantTypeForUncached(a), + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? - def getProbeType(a: Acquire, m: ManagerMetadata): UInt = { + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = Mux(a.uncached, MuxLookup(a.a_type, probeCopy, Array( + Acquire.uncachedReadBlock -> probeCopy, + Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), + Acquire.uncachedAtomic -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( - acquireReadShared -> probeDowngrade, - acquireReadExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers - ))) - } - def getProbeTypeOnVoluntaryWriteback: UInt = probeInvalidate + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterRead(acq: Acquire, m: ManagerMetadata) = + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) - def requiresOuterWrite(acq: Acquire, m: ManagerMetadata) = + + def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - def requiresAckForGrant(g: Grant) = g.uncached || g.g_type != grantVoluntaryAck - def requiresAckForRelease(r: Release) = Bool(false) - def requiresSelfProbe(a: Acquire) = a.uncached && a.a_type === Acquire.uncachedRead - def requiresProbes(a: Acquire, m: ManagerMetadata) = !dir.none(m.sharers) && - Mux(dir.one(m.sharers), Bool(true), - Mux(a.uncached, a.a_type != Acquire.uncachedRead, - a.a_type != acquireReadShared)) - def requiresProbesOnVoluntaryWriteback(m: ManagerMetadata) = !dir.none(m.sharers) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index fe647d07..69cdc41f 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,7 +16,6 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") - val MT_CB = Bits("b111") // cache block val NUM_XA_OPS = 9 val M_SZ = 5 @@ -37,8 +36,9 @@ trait MemoryOpConstants { val M_XA_MAX = Bits("b01101"); val M_XA_MINU = Bits("b01110"); val M_XA_MAXU = Bits("b01111"); - val M_INV = Bits("b10000"); // write back and invalidate line - val M_CLN = Bits("b10001"); // write back line + val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions + val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4db133d6..5e5053be 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -136,7 +136,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { mem_acked := Bool(true) mem_gxid := io.mem.grant.bits.payload.manager_xact_id mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := co.requiresAckForGrant(io.mem.grant.bits.payload) + mem_needs_ack := io.mem.grant.bits.payload.requiresAck() } io.mem.grant.ready := Bool(true) @@ -184,22 +184,26 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) } - var mem_req_data: UInt = null - for (i <- 0 until dataBits/short_request_bits) { - val idx = Cat(cnt, UInt(i, log2Up(dataBits/short_request_bits))) + val n = dataBits/short_request_bits + val mem_req_data = (0 until n).map { i => + val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(idx) := io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := + io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) } - mem_req_data = Cat(packet_ram(idx), mem_req_data) - } + packet_ram(Cat(cnt, ui)) + }.reverse.reduce(_##_) + val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWrite(init_addr, mem_req_data), - UncachedRead(init_addr)) + UncachedWriteBlock( + addr_block = init_addr, + addr_beat = cnt, + client_xact_id = UInt(0), + data = mem_req_data), + UncachedReadBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data - io.mem.acquire.bits.header.src := UInt(params(LNClients)) // By convention HTIF is the client with the largest id - io.mem.acquire.bits.header.dst := UInt(0) // DNC; Overwritten outside module io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index d6133fc0..e7291222 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -228,7 +228,9 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { mem_cmd_q.io.enq.valid := Bool(false) mem_data_q.io.enq.valid := Bool(false) - val acq_has_data = co.messageHasData(io.uncached.acquire.bits.payload) + //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are + //acceptable Acquire types + val acq_has_data = io.uncached.acquire.bits.payload.hasData() val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) @@ -250,7 +252,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr + addr_out := io.uncached.acquire.bits.payload.addr_block has_data := acq_has_data tl_done_out := tl_wrap_out mif_done_out := Bool(false) @@ -323,8 +325,12 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(Bool(true), Grant.uncachedRead, tag_in, UInt(0), - tl_buf_in(tl_cnt_in)) + io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + g_type = Grant.uncachedReadBlock, + client_xact_id = tag_in, + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index af10d455..32ccac87 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -16,20 +16,22 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] abstract trait TileLinkParameters extends UsesParameters { - val tlAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits - val tlSubblockAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = log2Up(NUM_XA_OPS) + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlBeatAddrBits = log2Up(tlDataBeats) + val tlByteAddrBits = log2Up(tlWriteMaskBits) + val tlAtomicOpcodeBits = M_SZ val tlUncachedOperandSizeBits = MT_SZ val tlSubblockUnionBits = max(tlWriteMaskBits, - (tlSubblockAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 + (tlByteAddrBits + + tlUncachedOperandSizeBits + + tlAtomicOpcodeBits)) + 1 val co = params(TLCoherence) + val networkPreservesPointToPointOrdering = false //TODO: check physical network type } abstract class TLBundle extends Bundle with TileLinkParameters @@ -45,8 +47,12 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. // These traits are useful for type parameterization. // -trait HasPhysicalAddress extends TLBundle { - val addr = UInt(width = tlAddrBits) +trait HasCacheBlockAddress extends TLBundle { + val addr_block = UInt(width = tlBlockAddrBits) +} + +trait HasTileLinkBeatId extends TLBundle { + val addr_beat = UInt(width = tlBeatAddrBits) } trait HasClientTransactionId extends TLBundle { @@ -57,55 +63,109 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -trait HasTileLinkData extends TLBundle { +abstract trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool } // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { + // Actual bundle fields val uncached = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nUncachedAcquireTypes), co.acquireTypeWidth)) + val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - val sbAddrOff = tlSubblockAddrBits + tlUncachedOperandSizeBits - val opSzOff = tlUncachedOperandSizeBits + sbAddrOff + + // Utility funcs for accessing uncached/subblock union + val opSizeOff = tlByteAddrBits + 1 + val opCodeOff = tlUncachedOperandSizeBits + opSizeOff + val opMSB = tlAtomicOpcodeBits + opCodeOff def allocate(dummy: Int = 0) = subblock(0) - def operand_sz(dummy: Int = 0) = subblock(tlUncachedOperandSizeBits, 1) - def subblock_addr(dummy: Int = 0) = subblock(sbAddrOff, tlUncachedOperandSizeBits+1) - def atomic_op(dummy: Int = 0) = subblock(opSzOff, sbAddrOff+1) + def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) + def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) + def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + + // Other helper funcs def is(t: UInt) = a_type === t + + def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + Acquire.typesWithMultibeatData.contains(a_type) + + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + + def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = + Probe(co.getProbeType(this, meta), this.addr_block) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Grant( + uncached = this.uncached, + g_type = co.getGrantType(this, meta), + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data + ) + } } object Acquire { - val nUncachedAcquireTypes = 3 - //TODO: val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedAcquireTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(a_type: UInt) = Vec(uncachedWrite, uncachedAtomic).contains(a_type) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWrite - def requiresOuterWrite(a_type: UInt) = a_type === uncachedWrite + val nBuiltinAcquireTypes = 5 + //TODO: Use Enum + def uncachedRead = UInt(0) + def uncachedReadBlock = UInt(1) + def uncachedWrite = UInt(2) + def uncachedWriteBlock = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) + def typesWithMultibeatData = Vec(uncachedWriteBlock) + def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock + def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: + def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock - def apply(uncached: Bool, a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt, subblock: UInt): Acquire = { + def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + + // Most generic constructor + def apply( + uncached: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire acq.uncached := uncached acq.a_type := a_type - acq.addr := addr acq.client_xact_id := client_xact_id + acq.addr_block := addr_block + acq.addr_beat := addr_beat acq.data := data acq.subblock := subblock acq } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(Bool(false), a_type, addr, client_xact_id, data, UInt(0)) - } - def apply(a_type: Bits, addr: UInt, client_xact_id: UInt): Acquire = { - apply(a_type, addr, client_xact_id, UInt(0)) + // For cached types + def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { + apply( + uncached = Bool(false), + a_type = a_type, + client_xact_id = client_xact_id, + addr_block = addr_block) } + // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire acq := a @@ -113,83 +173,189 @@ object Acquire { } } +// Asks for a single TileLink beat of data object UncachedRead { - def apply(addr: UInt, client_xact_id: UInt, subblock_addr: UInt, operand_sz: UInt, alloc: Bool): Acquire = { - val acq = Acquire(Acquire.uncachedRead, addr, client_xact_id) - acq.uncached := Bool(true) - acq.subblock := Cat(subblock_addr, operand_sz, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedRead, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + subblock = alloc) } - def apply(addr: UInt, client_xact_id: UInt): Acquire = { - apply(addr, client_xact_id, UInt(0), MT_CB, Bool(true)) - } - def apply(addr: UInt): Acquire = { - apply(addr, UInt(0)) +} + +// Asks for an entire cache block of data +object UncachedReadBlock { + def apply( + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedReadBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + subblock = alloc.toUInt) } } object UncachedWrite { - def apply(addr: UInt, client_xact_id: UInt, write_mask: Bits, alloc: Bool, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedWrite, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(write_mask, alloc) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt = Acquire.fullWriteMask, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWrite, + addr_block = addr_block, + addr_beat = addr_beat, + client_xact_id = client_xact_id, + data = data, + subblock = Cat(write_mask, alloc)) } - def apply(addr: UInt, client_xact_id: UInt, data: UInt): Acquire = { - apply(addr, client_xact_id, SInt(-1), Bool(true), data) - } - def apply(addr: UInt, data: UInt): Acquire = { - apply(addr, UInt(0), data) +} + +// For full block of data +object UncachedWriteBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedWriteBlock, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(Acquire.fullWriteMask, alloc)) } } object UncachedAtomic { - def apply(addr: UInt, client_xact_id: UInt, atomic_opcode: UInt, - subblock_addr: UInt, operand_sz: UInt, data: UInt): Acquire = { - val acq = Acquire(Acquire.uncachedAtomic, addr, client_xact_id, data) - acq.uncached := Bool(true) - acq.subblock := Cat(atomic_opcode, subblock_addr, operand_sz, Bool(true)) - acq + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt): Acquire = { + Acquire( + uncached = Bool(true), + a_type = Acquire.uncachedAtomic, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) } } class Probe extends ManagerToClientChannel - with HasPhysicalAddress { + with HasCacheBlockAddress { val p_type = UInt(width = co.probeTypeWidth) + def is(t: UInt) = p_type === t + def makeRelease( + client_xact_id: UInt, + meta: ClientMetadata = co.clientMetadataOnFlush, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(false), + r_type = co.getReleaseType(this, meta), + client_xact_id = client_xact_id, + addr_block = this.addr_block, + addr_beat = addr_beat, + data = data) + } } object Probe { - def apply(p_type: UInt, addr: UInt) = { + val co = new Probe().co + def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type - prb.addr := addr + prb.addr_block := addr_block prb } + + def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { + apply(co.getProbeType(M_FLUSH, meta), addr_block) + } } + class Release extends ClientToManagerChannel - with HasPhysicalAddress + with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { val r_type = UInt(width = co.releaseTypeWidth) + val voluntary = Bool() + + // Helper funcs def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) + + def makeGrant( + manager_xact_id: UInt, + meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { + Grant( + g_type = Grant.voluntaryAck, + uncached = Bool(true), // Grant.voluntaryAck is built-in type + client_xact_id = this.client_xact_id, + manager_xact_id = manager_xact_id + ) + } } object Release { - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt, data: UInt): Release = { + val co = new Release().co + def apply( + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { val rel = new Release rel.r_type := r_type - rel.addr := addr rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat rel.data := data + rel.voluntary := voluntary rel } - def apply(r_type: UInt, addr: UInt, client_xact_id: UInt): Release = { - apply(r_type, addr, client_xact_id, UInt(0)) - } - def apply(r_type: UInt, addr: UInt): Release = { - apply(r_type, addr, UInt(0), UInt(0)) + + def makeVoluntaryWriteback( + meta: ClientMetadata, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, meta), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data) } } @@ -198,29 +364,62 @@ class Grant extends ManagerToClientChannel with HasClientTransactionId with HasManagerTransactionId { val uncached = Bool() - val g_type = UInt(width = max(log2Up(Grant.nUncachedGrantTypes), co.grantTypeWidth)) + val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + + // Helper funcs def is(t: UInt) = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(uncached, + Grant.typesWithData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(uncached, + Grant.typesWithMultibeatData.contains(g_type), + co.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + def makeFinish(dummy: Int = 0): Finish = { + val f = new Finish + f.manager_xact_id := this.manager_xact_id + f + } } object Grant { - val nUncachedGrantTypes = 3 - //TODO val uncachedRead :: uncachedWrite :: uncachedAtomic :: Nil = Enum(UInt(), nUncachedGrantTypes) - def uncachedRead = UInt(0) - def uncachedWrite = UInt(1) - def uncachedAtomic = UInt(2) - def hasData(g_type: UInt) = Vec(uncachedRead, uncachedAtomic).contains(g_type) + val nBuiltinGrantTypes = 5 + //TODO Use Enum + def voluntaryAck = UInt(0) + def uncachedRead = UInt(1) + def uncachedReadBlock = UInt(2) + def uncachedWrite = UInt(3) + def uncachedAtomic = UInt(4) + def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) + def typesWithMultibeatData= Vec(uncachedReadBlock) - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, data: UInt): Grant = { + def apply( + uncached: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { val gnt = new Grant gnt.uncached := uncached gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat gnt.data := data gnt } - def apply(uncached: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt): Grant = { - apply(uncached, g_type, client_xact_id, manager_xact_id, UInt(0)) + + def getGrantTypeForUncached(a: Acquire): UInt = { + MuxLookup(a.a_type, Grant.uncachedRead, Array( + Acquire.uncachedRead -> Grant.uncachedRead, + Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, + Acquire.uncachedWrite -> Grant.uncachedWrite, + Acquire.uncachedWriteBlock -> Grant.uncachedWrite, + Acquire.uncachedAtomic -> Grant.uncachedAtomic + )) } } @@ -260,10 +459,17 @@ object TileLinkIOWrapper { } } -// Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { +abstract trait HasArbiterTypes { + val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with + HasClientTransactionId with + HasTileLinkData +} +// Utility functions for constructing TileLinkIO arbiters +abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule + with HasArbiterTypes { // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back @@ -271,10 +477,10 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule { def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithId] + def hookupClientSource[M <: ClientSourcedWithIdAndData] (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], out: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) out <> arb.io.out ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { @@ -336,35 +542,26 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { } // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = +abstract trait AppendsArbiterId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id >> UInt(log2Up(arbN)) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id(log2Up(arbN)-1,0).toUInt + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId { - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - in.client_xact_id - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait PassesId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId { - val arbN: Int - def clientSourcedClientXactId(in: ClientToManagerChannel with HasClientTransactionId, id: Int) = - UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerToClientChannel with HasClientTransactionId) = - UInt(0) - def arbIdx(in: ManagerToClientChannel with HasClientTransactionId) = - in.client_xact_id +abstract trait UsesNewId extends HasArbiterTypes { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Mix-in id generation traits to make concrete arbiter classes diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d2531a9e..c05aafee 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -67,7 +67,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && co.messageHasData(acquire.bits.payload) + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } @@ -87,8 +87,8 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends // Queue to store impending Voluntary Release data val release = io.inner.release - val voluntary = co.isVoluntary(release.bits.payload) - val vwbdq_enq = release.fire() && voluntary && co.messageHasData(release.bits.payload) + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } @@ -117,9 +117,10 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = co.messageHasData(m.payload) + def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat io.inner.grant <> grant_arb.io.out grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } @@ -130,7 +131,7 @@ class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - co.messageHasData(io.outer.acquire.bits.payload) && + io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), @@ -156,32 +157,39 @@ abstract class XactTracker(innerId: String, outerId: String) extends Module { val has_release_match = Bool(OUTPUT) } - val c_acq = io.inner.acquire.bits - val c_rel = io.inner.release.bits - val c_gnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits - val m_gnt = io.outer.grant.bits + val cacq = io.inner.acquire.bits + val crel = io.inner.release.bits + val cgnt = io.inner.grant.bits + val cfin = io.inner.finish.bits + val macq = io.outer.acquire.bits + val mgnt = io.outer.grant.bits } class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_ack :: s_busy :: Nil = Enum(UInt(), 4) + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr = Reg(io.inner.release.bits.payload.addr.clone) + val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } + val xact = Release( + voluntary = Bool(true), + r_type = xact_r_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block) val collect_inner_data = Reg(init=Bool(false)) + // TODO: assert that all releases have full blocks of data val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && co.messageHasData(io.inner.release.bits.payload), tlDataBeats) + Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) io.has_acquire_conflict := Bool(false) - io.has_release_match := co.isVoluntary(c_rel.payload) + io.has_release_match := crel.payload.isVoluntary() io.outer.grant.ready := Bool(false) io.outer.acquire.valid := Bool(false) @@ -193,21 +201,19 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(Bool(false), - co.getGrantTypeOnVoluntaryWriteback(co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId)) + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - io.outer.acquire.bits.payload := Bundle(UncachedWrite( - xact_addr, - UInt(trackerId), - xact_data(outer_data_cnt)), + io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) when(collect_inner_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_data(inner_data_cnt) := c_rel.payload.data + xact_data(inner_data_cnt) := crel.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -216,72 +222,107 @@ class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, oute is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := c_rel.header.src - xact_r_type := c_rel.payload.r_type - xact_addr := c_rel.payload.addr - xact_client_xact_id := c_rel.payload.client_xact_id - xact_data(UInt(0)) := c_rel.payload.data - collect_inner_data := co.messageHasData(c_rel.payload) - state := Mux(co.messageHasData(c_rel.payload), s_outer, s_ack) + xact_src := crel.header.src + xact_r_type := crel.payload.r_type + xact_addr_block := crel.payload.addr_block + xact_client_xact_id := crel.payload.client_xact_id + xact_data(UInt(0)) := crel.payload.data + collect_inner_data := crel.payload.hasMultibeatData() + state := Mux(crel.payload.hasData(), s_outer, + Mux(crel.payload.requiresAck(), s_ack, s_idle)) } } is(s_outer) { io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { state := s_ack } + when(outer_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } } is(s_ack) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { state := s_idle } + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } } } } class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_busy :: Nil = Enum(UInt(), 6) + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr = Reg(io.inner.acquire.bits.payload.addr.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) + val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) + val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire(xact_uncached, xact_a_type, xact_addr, xact_client_xact_id, UInt(0), xact_subblock) + val xact = Acquire( + uncached = xact_uncached, + a_type = xact_a_type, + client_xact_id = xact_client_xact_id, + addr_block = xact_addr_block, + addr_beat = xact_addr_beat, + data = UInt(0), + subblock = xact_subblock) val collect_inner_data = Reg(init=Bool(false)) + //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && co.messageHasData(io.inner.acquire.bits.payload), tlDataBeats) + Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && co.messageHasData(io.outer.acquire.bits.payload), tlDataBeats) + Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) + val (cgnt_data_cnt, cgnt_data_done) = + Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) val probe_flags = Reg(init=Bits(0, width = nClients)) val curr_p_id = PriorityEncoder(probe_flags) - val pending_outer_write = co.messageHasData(xact) + val pending_outer_write = xact.hasData() val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) val probe_initial_flags = Bits(width = nClients) probe_initial_flags := Bits(0) // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = co.requiresSelfProbe(io.inner.acquire.bits.payload) - val myflag = Mux(probe_self, Bits(0), UIntToOH(c_acq.header.src(log2Up(nClients)-1,0))) + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr, c_acq.payload.addr) && + io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && (state != s_idle) && !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr, c_rel.payload.addr) && - !co.isVoluntary(c_rel.payload) && + io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + !crel.payload.isVoluntary() && (state != s_idle) - val outer_write_acq = Bundle(UncachedWrite(xact_addr, UInt(trackerId), xact_data(outer_data_cnt)), + val outer_write_acq = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = outer_data_cnt, + data = xact_data(outer_data_cnt)), { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWrite(xact_addr, UInt(trackerId), c_rel.payload.data), + val outer_write_rel = Bundle(UncachedWriteBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = crel.payload.addr_beat, + data = crel.payload.data), + { case TLId => outerId }) + val outer_read = Bundle(UncachedReadBlock( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block), { case TLId => outerId }) - val outer_read = Bundle(UncachedRead(xact_addr, UInt(trackerId)), { case TLId => outerId }) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits.payload := outer_read //default @@ -290,16 +331,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe(co.getProbeType(xact, co.managerMetadataOnFlush), xact_addr) + io.inner.probe.bits.payload := xact.makeProbe() io.inner.grant.valid := Bool(false) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := Grant(xact_uncached, - co.getGrantType(xact, co.managerMetadataOnFlush), - xact_client_xact_id, - UInt(trackerId), - UInt(0)) // Data bypassed in parent + io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) @@ -307,7 +344,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri when(collect_inner_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := c_acq.payload.data + xact_data(inner_data_cnt) := cacq.payload.data } when(inner_data_done) { collect_inner_data := Bool(false) } } @@ -315,17 +352,18 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - val needs_outer_write = co.messageHasData(c_acq.payload) - val needs_outer_read = co.requiresOuterRead(c_acq.payload, co.managerMetadataOnFlush) - when( io.inner.acquire.valid ) { - xact_uncached := c_acq.payload.uncached - xact_a_type := c_acq.payload.a_type - xact_addr := c_acq.payload.addr - xact_client_xact_id := c_acq.payload.client_xact_id - xact_data(UInt(0)) := c_acq.payload.data - xact_subblock := c_acq.payload.subblock - xact_src := c_acq.header.src - collect_inner_data := co.messageHasData(c_acq.payload) + val needs_outer_write = cacq.payload.hasData() + val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) + when(io.inner.acquire.valid) { + xact_uncached := cacq.payload.uncached + xact_a_type := cacq.payload.a_type + xact_addr_block := cacq.payload.addr_block + xact_addr_beat := cacq.payload.addr_beat + xact_client_xact_id := cacq.payload.client_xact_id + xact_data(UInt(0)) := cacq.payload.data + xact_subblock := cacq.payload.subblock + xact_src := cacq.header.src + collect_inner_data := cacq.payload.hasMultibeatData() probe_flags := probe_initial_flags release_count := PopCount(probe_initial_flags) state := Mux(probe_initial_flags.orR, s_probe, @@ -341,9 +379,9 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } // Handle releases, which may have data to be written back - io.inner.release.ready := !co.messageHasData(c_rel.payload) || io.outer.acquire.ready + io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready when(io.inner.release.valid) { - when(co.messageHasData(c_rel.payload)) { + when(crel.payload.hasData()) { io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_write_rel when(io.outer.acquire.ready) { @@ -364,14 +402,12 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri } } } - is(s_mem_read) { + is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := Bool(true) io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) - } + when(io.outer.acquire.ready) { state := s_mem_resp } } - is(s_mem_write) { + is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) io.outer.acquire.bits.payload := outer_write_acq @@ -379,17 +415,22 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri state := Mux(pending_outer_read, s_mem_read, s_make_grant) } } - is(s_make_grant) { + is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) io.inner.grant.valid := Bool(true) when(io.inner.grant.ready) { - state := Mux(co.requiresAckForGrant(io.inner.grant.bits.payload), s_busy, s_idle) + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) } } - is(s_busy) { // Nothing left to do but wait for transaction to complete - when(io.outer.grant.valid && m_gnt.payload.client_xact_id === UInt(trackerId)) { + is(s_mem_resp) { // Wait to forward grants from outer memory + when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { io.inner.grant.valid := Bool(true) } - when(io.inner.finish.valid && c_ack.payload.manager_xact_id === UInt(trackerId)) { + when(cgnt_data_done) { + state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { state := s_idle } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index c28778aa..14575aa5 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,7 +34,7 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int, doSer: T => Bool) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) @@ -65,12 +65,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(doSer(io.in.bits.payload)) { + when(io.in.bits.payload.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !doSer(io.in.bits.payload) + io.done := !io.in.bits.payload.hasData() } when(active) { io.out.bits := rbits From e6491d351f8d9141dbfcf39b30b569ede8e4a050 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 00:22:21 -0800 Subject: [PATCH 281/688] Offset AMOs within beat and return old value --- uncore/src/main/scala/cache.scala | 33 +++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6651056f..609c3b33 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -166,12 +166,15 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = params(TLDataBits)/rowBits - val refillCycles = refillCyclesPerBeat*params(TLDataBeats) + val refillCyclesPerBeat = tlDataBits/rowBits + val refillCycles = refillCyclesPerBeat*tlDataBeats require(refillCyclesPerBeat == 1) + val amoAluOperandBits = params(AmoAluOperandBits) + require(amoAluOperandBits <= tlDataBits) } abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters + abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { val (cnt, cnt_done) = @@ -750,7 +753,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result + Reg(io.inner.acquire.bits.payload.data.clone) + } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -803,13 +808,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { val old_data = incoming.data val new_data = buffer(incoming.addr_beat) - amoalu.io.lhs := old_data - amoalu.io.rhs := new_data - val wmask = FillInterleaved(8, xact.write_mask()) - buffer(incoming.addr_beat) := - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out, + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.uncachedAtomic), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - wmask & new_data | ~wmask & old_data, old_data)) + FillInterleaved(8, xact.write_mask()), + UInt(0, width = tlDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result } //TODO: Allow hit under miss for stores @@ -860,7 +871,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St manager_xact_id = UInt(trackerId), meta = xact_meta.coh, addr_beat = cgnt_data_cnt, - data = xact_data(cgnt_data_cnt)) + data = Mux(xact.is(Acquire.uncachedAtomic), + xact_data(tlDataBeats), + xact_data(cgnt_data_cnt))) io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) From 6141b3efc53a2c27a50ce28ce3200a79bcc1eb63 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:02:06 -0800 Subject: [PATCH 282/688] uncached -> builtin_type --- uncore/src/main/scala/cache.scala | 14 ++--- uncore/src/main/scala/coherence.scala | 80 +++++++++++++-------------- uncore/src/main/scala/memserdes.scala | 2 +- uncore/src/main/scala/tilelink.scala | 47 ++++++++-------- uncore/src/main/scala/uncore.scala | 6 +- 5 files changed, 75 insertions(+), 74 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 609c3b33..794e3447 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -747,7 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) @@ -760,7 +760,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -796,7 +796,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_uncached || xact.allocate() + //val do_allocate = !xact_builtin_type || xact.allocate() val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() @@ -921,7 +921,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St is(s_idle) { io.inner.acquire.ready := Bool(true) when( io.inner.acquire.valid ) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat @@ -947,9 +947,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit val _needs_probes = _tag_match && co.requiresProbes(xact, coh) val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.uncached || - xact.is(Acquire.uncachedReadBlock) || - xact.is(Acquire.uncachedWriteBlock) + val full_block = !xact.builtin_type || + xact.hasMultibeatData() || + cgnt.payload.hasMultibeatData() read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 6367b44d..201e67ab 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -174,7 +174,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.uncached, clientInvalid, clientValid)) + ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata(Mux(incoming.p_type === probeInvalidate, @@ -189,7 +189,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -218,27 +218,27 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, Acquire.uncachedRead -> probeCopy, Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate - )), probeInvalidate) + Acquire.uncachedAtomic -> probeInvalidate)), + probeInvalidate) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -271,7 +271,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -286,7 +286,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, clientExclusiveClean))) @@ -306,7 +306,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -338,10 +338,10 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), grantExclusive) + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -356,15 +356,15 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), Bool(true)))) + Mux(a.builtin_type, a.hasData(), Bool(true)))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -396,7 +396,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid(meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -413,7 +413,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -435,7 +435,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -467,13 +467,13 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -490,15 +490,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -530,7 +530,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -548,7 +548,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> Mux(outstanding.a_type === acquireExclusive, @@ -571,7 +571,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -603,13 +603,13 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive)) def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -626,15 +626,15 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } @@ -666,7 +666,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isValid (meta: ManagerMetadata) = meta.state != managerInvalid def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.uncached) || + (isRead(cmd) && outstanding.builtin_type) || (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = @@ -690,7 +690,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = ClientMetadata( - Mux(incoming.uncached, clientInvalid, + Mux(incoming.builtin_type, clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( @@ -725,7 +725,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.uncached, + Mux(g.builtin_type, ManagerMetadata(managerValid, meta.sharers), ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) @@ -774,14 +774,14 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, Grant.getGrantTypeForUncached(a), + Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), MuxLookup(a.a_type, grantShared, Array( acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.uncached, + Mux(a.builtin_type, MuxLookup(a.a_type, probeCopy, Array( Acquire.uncachedReadBlock -> probeCopy, Acquire.uncachedWriteBlock -> probeInvalidate, @@ -799,15 +799,15 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_PRODUCE -> probeDowngrade)) def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.uncached, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.uncached, a.hasData(), a.a_type != acquireShared))) + Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e7291222..e58dd08d 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -325,7 +325,7 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { when(tl_wrap_in) { active_in := Bool(false) } } - io.uncached.grant.bits.payload := Grant(uncached = Bool(true), + io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), g_type = Grant.uncachedReadBlock, client_xact_id = tag_in, manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 32ccac87..3fc0e4e7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -76,11 +76,11 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val uncached = Bool() + val builtin_type = Bool() val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) val subblock = Bits(width = tlSubblockUnionBits) - // Utility funcs for accessing uncached/subblock union + // Utility funcs for accessing subblock union val opSizeOff = tlByteAddrBits + 1 val opCodeOff = tlUncachedOperandSizeBits + opSizeOff val opMSB = tlAtomicOpcodeBits + opCodeOff @@ -94,13 +94,13 @@ class Acquire extends ClientToManagerChannel // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = uncached && Acquire.typesWithData.contains(a_type) + def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && uncached && + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = uncached && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = Probe(co.getProbeType(this, meta), this.addr_block) @@ -111,7 +111,7 @@ class Acquire extends ClientToManagerChannel addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { Grant( - uncached = this.uncached, + builtin_type = this.builtin_type, g_type = co.getGrantType(this, meta), client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id, @@ -140,7 +140,7 @@ object Acquire { // Most generic constructor def apply( - uncached: Bool, + builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, @@ -148,7 +148,7 @@ object Acquire { data: UInt = UInt(0), subblock: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.uncached := uncached + acq.builtin_type := builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block @@ -160,7 +160,7 @@ object Acquire { // For cached types def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { apply( - uncached = Bool(false), + builtin_type = Bool(false), a_type = a_type, client_xact_id = client_xact_id, addr_block = addr_block) @@ -181,7 +181,7 @@ object UncachedRead { addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedRead, client_xact_id = client_xact_id, addr_block = addr_block, @@ -197,7 +197,7 @@ object UncachedReadBlock { addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedReadBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -214,7 +214,7 @@ object UncachedWrite { write_mask: UInt = Acquire.fullWriteMask, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWrite, addr_block = addr_block, addr_beat = addr_beat, @@ -233,7 +233,7 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedWriteBlock, client_xact_id = client_xact_id, addr_block = addr_block, @@ -253,7 +253,7 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - uncached = Bool(true), + builtin_type = Bool(true), a_type = Acquire.uncachedAtomic, client_xact_id = client_xact_id, addr_block = addr_block, @@ -308,6 +308,7 @@ class Release extends ClientToManagerChannel // Helper funcs def is(t: UInt) = r_type === t def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + //TODO: Assumes all releases write back full cache blocks: def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) @@ -317,7 +318,7 @@ class Release extends ClientToManagerChannel meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { Grant( g_type = Grant.voluntaryAck, - uncached = Bool(true), // Grant.voluntaryAck is built-in type + builtin_type = Bool(true), // Grant.voluntaryAck is built-in type client_xact_id = this.client_xact_id, manager_xact_id = manager_xact_id ) @@ -363,19 +364,19 @@ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val uncached = Bool() + val builtin_type = Bool() val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) // Helper funcs def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(uncached, + def hasData(dummy: Int = 0): Bool = Mux(builtin_type, Grant.typesWithData.contains(g_type), co.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(uncached, + Bool(tlDataBeats > 1) && Mux(builtin_type, Grant.typesWithMultibeatData.contains(g_type), co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = uncached && (g_type === Grant.voluntaryAck) + def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { val f = new Finish @@ -396,14 +397,14 @@ object Grant { def typesWithMultibeatData= Vec(uncachedReadBlock) def apply( - uncached: Bool, + builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.uncached := uncached + gnt.builtin_type := builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -452,9 +453,9 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](uncached: UncachedTileLinkIO) = { + def apply[T <: Data](utl: UncachedTileLinkIO) = { val conv = Module(new TileLinkIOWrapper) - conv.io.in <> uncached + conv.io.in <> utl conv.io.out } } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c05aafee..cc4755f0 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -260,7 +260,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_uncached = Reg(io.inner.acquire.bits.payload.uncached.clone) + val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) @@ -268,7 +268,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } val xact = Acquire( - uncached = xact_uncached, + builtin_type = xact_builtin_type, a_type = xact_a_type, client_xact_id = xact_client_xact_id, addr_block = xact_addr_block, @@ -355,7 +355,7 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val needs_outer_write = cacq.payload.hasData() val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) when(io.inner.acquire.valid) { - xact_uncached := cacq.payload.uncached + xact_builtin_type := cacq.payload.builtin_type xact_a_type := cacq.payload.a_type xact_addr_block := cacq.payload.addr_block xact_addr_beat := cacq.payload.addr_beat From 57340be72b7bb9dcdd8f38b759df4044a77812da Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 2 Feb 2015 01:11:13 -0800 Subject: [PATCH 283/688] doc update --- uncore/doc/TileLink0.3.1Specification.pdf | Bin 0 -> 172795 bytes uncore/doc/TileLink3.0Specification.pdf | Bin 123869 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 uncore/doc/TileLink0.3.1Specification.pdf delete mode 100644 uncore/doc/TileLink3.0Specification.pdf diff --git a/uncore/doc/TileLink0.3.1Specification.pdf b/uncore/doc/TileLink0.3.1Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..23666814d9e8738924e59e5f9f8872e7fa58d725 GIT binary patch literal 172795 zcmb5U1yEeU*6$lExJ!Z$?gI=uxVyWC4DLEO2@)*0J0Up1-6dFpdxAr7cZUFPIP%?d z>c02t-YTklX6?1Q|J`ftu08A@otl&k8z+z(jc#r4U>Xg?0pI{QncAWW39)N=J6izQ zB_U9Vy^|FhyBfsG!VSO)yRLyIB7$b&X#U~{dNKT0z>D9Dx2uIC^k44m8WwI&?yhFA zc-XjVu1;o}7Em{UAvFKrLQkWjO#K z(7z?H%Q(TNfLUc=R<3_o*=0BZAnt#C1OY%i|N6)U0P+4i3?`mk*}~igBH`o(Fnn1s zFDD4V2L{0=6L)lUdQtX2%m0S~ja}2-6#9?tP_l8fLt~e4ay7Sb{bw)-EKI|~3<@yh z<x??;0ATIfH| zyTnkuesDcybHpK|)O7z5%eo2M)pW0_Oh~DZ{ct)`-4vrh_f1PG-2{2xVS zI87fSa(VyU4nHsmvzPndp>UBtTQ{D5M`W#j`b2blca!sUYIv(~c5u`6m%wR%@1&os z^W*b_>NAPN@$>c7!|E549~|#oezXK8QH!nn6>4o@X&}jARizKjiGNH4JQ8lm2@c=@ zEZS{j1ykaXW#`3g@cp-momXzuPMdQq;D^JOmshXC^ zNJR2?xw!pn$$S2nRoxM#AH7)EXnp@gGl|Wm{4SCQsW>308z|oE$e#;_&n#$|(#oo?!egNpp?m=%Hh_wnVV>c9f`B#%A;<`!S{Z2s6} zJ$d2upP!yZX)o?N^fR_sPsrClUhbDWZ z;Tn@N=+WePf0?&XHyS0a%cO?2jD)ajru47_CaU-p3NjM=kcbLD;0yQ#{bj{`18v3( zW3g|Z8WCh0Y7`$;Wu0oIGxh&6N%ba_B@3|7nEWQb{Y37|kd}2WD73d#>;1&V%;riM zk|%z>WBp}xNV&4&f(Tn-fNsjfU}`T zT_2T*Ki0t8f{6E~ywaSDQ+1e928HtISB)ofr{G!k1&!BHRi7aYTVc2MZHYYWZI1zr z7m_zL-aa1?)cuMcYK|ER66G`Kw#cQtttM^|q$)tUCl$c^9@5%R1@+VO+=uTS-Av?GuuspJTM$X*@H*LhNJV@GFK0A6v0IdZ^W96H(j`4B z1p+A@+SIMT@H@>Tswg^tjhjiX?lTzEf6Md;5K6Z|whyzu9LAbE-Y!oi=Xh))0D&Wr z!&{w9X-5OUaR^TlEVhP`w~mkT zH{I5qE%)c4o3dp6+{%Gdzq$}@eV94N6~9h%BQ?+J8#F%au>Tx_v)KY2Hnz=ATg*8W z6RJ>gxjoqRghJY_g$(7!fRdINlKm!=G~moo zq8q%{y1gW3>0;!Rf~nvyy(1F|2TR^D#3%@4@I0;p_qWanzU28FS^&Z0b~3K+o^@W)~PJjh4b=3TsN$GwPS6r<8wcJA(L}^w1B0JoZbmj zkH_f09Xk&JC4%k!SAJIkT3b#dSAGqFC7;k$1FoCdE~(~zT>J`^Y8A<6<4P>V|7aYM z&TG<3`2DW)Ol1)J?DS}D3!A+=GHjlI>0d}|#;4l?Esc_u)B&+=wUJCZ+ ziQl2`U2eAMO6kIHLPGNkiz9ME^Jx=PC!Whg2qY{SxM%O1PsTS%m+g>4gmr*QohB!u z{TNT078Hg#h|PKC^3B{_o`qLv+~NTejbSONbsAuu>i{X1j7A(gmc;#M&CEYrqp zxs&RA?4ot;GhN0tQWY1@5kj6OhS}%FxyHIp3v)779w1E}1T|wkD}-^3OU{(*-5~>3 zYwBHYpLkD<^WjS9w#D&KaTwmV_2d3PxSt)~&Rm>*I?e0b+zT=Pw;tW1>!6Zbe;+3*NYZG`u}Yf%Z*#A*NKw823y z=`QS|e&hC3WmTbS#Y4tt^owVw%`BwdGIDxel^U6Ks|0MvK(NtycQVES_q(7jme;uk zf6ZOL4_WGnFUkTJR$pZr&=IE_(r^M5v|(xy!2Xd)^p(fSL;`y@Ov_lpXy4*vs8r{& z=Bs%ulx(NigI~+Ji1Hf7h7jjI~?TXPsoQR_TompGXY z1;=t1E@>Tze}a>;h6~W@MvhA)u#O1#K6(VpiBFcPu^zAy{46Uplaz3Avo(*ga=p${ zr^m~0Q=xCX+{==3^&a@qe@I1D$NBSZ4D+VK0Sb;ho3!<(fvamJ73D*)FS>g86k>Q1 zj(&)Gk8`<%Obzzr;b~zUv)cFUtPlmUbd{@_1ZYlFq1Cnmt*xDKRH5^UqQYij;a-Um z)iZBXMwaj!Yen?_{E+&>qRvDXhquNWQp2d!U|C%``pg4JuAn&z&Y7u!4{c&*#80DE zssU#&U3Dt=y}fo{I<`b20dWFP2DX_obmch3?i;(%?LCR;qvRo|l-6F5T%i;;sxb~B zk9A{10qHi+^r<6tYnF+0z(B~r(+vgr%5Q~(%?fuJ;#UV zriT>emd|}eP=}Zq)I1{KS=g)oZl`MW7t^Xw*++Zk!%1cvB71VeVSe9id>Cf1N0z$S z@0g;dhTURxh#cut9)7^&mA@r9`nouajFsSHV3;ArW$_vy7@}I z06;x336?s&mK;aEDKtTMqSXG@7yx2I-{cr!fh#RBu3LNi#kbWyM&e>PiZ{QBJ^xslQB6yX6){L* zq2!e%qo53wMT{|R1OUjdwY)Pok;XofFyeC6wNV<=LNEBBLUXhFu+rUe!4YjeCsPmx zbjioLAek36eqSu!!d`w#M5JoEMx0y`>1ka{Pm!2a3jaljOB};r%aW(RdtI@Ttc%dk zUg4_{76911U{pBar&ilPmiSDz=*5pdR5OZ_q~gE+230~34=ZG%M8WY-+74Wt3a@@l zl5tqj`kXFmu_P5iBJ}*1;-UrZhH)Pv7m?~zLt=sKrM>KtZnFhiE0*c75WDo?v;2yS zh@+77P&>_Aw{f;5+^E4Nq7#%Kmu!Y)h^`bednXJFb+2$U9f@ngJdH@#e;RZwqH)3* zf0>3TmaS}9^Wrl zee9c*{LSjRkqAj&XAmJK63w|a*P}`T=0qt{8mVuLRapf>ZOS^#5!44mYg-i?L$s91mhMtI$h%o8^1bUv zTnbT@Sa)sTh5`NdK}O>@Md3I(ae^SS==IwI4A_0imS}94_*k-~%z&p|jGn{o6_Gi| zpgcMm+$Ulaz9KQelT>dP)UK~~`n%_HCTAMAqpiI#(;{X4n`l2@DuZ>T_sW;A%INEf z-dQ>}XIw@qaMUKZYp!{3=DQ^nz?u+acQ82IkM83G-p!G3Y&~UAGoLEhS&wltn;Y4ZP3F_ZJGme0hcWIA`KA4?C6JD!Zn8Eg z7&qxoosN>vp`=19=23cjQ=*6sJlG|oT0A!Xj-oCj${J^|n;lo(FV{MuinDd08IH#f zJ`a$~5uJ*$^i)v~8BVwIxHdXd8~u!}IYE}2w8yAbXjssv)8QVMDWIAMz1_>(-FO-?9ZZI!%v620j-O?2}rzj3gP$I5Z`L;r*flT(aW=u`yx2cwuI46f| zD$ro!GmqPOpZh2vB2K?H@ub=4HkyfTkVOca1EK$*VmiRqX(ZhxZ-it#d4e+bMcUU!B9dn1Lg>e1>+ z_6suFuzC_QjBVGf@&LmSqb}P_!=|N|usMztQAbI9UuEdOw0!Oh0A;;mob{>}@F@nq7kIr;bsIHj0uPy`r zvKpIBy#YSf(si6ucjJpmBl-Xfdf*CFB(8t~83Wz)9$+hyeyv>E6L>LBZHnT_WWk`$ zEP!opXi1gq&fIKh$;qop`B;`{GsTPxYWTQj1yVXeuN$5lGr(>4c^*>H6*AM8Xeo7P;k-)>?zCdJ0Pv!@+v^ok%S7{X)Peuk` z=OLuAG|Q!^HBTZ8CSyt!O8?kTvaS7W5h?m-BGoJxyU=*8FV9zeh@`hBP|kqk1Fji= z7P7&oZN&x=BAu;iVHDb>%RyV5rg0<+KsXX^&FE$?tzaR^I?x%OVHtPl`d%fzei?%uXPn)yfRvNRU9v&`ADuIayeh!Q=pRZCGN4xc z_2ynwHH5O^V2I@|Dz&kD z%s}aW#DQEYT=U_Q??1(8!bX>a$Vf=Qhjy>s4Vj?h(4+_>X?N~YK!}HgoGFu*GJ&$U zoIq)!3QoG7xV%pGim;ZN4{n~PgR4A|U;#Z_)(O9Ol3}EkwOddA_MSD6L5Kr? z>~~9s+V`1KS@h-eZ!v3Rx?^5ng$Cio&qTJf7M8Y3D}J}rXzWcC6A3@=lvdzN6-!q( z;y*~uiqH~C1L;jd4w?8oT`$cf(M+c-pd@dS3B~cO2xO(CcDH6l0P7!uKVC31gqId6 zG$A-p?50#c95mF8Ec+~vfOs+DHo+XD ztf9U3RmtfbWh*&ZLHZi^n*@hnK6$EiYXuYh*f*N9R_}PL`}}D%H7UqIE?E-u*r@vI z)rVg~nG51UKlU{;o2b4-istqDUoUW~4_g9@eCeP#f_i^KLiSN`KM>Vj9+r^ZXy*VH zmOGhG`B-%h8kE^?w2LL(+^3;>ckpc{e~#N5wPQYbeB5g?q{Zn>URQVZeCH&mB@@`( z!~R%Q>&)^)NL$79+YMJ|bm6k<4sXlR#~*wRpqmHxWykyG$lOExb6{w#vc;FCO#4K^ zGoVQO0fbO88~xks26Cj6;IRyE=G3_PdKY&*HO&ul|N z0eSJ-z!&g=sDsjtU+*w}+KbVW;G`Z1iZJz&T2WY!Gn4Fl27O?|6 z4`&t0!`>AGx9^EQcThS0k(mx0Z6T>D`h+Rv(~N?Jn4*gHNwrEP9- zTaj&Uk{>Hzf3{iJKQpChP@9eX#;kriD)2V9#ZNr`BmYQb#Mv&$mLa?cbn?@B+M_-| z<@E3y0iS)Ap4R?4Ir&^R>e;aSAL0VS%x`&fwIHvyw+zuX^TLgw&9uNG&h z8O8GZL${`@3up4`aIg(+&5EV@#4fKY(lH~?Zx=J-+5!hAsExDRly$Zp)Vs4iIPP+G z{fh9VM6chp#KoXZ+2XEP$3J48eD8i29>#mMSA$i9L=SR6AN-E#u#z^gIe^lQ^oC@< z{!fc28FL{^#*ntPxhFl~gbVvgar_F4ObN9kGFeIa4-smNY8DU;!(BkUpzf(sr`8be zE32DN7^9?qXKxTDxgHdsZzyj)M*8kz_hOrS@RRiY#nD*7+-BZ*Ti-TY@Oe|bu-%|p zwua)DS2+AAID1WbvU$<_yYM%1e#jtNqQ~E>Re20;vg^i^=C(jpa{=-CVSnYm+EI8} zB%@1nKC_uJ_64dYWBeL2gB3z$71I8U@GZ5)s?T~@1I_fN^9!Rqs_>Efd2+T?pOM_) zRBW}E0uqYj$hTycID7;yO+ttmxm@{&^K<$4%=pNid8Xm=%&=LxI%vj!DRD{1>eVmJ z^x{u5h1ANpf$QJDp0k;=@IWAD#5yFyQQAJo@;iDvqFpP?>f^c%ogY`Xz96W~3pwbc z7G|tBE#GHtYodCme0}|eQ`nZS_fRAfs%tf0|8?v22SF^NzBCJgp{eE^dP`Y-gARP- zqHmkL=+Fs4z0;)$8MoWj=AG@nynEK3{a#OxK4Ct06S+oW?xfCNhs$1=4&c4h?agV~ z9Y(0%T)heY)%c|!w9W2MOcF0nx*+O@D|2X_AN!Bi*VCSLNgPG(B776DdQ7{wH9WCo zBW-fX0{HxPeMdDPDm`k@8T8GONR}G8x2~KQhsXXyWyxswRAp}6a`~BLdp}JP+3|Hv z=k$8DBj8U&e=n$)#Q324_*i~bEV$uM50l?e{|c^ZA9emlT%_u%&WyKv-9Um}8O~q# zx*%8O@BO-&%2(UD`e~0dqdh;gDqnkNx|1nq!=3>e7gim9ZGAky2=}vPkWKBVP`m9@ za2?l|5WzCO9*b$Ypy81fOY(cww0j0MZ#}|mtJ!`rXa&2{$W08en`pG$5IA>I&jvz&#dQ6L zSPK{_5i^{!?wgS;iLOfY_RBlhGX4cidvUiiucM#D>e1x1H0O@Sab?>hxk#u>_>l?( zUt5`rj~U6a+hKv*!)-Oa+sCk`JIK#a)FCY6v$ih$aJNR~;CJ4GII%S%pGEkNjjZT4 zJVkO>Dq{h6iy<<&_hhay)03$YE(0C3sxqEtVKlG!zoUp9E~0c$!k$GCzvCy4hnPdV zQJdz-;G8*vy$7y~4BNM<7Bd}p*(4Xn^cjr@MISUay!H?$D-JY)1?y=n1+`Tfo^pjz zP1)i$;+k7OkUSQ}P!0rlmdmqzXFc8PpaiSBYvbAObrX1>N^ae(mdnALOqvYgPnN1C zCf$3M#=EvVB4W_0J|h#w6H7u$k!fn8DHA~Cft%o1QI#dC)p&`Mu z!PH)!U$e9goZ`3>q5|Pb%wmLrJtEKUy-)KuE4c{a) zZNrxme5B1wG!m9TqXd;M$5VG&t9QYz`7NU6DOAP1CXI2uK>X$eDEgjj^#|{UG>DEA zb#|5<&|12`Pf**R6^~%HGi=GWexZZ?A;AS9BD)7d2sOyBz)qdpbTy#)vaQ8qJFYEa zb)ea8B@g>`a2Twl?|B_nF4K*O|4{BaMBU08rAc&E51HEdDaRipqO>ti{=1~Q&OP_C z3}t+t3uzZD>zNB@5JOYNFkX~gT%)hs%kzEsT}h~&b&9eh=IjKk;a*()pHkjM32h}( z%t2#D6UjPz;&{ZpP`P6XC)uSHXh*0e>>Nrvb6(DgXf&9~p9e`~A~2xa_ZuZz5gR{5&7;^ox!%K+A{1PH*SOq=b6^|kLL`ANDSTq0T@)-(b7;(cC{cWom%76K zcOyl3qdPx4s4X`FRD=ECBD9;k=OJu;flg(oqrunZdIl78Ra-(xY`tqq-x*zc0xcT{ z%#4nI5^~yhiaTjlpngYHArr4kX?ytL-S<{?^K#%PWT}?rny-5NM}X3P1ftif9oXUP zh&Ks+m{L3uN%hRTWClZC$)Nk!c{>uL2d)0q11j+pr^j{Ed7^${^19CNDO|OCFWZIY z!-%eGqEx=v&mny!bMiw)U@F0s`haSdHwpmzt+^<_jjx}~+a54A(maTXy3O2`%NN`< zWrcq*yp}j>)Wi-asqdREBpZW_;kf{9ZI8x{#!8WVBpDJ)ga|+N*@dSFfh&_fh-sN& z{&>!7v_eqtPi<11#=mIK-ftCp9gwDM z)mx77kuMvwPU-LRKMh!Hp40OPraUm){*9vmsipKu+n#+I(!eb%uRoo2TJqR!z0bqz z&8Ms#j)R~xhqH*22YY}m@AcIO%FIVpZ@bH+`FemhSOtf0YjyRe<}N`)KmU**#ZbIX zOU)4=Npkh5`*F{|{aHcmPVQMEDUR#W?vyY77xu{s4a?E&z@B5Oj)y{m`-#3m4sEUW z#Um$`EJ{Q9_^bJa%s4pNtD`vgKcoY{ly3s0esvSg;tzP7Zb_oepq-G$FvcJBq={xO zMPZX}VAa)_Us9QduD+&5ChD`B3mp}p@DAHsCuIY9%TnR{U;OxO@{w+EdJ!60@Dma| zBf*F(Y@^Q3s~umak&n6TKV8>uL><9hQGsDZ&0JlQi|551K276Ue}!siMOB|em1t+k zYpHsOht=l<2IQVoBs9vI4!h)Zxdwcpi*mT(= z%joVj=y2h)w6Ny;k$2JkxH{q?=OaMu;j2MCWW^2<;)6pJ8_)LAIa!E>*(Xm|#V(mA zaQ#_YcjW}*QqC0a&~YiP zC~Hb4K|!C?q=lU>)9v-o(*AMv!yyDj?WcghP~H!stNfXrIZj#Rxr9UwU$EG$LkRUm zJ|xtW=}3*5`mC zJ`L)cV#I2))|zl>-e!1iNo8%3?rKn2SGYn&e?8u4q%+!*q}7Rz_GIY&L^sPXVtwT2 z4j~I}4=f2o0LclhT4ZQ5P9z279oi7_LTJr|#i1-#dQKD5tQ~Wh0Kd1mKOvYMVltmQQcqc3W_A zQqx`RiSgN9nx#9z0(Z;VjE-Rv(aniiXir`d2=9PHZ+hD?^lKL!RU)8zmzcwyr+Wd^ zGwacU>VXj7Yst8>z6TzY*!-46Ce*p9gpP_s7#d4BBB;ms;}~A5e=fu*Z=IZG#7TT3 zyDoP{$8$ea>KA6}uWVaGP7I69CabWP0(8uyFV|miMP=cr!$1i@6xLRBHwD((+XSH$ zT^Vk>3nVxvaV7f`lsbJGp;+kE37QsLDbJ(t?wIJ)oUnOO3MY2JYD;mtL^P(ZI_$N# zA#GLFjG&&(X4Q-n4oIDo{*I@5Nrj4J58u$S7>BLuOO_3NjwVLgnq|hI6&iVJEYBzi zK`Xrm*3Y;OL6Ym{1$($y1=Yr4R&;|qXuR+l!as&)m*I`V*_dhPIRMUE_g#d)F0=+X zC7vRbve|S}p9bus1mqc;^EA27Bvn7)aULKze^%OjbtJsu7B)DIczJ*EbC^#-%U8iT zKf-}{J-snMY4$h661I`c7OU2MX>)BX@i8)mo~2vPNmwuAZV|4j2nnP8@x^0+E3={Q zDWV_}(baN%%>swL`}zZ%!`sGi1(Y((-ghXHzTPo&GL`pAkarGm$0Q2SmqKeG@l6;S z-{7Tw)lEw5&Nx@cZSW(8`{@)~bFJACG&*jG*RR_VjKwf69EK^!om2|PeGoWmZ08xg zy?Db$cAa>(OC;)nzRx5Y+1f8|E#Vue{_@&EV}fklM9j1LkC_{jU>|So5&kN$se;aS zo|;%7x}SL9A%-RB~tO-N99ASu%jiVKe@-m0fMK(}xHaRqQ zDGN6xB^eZxvks8wK~mnf+g>_>cWylW?*(|38=W5B%-FHihe-?Aw1pc`vNn|CJmWR|^YG z3s)P<|8pJxpzQuz$3JPi|IqQ0!2cH=FZq4RwyTXZ)X5dV^-nbK|EpU5hrN@OmwJhO zsVxw!ishxWoMh#tlp)UltB(F**7SCRS~$o%S~|U8cN!K}Hn5w+8^9%QQ9hCCcE>F0m}eVjlLFPO*pMXezZKN!f* z&GoW89xy);2L18CAVMx6h@X?o7{JX9t6e^RAQy<^B{MJ|4qhM^H{S~}_m8hIDv+0x z?xWPO?&X=J7vG8#7 z0J%9iKrlsMAn%J04@@MO2UeXt+^`AxdHDb!epnO_?~4o`4sO_xFrgqYHxLZs1OxcF zIDi~{{Qu$3!^I05i4$h|_q4E;^Ya6F`C#tcumSk^d0_K_d4b&guvPwh8e`-CxFi3O z&HpXw|0~^S|G+k3$e;ymq<>O{|FEb)9ALQVf7}-^M|+5s+skPr{&F%v0bnkEAefh% z|KDc?6u<@ofqX_~Zle zzYO~S!$5L!@Nja%w&;IBkVn4Wep=GMcfYmXTz7QPb`&H+QRE$qN8#aWaTCM>LTPDf z>4U4?1LA@U_?J$J;50f!?p2h z?_+Uko21y|%E_qjs_4m$`_7VE>Hbowrd63{#Vfd1!sQ5XpoN8HhhCX$X*ekgc$^%A z-_KI(>QG>D657t2tgaTf2An543e? z@BK){?yWY4RIW(8&s74b%3fI;ZcK&pD*yiJ@Jb-dWC2adBUoqL82KOy={Z0TX>R9R zXa^_8$=z(mucy+d=^>RnBKv)QQN)#ar^_t_Mw9!pzhz0;2sGW;?zI$tIsq7405~N4 zf?n1`L6%2+cWEiHY|gPdlnOF`>61_mPV@vE-f-FX?U#)9)O2bQX6(pR2tO-j{u1;& zqBy;OKtg2bp$c|EQN%UHaTH4<4kif({TN@7_F>5kgV-0_hDs*Rq36wgdaM#p_IiD$ z6>LAp+#r#x72szE$vNdUkGOg7|MprGNr#EZ1^>KCYAMKzK?vX%e2ZHxM?7%D8f>!? z$Wen2L6_j-Uh@gtQ}-8L2RA=C^|3z3ni74(k=^h$o}l@QPfrDWV+X-|M4qC)o)yPt z3t_%z)01MGx`32zLv|sLrr@0u$O(!hq>O`t9+l=jNsKYgUrdZC?DqsrZ-Y;8R|2$* z_=snHxdPTiI-OqI!~Rt`@um5%Qc(^Z(Cn!2`xZ+GYW>VJSq}?BtuYqk?0cLq=8e?f z7>9>Z4nmM2MC%`elkKr=`08D&dm;xs-6vLb?K+X_xKYvKDah+BUNwrLnJi+qSc|4M zBjWWpK}7cNSc=?&2d9h}XN|sh%=CC!`fvMo-zZCzcSRg**&IvG(g@L$%eLjlhC=%< zY5cK6YBRe`A~zZanh@3?_Wlvhaia`>U+pQKEM0G$F0;yguVKJhA*2iWf z4LT8G)&1_(iFMJ1^M&M8RN(iCIDa6Ns|oi)I4VD`L*l!A_omGIvonky!;-cmIN?NB z`!1rs*w2uUl*TCOBTkc^GF+4vAA>ezIkBQjex4aUc_f)J?I#}{yyK6pCUL#Hm-fg;MrLxARiDcNGZVr?EWT#@<`SSyFaQd^E(VwBeXqPem&Z>(4rBmuV*||^7M2mM%m4R7i z(u|UNh{{v^Ne#$XiP#g;t!Aqh4ZqGN|?u)p1HMedG!GmL!Vur#@w6owg6< zc|U~lWSVgtHEr-ZwA}=Sy;tr|TP}_zeuwq^FQNDGC+E(tRB>D<%S-qfsM z&TLGuhVV62wI>dP+^Qt~?DG4^!|p5p@80uS#p*?*>c7b#GZb|jwTIz#b?34=|PPg|%+5F7b@i z$eUA4P&6@E(l+=szK8Z~m(lh!zU-6>3Y=sU_yBMTR2 zS)h5I@@i)VI$``UbLF7mVvp?|<`i_gOAiA_ZUOEY@7>hi9FDyA&dDK7dys5T-I!<# zY(IX+wTlyKMLzG_ILtcD!9R3n5G+e#Un7dfBAIw+>`GV}+X70%enVX@<)!CGdHj{~ zdo)JeN=S2bHg5SPs*bXb4mL!c2T>g1O(bjJh0%StxOftGZQidvOvsu{;yfNCIx3>E zALBDQ$(x+73;5)^v~W_==8ve%=@CE zX*JMu)%W=mcI?N?vl>;A$HR*ah6x=zmtx&d_y<2~oY`7pmc;*F0zO24WHw^4CvYUB zAXp*5T<3P_)MHqo@Ynpis+~yk?MK;X+uDp=%QCZH@a{nd1@DSQ%8%7@>J_h1e3OJH zzt~e+n8eB9^L+jxrD$rt9@lr5<6Bb zB4rP(k9AS1$4{Hrv0abm^9(@V*h^~9PVIK!INGb?pYLbhUFRYyvfyF}6?YGjbyWbdOss(xr>|JU2q#`2O~4sXpt9tWc1nyIVU|HKjS%c| z-%tb}{4r?nH^LT`pcImgNr(;q@>cFGDg!+3;Lpvscfo!Nw0TrArqq_yiQ@x1oBUw$ zCy7n~5uSKCZ9*2UKN98bAd_oG7L`dt)xhe&Y(cD_WG9`~py#IVx#u~;9dgFD)+@2b>dRFu9{bHEDR%xf&)=KtWNo`EmjcQ*tEI z`QM^{e5aIqJxHrhORCasBY+swgUbIp=e?sK7G?L2IefZYmN{O!5*mboiMrn$jftwz zoSBJcYg0i(oOzI2L%Jx@7W8R^wr;?_IcJ0ls3EzX;B7AQjSgLo{2K$NIh75qa-T9$ zyyX=9q8y99Qe}|u0A6hpE6uPxslNDvJn|l`u{qryec6ERX-^&WEjgC&d-iq-h91XT z;dV5>@07X7_cpJWFjv3l{K38DYDZ+pXUjrvFBJSa!rGLo%*K|kVWh?Ho#3_}e1^2~ zV|!;SW>B7z$5qPKb&4>i4>9wXJt%a~-9Y-cT%H(e6r+dIH(lYuD+r_TVX3r_AMy&PTnwoHk``ND+bPQNNOCe!3KwF!m<%$w(RMG%ORHz z(lI2zrho3CsYDL8)VoHxBFH!O)E z3tZUom%k0$D3z6~G?|<0asK3+pI^fGwJ9AKTZe*bCFxw%-v*kB@5-g%n;e?qSl*KN ztoJUD8H9U%sFsdtVAXuf(mgBmTLOEA@Ac|=!J9XnEi>!USJH5B^F}dyl^^1cL_6*3 zNgnKZqe4mO!x&0&q7#hTCx6usB&ty^20yjPx%V#!kCAAZME7o0LUdfygnDB(Ts=Ce z{n2kch&{v8kBn3_wBAG$+NL#BOJ)aQ|J;NpFoWu&3t+eA`A*}JId^4a(|yYTKCTG(F&!JN+LtoaBXnj9DQOZ@%0D)V|>HMz50 zj=7RNVBTY9_SHEZqwy5?Tr}Uu@nyV&bXU9RpIWODKWgvDH8XhyESu3xTQqer5zE4e z#-K<@L+ApGl;1Hx-l?DL(>jOcc}LO?7OZ+)?Ru3`-(Xkt+z;}!RetB36(QsTofIN; zlo!?#CE+WfzY`jsQ#%x4_%0yAe2%r^cu4pCAxVug@W(rnU9K~czfpbKP%aa{V17zY zSrzZ*)VJ;WsMyyDa>aa9kh^ZB2T*V!iXu*>gkID{#6-|UR5eO>^hEGPcr{KgTrNc} z76eX$t;a9e8^sOR7RLz14absZL?Z1o2SP3hL_mVDhdWpy7%i9ucL;X~r-(XwLj6uu<(Avm9*zz!PvXlqt@D#o~ynhjS+p> zN=&lmV9GeUI56RSA`u$uP5M^h>Ni)?dW9L|d|N;+trMH1uGbFN=INLjKEws#G2KFj-?H_TVr39~|GQDA)th2o zD7Ai+Dzogk%BN_*zqF~eV08K<=V=5Aal3l_HLit@KdWM>SoV9S%EpkUUOS<2l-A7c zal3I8-k$n)XKAMKb5Bu}@|d!A=6+2sX9IZA`UAta6M%y>qK0OTT4NfaRi(>%rg~qq zFhsB+cd`Oy4F!Vo8c{e1OFZ%`U;u>#5mj6Y2Zs?}+_@6TkD`g`TD3gl@so4^)5kPZ z|M9PLtG`&TMGpub1P%J18uR8>69^wNN%}uZX&gF*M?AWV*H9KkJo4ntVF#Le2ESn- zi8yLC)n-zeAov;C?^5r@NPw+wuaIP_%^D$(s*4e!qD~M^&8!7i*sgo6o9dfX>Ro9Z zY+rh&HZw=tr#YVP@#pNdbNlErrWrEc7sz z!AO~;yUq>Noz$S%J=t>yM`mH0S0by@eLlv@I|Nf57?lo^ZN5;--HZ#f0m0%|ZDde}&u%9_6s>vOTh?oH!gi@Hq9@ z>iKL3XQd+MPqYJHu;ZtPniHI*2+tDV>St5zKgLHcD~>&+GnfN4gx(UD!g^uY`^CiO z7%O%Z^pxKTY zIi&Lr#LQQFkB=W{J1U9PEoFk|9aU;Gj}8!bO=P=~`H?O+z~8s=;rf=RaI=(U33i~E zqtnUzMy*84%-K?|4N{DgW8HmI6@-eYOu50$7^D~kB{~y0*@&9v;#Li|Lm^DmI^4be zHPp%pZsyE$<`Vt9^2b*>?nN1+8Ls-j4$kI_KBs8VGAm+iDbQ&&)k`k0WT(^vm^z*| z#^iorXiecdFcl^+nx)gKr}4Yw%_ronq~%x1ep#|Aj6)X zQ{rUw;=n0o{%K~Ky+*u!8P1TF9#iqRj%0QPK`YhKhXCI}@pCG$GPjrtDT?~{*Q8?a zr*Ga)5c@nj3Ff!(R=j5D4RwZ;Y76oh*%7Mcd?*r`j_3YcX0|X;TE=6AOwj8tsYF>& zc(s;ac`c>}NhkIbQ}LkHsQZ@}7_V&E0a* z{cPvV=^&o#0a|VP+Q4tf*?Y)Vw3_kdED7@5@>KAoQ%Q;Mp1rMzNU3mnH6v}3<TC@=ZOG7iM$8`-TNy-PAqG$ zx0auJ*!%NJj~G>LXC#R(ff+Y z!n72G#~ zX;C^e;)vzWp*;EBO8vE+;RNawcWX;}bnzhN+%#^VM-eMD!`KwPp9Vv$Zfp`NUM{!g z&f;;GLT z@?S~|bTmFk_gx>DPwgGZ73M#zzV}Gqs+%x-G})EI;}iq9hG4#J8Jsxyo9v1duZ);5 zWBRMY=c|WQIIn?0aH4jZ3XM zeja%{ig1Oc?RW!4s0enu7zaq#um6%lBrTchPgzTg`fjwDs|-VN0WHH$6h&e-Asd64 zpDlHTy87g#x_Z3MWIVr{?Mlp`ac0qSm#f`Py0Cv0Y3luHlC*K}u%NDk@zKMql|S<# zc}iPl&0*{L5)=ECkRNnm+Qm#@XKOSkF4g3G-#9i)!dQP|%rlR2kYgo1A&80z?@on! zMLt4<5pF0unI2@O*`meTVJ#*UlIC`u4azRXG#L3}7F(c7aUP9Vm0IAoLuYg2cv-;w zJli8*gJVkdnfBxp?K9ul*^)PP9u1?7PMfxZW^|zpnr}36G$n~DsWi84f%;HG^4#`Z zhQo3r#UgK}y<+KvwtYBl{K#WMw2ZQMLEtEUx@px&B8r?)qFlSG0tE{W6>9b~-OeS4 zIcDEFvk<{IHcRUU71S!7C|x2zTecx+bVFoF_{5n!)YKB1m5i@td3dx8O)s^a;^);vWh^_hb>7Gc?v56zsw)=V zOx_l$+D*qG;&U8G*5_57h*j(n3U$DRF07XTVP~}GEdn@$0#Wg z!K(y-GrrhAFE%{LEfpi_emVQC8}02fmU=RFcqo_Y+n*v_<*46jd9i|to&8p#@BTqn z$?v4zY1v`VJ#ld7w@I7l(g=0?&*uhyvJ(&b@J|%Q-v1AA_ZVc!vMvsPx_jC-roEce zwryj&r)}G|ZQHhO+n%;<>+N&S-uvDg_aFD`8&OfKR%TXZR%J#-tg7FWD>747%^>7M zfQxFzs_*^oW$Mfj%fO!@)kM(>k%AJHe#2Pa82%kS0oaBoeZ^P)f;`?$+}(x<6PJUz zk@9zNIb2l5?_h5As$pkaDVS2STp$d98qv|Q$Gq=fwJD?im|zi+M?x$`_l?m_pxHl-GFJDE@Y z0Hwr}FTt-L%*Nz%v=Mg@7PGr;b%AYhP-k5Rf`eMF08NVISkJ$Og`TjZoP;zg7V;pWd% z7CjHd3XV+ns&BTN_8%BQCLRZzkmx??b+YXgP#dSjKXr#KB8%&?o{w6Q&ma7Oi0r#j zy)#+^5g$hZhgufy*)c>}q`ZTYI*XK#kC0k5LqMd$-l zi;~1?>x}HBQ5_(^+$e@PxD8#bRg|e!VC3H=Vr~5bQvt}2bKJ!uiC#I?ZxhYQ(L&ez3$pI+6#PqwT$`Pd%>Y3Ob*?N5#qFAxX z-%5;#t_HPo9~E!SZ|UyJAIgi$;zs=KCq^(o1EtrDH>;G);zZ!QpykYnn)LpVh_z=_)w0R@cL@;H;{~%8DFa z+yHLsM3g5cBy_rs1{$H@K@Lbuz3*>4%jkRg0QH32%NvPx9sK-sW z%WV;(c`?w%`-|mfaO!ddW*+OkGG|Bb=L6`Ve(7OvVAe})Ijr4KlMoeD0Kd4ZC02pu ztWa@wYM?4(LWzlsNr6daePX?$$&cARMn@zn-QL21@Cs~pAnXL4JPNAaJjfLUJEif- zbuV&u%7gx6Br6JC#_vWLISHjAuGyv@f<jh?_0>a2k^0_!#oOb z;M!zwyS9NamO~CAl?BGOFTIg%Pg_$vV;smd%hKC4s;vIZf&dwB(}NzGQ+ z+tE2UTqS;J|+drALuQ(MB%ERyzXyl8?jIi+>T27EgxP9*|2m7dY5nAiL(1iY9G ztq0AZUKnzrCh(;Adqyt9sqQ}63KA_JzDg=LSMLS335bvp4^Pso^1uRmyJdD7c*k3A zn@C+Zd~3UcQ@OL^{KrMW^U~0T7KhCWpy;4XX3EPR-lN`YF&WzEsH$H*lWgN>v*#(o{ zeeGf}!UU`Qss~aZtcl79LmkoE6S1_|$4%MlEf0B_Bpw?(gO7kX-Iu{vcZOFrfdqfJ z{kh$d`Z>%RB|;*a)MSc0rj}GBsXF2L>NYl-;N=vaAatR?QQT!P4s9B2|G8{PjccKA zmgHMTJc3kc!Dvi8pfaEmCygBmYB1gcJ zD5X=c6S~uL^Hm=OXmd8t3qNgRH=)a_lNI>UvY%sRU3%`A6-iMg_cbXp@u=~*xN^xW z)CxPKD*f7Ag1s&!7&V_THCFcC3+vQND*I#VP3)a#9(55(_;7fFzF^?DJeolHf)3pR zJtR7#j;ji;J#2BCGqQ4z%w4yE_c~}Y8(^p!Rt-eShwMRucZ)=}-Rm>9P3Gn8%43F0 zE^$)5*7~q>e$s}!?XJ=+^a4W}+hyp`{a_0IC zkxYnxM`R9wOw!tPzPV{j)TXJN@bD+%y$f|N%XtQ=Ib|A}EDq;m?@R&798v);3a+Rm zQho?5=$jra5oshu(0D(kBJ+^L&`h&&aAm)(Nt#ZOPSrN!YXRN0B#c-H@!3#43P4N| z(gFS$iyxHD@T*hAsi-Bw3=;OEg=#TqK6raPwc~-xj6Q9O7Q&TOa#R~uF?1@JOI`EE zFQ1O0OsaHUo=vI7j&ib;1b9@jA>ND;DM@@jjbkzr7}yQs5ng%-Wn-_kD6Xng6`Z~N z4zGV4qONMS7L9Qj3c1NyECrga!t=Z)+c}L@p=+to>ZnRRyl5r49I!K_*^%sR)|ih8 zP$o(0sm7{fl$2%6B|xZsmx>@!c1>#}7o0(w)@B;JP=d1|(FK|0bIb=Xq$8&1Pz&0z|{$%>16;cLc7vx2VXg-l zXL8w+Vn%0%f`UkuyvM8?PUx)1^|GFDOL2=5o--ih8{j$P+s{d5&wAMvBnz<`2yMh? z4TU|Y8HgNq^|HRO29o4kKM_kZqjKZR)7`G|>QXB7DMpdVKnV+yU2b)2tMQNbld)Sa zFNC>Qqxa@~|4gpW$tu!DT+hf(j3=uyle?X+(DZLPpDuXN4(6c7?aTry(P-85X}I|V ze4I$LQhcg@D~%DK&|*NFOx;-0;2)s-ROyRXA8xEvw;wp^zgaty=2X*7s7SWHNp%ZD z`7s7q)6u)w2eR@VnYYFY=d+EcZBDG%a-;1AP!||yX!kscX#s3iP-28^aRLm3D#St-A<6hC7 zZsX^S1U92&{h=xywRfi&{u^wh942*>ws;dEzRTKW?c`zcAW@pB!6rF1C5uP6!q~!G zYoo?pu1K}Hy~D!AO~B~%#vpHAOd_#3y-vWw$*FlwsWnY_!rY1G%+%b*sT7r=bbUfY z0&A6~lZ%#hZGFS%S8+;SZ7mo(g&{97WssvX6zrRqx>iI*Ky|9W&kr=cR|)^HU50Gr zYkK=euro6xPl_`nO$1HBWl~O3PRM}awLrUi0SZRF$fNJ5@;Qis1e5;mZ;^bZz#>0l zjM(h)DReIey>2;Pt52)YWRFpIejv z_zk_eJvB0}I@WSVKwao#KD-53UtStrGGaI0;($_aOS!GwdB2c&Bi$PKsntOtO0&~; z*6;Z0eK!te$&EX$KzdX5hUZ9Il5{(xH5-NXM^AJKhc5!uv72=@TGii*=?2$uzkrXT zZUF?%c&83lv%^PX8dPc53G&ct`?Sd}Gcfy5>mm-|n`#TrMV>!`${LsUDlts5{s&s& zqdl~0d_$`LIq{ZNx2*i2ew1nx>&)y4)Jt?95a#Pbq(4&7)6qib`@`9v#dhV#(?%jb zlezZLc&Y{`wJO*HUhuTm4Ivu1kFv?0s)3y&%sBdv>E5N&RkO`41jaa*Zgf^Xz7dP0 zAeDOk*3`KJViw(=9=hJcVY~4*v7@#N<<^h{*vk)JyGA%?ecsD+zITT!V-vVYCK%;* zz3KK}eq~L!#+Q5C$q-7Kxiea+3D5i#S;t8gYiN5?{V6K;4!TpyqC-|JpOL+6G>-#v z>bDXxqzIs6Hi3?oUp}u0o;?oBnK59M^a*pnb8p;M%l79r8N%*>WQ{r{;?OPwUdxZ> zD5mQ=b0pjP+m5=X4|IyKm4W{UU;h{L{>za5U$F9vvi=WNsUbIJ+DD7feoFC>jO)88 z20b|43i~t-?(hN>SuYLRK&k({)m4@6pkB<2#L&i|dM@m6`-jYg;Dr`oHN#JnK+76Y zw+(^bE8u%{YW+ly-{RTPEwADFv7Qp0(Q?Kry*rx62u~&hvEYvjkpZy)WROp>b2WO>)M7P| z4_?a?%M753!HfZoaq+BsT=MxjtYt~=;|`@Shv0%&sn`0O*Xo7lb;kUCua2%VLB}&E zsMAe;n^~93wCd&+U2rt%{(oQXe+&2jC3gSgWdFE*TDbs%U=_^GB7Yu(|+;huSr<{#j*dL_fO(ip)VZ$H9_oO>GWR{#4>%U zXXY<0{jY7dFYx_O=9jPk65kBeZ2vOP|Ei0bnVFi6?Q8dcHdy|{kn#Wi{3nB%j`pij zY^?Y!f7SGNiN9<94+#IOUe>?c{CBCp_xk5}e<%E1;Gc~Ds_3t#{FB4R$nb^Lzx4Uv zDPP3=>m0ry_+L$8X2xg!!sCCRJj-9~{hx$?HvZn>pM-xl{$ki)*XCbf{_k&ppNefzL+wMZf=jWHv^8Hs*i#;=enCj)e`M zj`ho^|9#!)Sn0ny`#)FU3*ys$NqpunwoXgS!1h0p`u{UB;lBp^e+K_wNc}%CC;ttp z|1<9Y7f`2VqNZnI_!8^?I@G_&I@On^r)HyL{0g=Ck5Fg*8i-%8|Gx`$7CI(s)~{HB z{|l;rMFvnaG10Q(Q!&tgG5jy7PESk!SBL=}EyGuk04*Ei|Dfvs?H2ksRcB=W()0f} zRj2=oL!kd3_tdiv7x!Nai!7ZF*3t(xVsVWoag0(>RIP=voy)bi!^!4RS7 zS>>to4D7w3S-(XN)%2j6-V?OGVMT#O^Y;~Q9I9|% z_mzCJsOD*TuD%u^ja=Q(gYQyKxKMV_3YB7||H4?E-{ApYJH+(V#fr znl;se+H-Bop(d<&gFS(ru1B;*>=lXo7)`##N4wAlR|W-rSEm2|ES&6bu%y?{HM^cm zop>3=>PN;4!%H+c>xv&p_!7#RxK4bc`vX~s11bZ?Z{9K)A2xVi$7`&>NtGlFdeYW~tGxTJh|UY6kSnbI`>q)sC&npLQ~*<{P&7Un|KQ!= z@}%#w``k*}bf`?EyN@0D0P`j9_yBikjH9r9AH9tz2mV2flQy>t=?(S)AV6jLlQSmr z2qmvckHe0(3*odt#~m#Y*#?_l9=SKnSdfLEyP*d+UxoG4XT}Av;WMK<32WDTJv4Y% z6&zW!cHhmt$@^sNxyLC#;h@)sS!cy;5>~pgK zxt0~;jJDREJ=|w~6!gv_klHszLC=U@gYep_Fh$z)BYdjg;oQ$-0V<;B#RE~pTH`0; zWAL^1r;6{P?)r$gHFqLckO3I9cB=r;q{eW5G2lc`?IRhR`w=|guzH+u?^GopTL)dQ zIEc`QPJ3f*SsIvMm|0fP_fHOdnNp_y= z1IHdwPu3p6pez%rqmK^SLs|N@&pp3PHqr<(1$DQq$d`tH5hXKQy5G3R!Oo)c2tpog zD{SPY+*Nfd42&I1vzvKrzip7uV*?!u^)t+;=FL;KaK$6Pg+M4_^-Jx zzEJ8}k33SJn?PLD7QAUK1UE9r%0#}mF6u}ACmgfb+a*QGov3ml!?H%Ps3$i0stb7I z^cNPvkem<}`S^R+9?jg2^|!^)$s1ZPzz?-TS7bbB&*2LE#9Ux@ijElKNqJIzokupw z#Q3Ws?APpUt$YWTi3P5zG*v9WbZ+(N_l1AveRv@BwO~{ddh)TkW(vLzBR{CD;7tny z4utoaY%BP<;d{eG(dWkwrEJ+RNS^06DU(=!8CdbXYE~7oMv9ys(wya0+w=-Uagd%# zdOW8cJKUEYxyI}11i|-ZCu!RNRUGhbA$J8CX6`t_pinAAiH`GXP^TcZ{KBpz8-xuaL00C7>_o!UY`N` zxvPOZ$G1ls>4r@sl3C>&z#8?P8U7w|4$%)(CcfdDb*Km5S%^92jmnu&m^Kw}*-)sJRR!U{$Sb9@3X-FbW+-lk9#59r1?P*~enk z@hI;Jd9%l~ga<|tN42cfzrogPo{gitk6E+d^?g9OS~Is+i^h*buInJrA~lK`9OW4g zr9|h4HTT?1B|w?@#u$XPKzwhT)!-t-d46yb`K51BfOj~!9p=3HiZqcAb;_U61)H+B zUI=Mb?92(Ma6!Sm;B0ODp zP?#N1WiReOV?o{eji3gXdeb*C&5O!1s^PY=f%jg$j(Yf7J^0Mu9L%T#y`s!f_T#x5 zMZlo0)^bYZFWJfTDu$%=3fH5poVHSToQ0kOuXmJx465)9@?d7s%gj(OA{#vY@-|8w zCimwN@9Q`wT=rHLkl__s1i2tRo$ls@%@}d%pQAJPT|;dJo%!+jl(-R*<75U&ifC7T&JtsZd63@wVn2<^jg$R!^Udou4>cFO$6Sj|E#+y#VeM_(BvW-m z*KqC}G{4}FqeXv4x_Esqv3fmkY9r`ZD~=}f#?++k*wXxAKxBj+N-dsOA)7|v@of!9 zFi3a-QzptjqV}~6q63C~rke2tRPLHl^eCPE8mZFCdU0rR@vwI1`vB5KxIo3@wafR2 zX{;)Oy}_wNXBl?CEN*A%51?Z6`jx_CzXQL{O8Ah=yI`0E5w}y-Jop^=rZ#1?JlTIm8UAvCBh6EiBad5FuP6+ zbTYM2;G|rH>|;MV7!wI^Gu05c`WBU$CHX*|L_)Y=%*(pq&aFgUH zuHMpR{>DU)?)!Fhyz!tb(kHEFhBueah^|SV-A(bf{HA;_aV7nZbfUI>W?MX$bWY96 zXuO!iQHnkEJ*B;go7Wq<=f`+9@-8%uIAn z8aHNFVykqPjkFV3=QNjBm&+UK+e(Ro65CSpg;Y}+DM^PSX6XsZ4lW1d69~Nf6c0_F zrxLA8mRAeUqRoAoTbAp?7wEUgCyMvF_uF^qcbTW97nYZ%7pK?9C-|q$TfA$pgHLu@ z1PCzp@1r1nzyOe4YEkrb^i}o3B(?czEE%b?RNqubrgJm)g}YMn3YTU>zKPmY(erhO z00*7`GeAR3a5D(p*Hi=*1Zg8Wwy=cDIuw6a>@j!+?%B@)O)n5%FW}9VP zYOgxyh;p@#2Irxx5e;V7uY%{{t1As&C0{HW3(XGkMebAvaW9G<71$; zZjSZ=3;6tGrL+#-i*A2>nc*>O{arDxE^MltqwekQW^O7&^PBYxvD{<$uKj>X{diGY z7F^~eTZUulZAtDg(4rfe2KEHI(&v)FB6%6&jM@xBQZLr$^$}>&sjN5odG_arkq**N zMYiR$f53fC1lDG91mnW zqgIB~>Qh&Sf!@M&2G!(6&w*NDhtBjKy8=-9cJ;b1!xwEKaYCPN8$Cf*cH31WZUi*x zApe2w(Y6rjU(|H0D?&aBnzz$H-L3#TY)iZQO;O<}hLgfw~)vJsF zva8K0e%VnCM#2j!=`Ti$S=jACHaE8gEGcr_iHGYSBsZz$*FXz`>m!~F`RRYDgRUa| zAx@^_j|^#=omh@fG4k^UWu7m2SWsvlfj}nCXt++FX*IV&P@rB=aq?zg#xQ9=kk>{#n}MpPAJs*5jN8+JxQqDz9N z8&G!z8UkRxVP6qABg&*CvWo7C>Cne>iTnB&fF1t{1VTunn06t!Xrdj$5u$*%jCRPUQE=CfGBB%s9r44 z;T9(w*w8kNI~eH}QF@@vmYr>gItkB?<~{OriaLSPNYx(84M&SGy%=l>u>+B0>J~Rx zysS9pzCtd&q8!tLER~74gMtj~yr|T?B-On5ZgB)-o+Fk_t*G5ewsa~lvI&yIjD9Nr z`n&+TJdG4Fl_>Ll40fvUJ0tI$31BtZa9WZ*i~DB^F?2`2VXcZeT-p$>LD(|V!*{j8 z-?c{8hwd>Qz0B28<_v4Sj@24xun#1!92~=Nd6gf2IC@z%Q!D6aoUe%*`bo=CcOoyy z?Jv>;B9CpH2v|$dpTgdVKj^iFw6;&5!8&8CdNm9f;IFCN zDK5s>p5Z#9&^3Hez0^VUG7@lyuVR-dVgkgO>Ayo-uS&Cctv$h!Qa9^ zA$rDt`gslW?BLz%y|I3NcPg2Bg7l2d+R3_|dxPN_>)O(~sd{4b%=DV*+D3nBddqs_ z_~7aq_S#B%GWY;|hJ21!U1<#~?O}I~dhK^zZ$2|=f~aXzR1mgQgE;o8IkT^Vt!;mM z1KsdPDZy8TdMY ztdFG-7c?w%15&jW_aw=kl2FTcCe56jaJ}LVl{I*HRpL(Fl{K|{`<&MvO0xz1Z2v)| z)sKEl+zj9;JR6a@E}AKFLE}n8btm>pt~xM#QvFksWkjVy@xXv=B#4>)!AFCg|(R{tpV`y@z)al&QW9o<3 zTHEEsyA{sU>)K_z+tsw&CEmx@u-wy75T1wo*%N7&r{m6E$&^f1%Ra}(4UW}wWRv@l zTIVj+=5^ww^O#fnKF5bqr(U(ktF}(t8q{jzh*icCJqu}@&~3S)RGPjaqh&nqRnfD2 z2d=or)NX;$7q}OJaZ2TKdke728 zbYz2aqWf5Epl#{_;@<3`tL8b`%v;&~A)V4fIp4KN>3k7x(P43Le_m+F&y$awQdz7L z39`f8uHyh}1x1tpA_vAvu;jNLiWWZ}WDp()LvwCqTv2MMUWIW=GCB!*4TQ&0N75CL z5W-88JZt(jeBI4EhTKa71g3os-!M9tWQ(25Zy$2)~IamR7bD;rGQohAAmkUjOj zQ4tc-qY+aVt-M!qqO@}x2o>jEAi&YmxSU-&^}dGWo9X|a0zs=CL-oe|7B)_IfGRi*f6O3L{6+s!HBJ-FDW`# z(?0q~I6v%#U?hrLcY(%uZOj9`tBcOSm{RB=%?k(-#da3oG+7G|&cHB`p58yWn0+U8 z2V;%=I#DW3vZ9cCQ?L#kvKBrxrb$3zO`qVkL$jtEy425_MVLRKLn7A({!d0ds;^2) z0f8#D?Zd+1@@rW7=u3Lw$v$`d-uYSje%mD(jA zab|Am4Te?QeH#JVgw=X9I&{Dedtx;a7W52J;a;vRoFy40A@K{s<1UX1;>q^HPbTq# zSbok7frFd1ye0XHrB}S)rIG4r4JbDv4a>9w7W7u(O&tWDeOGsiFyP}L^=e&wii)d0 zZe-_kaU3HM|sE`aM?`0)Z7C% zEgiTzzaRq4ALj}FJAN2GO1^U8VR5z?dLTNG8gv_#5lH%>#UVT21yM9I{k3Jc+JvKN zV|(iqq;=+MMsc^h$XRKPyn$80u!i9X5|&ehf@7)N`ID(i6a};4jU{eejmJC)DCBGL zBOqEtU`yq`GX#EFk|BBD@m4CT=nhw!UGIx;fTP4#z$?azX|;aS(kw*B_utlYct-=q za2?GP#v_{RA8;?k=E7$}v@gV-UD;aQFs^7!-Tq7b{igkz{fPa<{*qHWk1(Dxb92fk zcEkKAgn{_K0bg%|2^8+^+Eb6ob7V#p{h<-#GkeQYGW$(3`pZHxyQd*tfrB1q7Te#> zG{o^4DA255^?*d=DYYY}Dhu%}-#cpBQau|c;*K@vI@#Va_#r|wJi;TqU~mQLa-Mpz+F6C-+{cq zu|}5!9k)SwWOY_dKOSwOx+0iB%?jL|SCCs8SM|Afvm53c=Bp&r%&;^SsI134ptYi} z><*ftrd0!`!3; zK~u(62EvG;+=-O8gQWb%8rV-v>qhXkzUr2Q36a>!EWtoZ>b|P^*E*ZvYt38biP0;; zmAy`C^RQR6-sA#Sq-Sbe_T*hKzm^ZQl%8+@|TIqr$!A;ZvPyQLZ z1e@gAMS=XF|I zh|#{~y8a$glp0rYp_g2%Q}n90ly~8$vS2E?f?P}@>gX1q+ooQzo(4xQWTZP-T5%!r zedXA5d0~KSvOqsP*zk*>>+!|4$!JHvI+|C-omx6s(>J!vX&kkVX&_K7$j`4FTSf{D zhl4R#S4U`w7j>c(^nAE1=-XF^n6{J(E=VkhCsdf`-jx0+)mMxiEo>~EFwIk~S}a{K zRhd{&fHhTf*fXd7_2*mTh#_A;2gvtkB<5c>sE?rm8URx$Ctk8hhrIO{NBnjE*Fe6e zHm;m9503@Guph;Hr@Z`&meUnlL9(agmJ6k#S+bgjP0^)Kld)9(l+*+B2pL*Z){kW* zh%t<$V$G#(-UwsqLSnwmsddHmDCyU~pfds%9TVRsGd}j@aoY?(u5C6Dq~WM87n_Lc zxmy_HseohlW-W6_{b|~ur>2s^*w1_O7-MH+HOg_GONygJjn3H<5*^D(K;mt;2zYea z;*;?bY)??Gf^E0tiUh|hLKAQGJSdY$AQI<{yQ|`F%^1bmqy<~3;^soxEyN^#jk0pm zs*p3RY`1id6YwNnzb4RMCs>P;+mUt3$3r&!3T;+bCK@O}Aqd%$4{a9BmDlga6ROn} z!nGi~DV{|Jc*WZYjbThKCefu2(5HMJ8wm!DOV(Cc99Q@3V9Q=mVEK^2Qzf&nVwSeA z?XJ+5Gdl7@#BXw$*Jlfe_=|f0*aHHe*wz!&lOrLBy21t`qQsgbf3c$Xq&@B=_uZZ`JJ|a? z@pBP!wmfBg=JvMC3GZCz4>&rslq;caP?sd1FXWagqTqL8NL_|OUJDi+#DWT~2aYM6 z>4*9_eUS;7iR6d=d3?GiN*nM8JWfJ|tU!$?v@lz1ylf-lB6b;2~t}m$$?uD)&?Y`WRpBCg?77=93 zc{He*4W1fwP|X>y^LIx9P8;FQ08H8-5C4V(Zy@;`E|k9S%7U#Th27e$eMWq^_Cans zgeBqef@mVRQSIModZ`~Q^Wp5HA^NV|Sa1C9O!iP12e8JpiPH(9%p7?U+)Damr@D!h z2S!;#!b8>j5S?|fB7FqlStTE4B+Wg(y9=c65iBCAxo{(eVt30^llje5LpjDJ=0o3$ z`vJHJg9vb6lOroC{VS@GyB4m%zUE>-w%rRDHW2kDnMOUV!{Nq%Rc^L4Y?h_PYMIBl zp)&%0gWd4S>lfL`M<(isrvfbiC&crERps@m^ex@adoC{VcUhJ;j$tyY=nTX zP&h%XkU%Eq1xVt$qIdTwR%76DWM*hq;@(g{a!ko4U#KzQ!?Z!(#LM%Uj-@V$@Fjws|T#dFbE7Q%kvciNR8qY_k;}2s$#i@3uK0gvFm(V3W|T%P?C((jJo8?v z^qS|JtW7BoKKCAA3?khM*$0U~B%n>d&n7Fe$4QaFBUv*^Wz?{gSc}cH#*BK9nQb~Y zbxmkA;1Xq-MIQ|9Qb5I98642^M89gj#01$yCMA(h{g>|(*Si(XPGbTTRqcB0ND`Nh-*FcyD0N> zgwXWqd5C=cP^Yg7ZKNj@x?I#@pVo*qHx!xxoC`#!Pm(MJmc-*IW^D#`zMeB`1YphQ zMyY?3A&#+^=*KvTfq8;^h$q;uEr_LFlSV`lFaOR_*Q+skJbEmqEuCa&)R+|OE`?qm zqCD>ypshornO3g;+7|R^iUTOoN!VzxZ`1*{#2EIx7FVlW7xLA zbqF)Dc?8xks2vJL@+V^o>q9+-UuXAODrs!jWYVu>-a4T&ax(7N6|QBnX;6gGgd=n8 zS@@ybkS>Hj@31Ftz|XcF8{utxxsDS>jhGP+-N0N$G$Wfs(1b-Ud^D@UH5J5Y6%sH{ zw=zG0I)|>qJK(f-elqgID8cQ00!!ybLs|Ah#Ww&gZO`M$>U9=^h)c4)fHwiSzfU`X z(Q?9Sj|-=NfD{^jGyF4AzVl_yF2ufZB!+wfb*qFl14knOi*f>OvHKq`94~F}l5tAu zl0^|Rp>_x9iMv5`M>&8jeVeVx7-Ss^M-sF^k?Q>>l?7t#h=PBL&1)w)Ht1-3@ojhk zxSn5$hz4T1m4lZ%vf*-HOboJ;6ojecSSQ$^(w!RMorI3!yWQ1cEfCz|L^ zG3iv5Rhpi2FUS_r07B7SRu4-;o9IBR8i+y%#x~_L*Rqgp*T39;?XUX?>g_OTxkF6e zZp>z07wk4yESWekjjegzMCf)QY*~iDk#AJg zTh;bfb$@^O=iBOpf4a&8UlgC4;YOBc8cm#jpxY%u7ve5F>4|A%;%_Ri_lRnavz8Js z55VfMB$FxwP?>NzZ(AxN9WZjap;Jn@C4kAY{=?IIRv6Wz7J;y{4Sc1z9c}Mw zVDXSd6z@d(aPL2=3x((~K9IB^i(9khGZh*>dCl^U`{~cRKA0<1l?P!5tzrGT6IL^v zMFx@UBxSQ|4Wnx=&Cc46o)nHx)&su0QT{njx?SgDOgIulyHG5UB$D~anCE=1+DY8m zA+v&%(cPrfwuTRkSriOjuqs^CjmB`OtRsni|3YW_Si>j*o)VL;PHNMTnD=g+{RqOyj%dIv2`pY%9H_${BYGiv@mnb znbyY1(xCcDKEk85*{IsS-2qn)Q7p45A7&bo`>**2K%Bs=0U-S_NK0Ua~X}M(eOLwuZzCu&ya3L=}-G zyj3h1aCDY>;S3ZSCju1b^2a@jd=Y6?<|C7`LzXmD!B(3D`) z_{{>>vH-Y&ZIXD!$E4RbqxDb+KF)_)|4;Q=;L}UDQZ3_K5UIF z!<6X>+8W;V6;#>;DAQaX$^K&`W@qP9g0xe z8|Gke7_FgVK&F+?uPwW1O|GaU=Jx`0w}h)b6^*32NMRjNbuEXAqRsLF%5#f0b_gu zr6(%oLW(IY27~LZ@z7XAeoVndCm^7>CiF}XsgYxgwrak~FL)YSTWr+NXYSp|W9SoT zis6^9rs70eB9#w9rV--E1_p>h!i{~&&M5h)#rU+tx`G40*|i()TZ1}js)Sge`eja4 zbo$To&y}k>xxV{9-4T*S3!>?#u)+=H^2c=LiT6Jljc^k-0wLq+;Fs2_{Yivk=gH~g z%8vW9!RBAA{C1`a9FcJla-mhfkW7xQN9Z%6XJB2y@VhC$Z>#P*_W~xDS}Cb3B1@Wr zh=P4!sl1S)Omd_bl19e&?;mzMd*WErY^6@*sj?l+1j?e{nxC*ryY00V$9JZJfH&*;1;v@N?^mtfuc(=8zG|>5D zKZWf+_$0VDWgq8Pr-q{_IEq?r^VgTyUV>(AtSFl_pcTh}n!9bN#3*zQ>nZ{ExW011SXZlrs(bHqGfGOER*^Xqy>H;+| z;6L7i--SrU{^ArgmQ*oZ>j{OFeG4z+LP7S{0b&FKC98Q>Npw%AmFwqA_Qeik;o#PU zC~n3oiE^&HRfXJ#f0`yTJZ#9Dxw6Gle%#oL$vdFLRDM3_y+Yn=OsV*KJ@8C36(|#X z?F7mWXF6xK(tAZ>;&fbz=+qG(*I8scpLl~-9!2b_A61ErwNx&5tQK|p=kNoKe^ErZ zGp2CoA`xbx=SIigoZi&EQ3vCs+hZhwmcxa-JeIi{9ky-Rv5zA#&zk!AP(N%Y)m%*4 z5YrYQ5jeJcp^KsY!x%4`RuwDhE!)Az0aeXJeP2~XGy9ZsDBMQdPq1zxeR!a;r9qAx zkweNg&%F*3LMBvVrY;|)k`7$biMLw-Ld>267?(+oG#};%rQ-vI0T(qP@|f%V5Bad( z+VSz)TB!Lvv3>$OEmJJpq142<37k@I+N62QI;h=|rpqp>x5ekMZJEqfN%jjKq15{I zw7Sm_rA{o)kJFYZFENua2_~ypHy%!z7b~_`%{ith9`w$;<LTXEk?q&bYYbK8bSScGd-Invf;c_UI649YZirLnI+l?4!nQVLmy* zw~e#ka+p0D{jZeI9(~y&;+JWEZheN}rXZ~do})UJbBl+xE{|Kt-#Y8pGXK0^T*ZLU z1oc(HOl-`T3e7apc)XhlherDz%A)skXtOsn7$+5=G^iE;4-N=6q@v&yE9y?D8=6&? z;+kwd%h)r;R$RzmfKip5y0U0@+RH#Ha#JTWRKcSBjufzw6gZa-19 zy|C64vlCYa>B3pP;`TTRY4;#=+n6bGq!*=bDFV$B+gcY#a!Kgq;hHYF!Cm??a)3s*d%K|KCsz8D5KL7cjb z5dmaD2wfpK6=y70-36xKuTrH&C)cNm%E2&EF&j`yrnIsEJb)?$IYu)k&ls+sI~Ppa zj2#AFMRQiP^l(`fSFr9tGsCyUK4;;;RejyTTA#xD6HMj;;YN`7%u?-i%@#E^?ni-C znutl>pxnu=h>5LuJUcQf);N8#9oMY9Nu7VKx7#Ta=dHiD5g$V6HjFyh$VpFs!ScG@ zn{$aS*7WPTY6yp!ipfB}Z)@#~1CnI}!aVla#=CA)Z%ru9PmrOXeA2QqWH-{l4OgC} z$76hXp|#1t!%LVMRb#oX=oX`9TAf**!4vQ8kkFwlW|GU9DUy(r-%RD#jAQ>l#_l;f zvZh@a{$yfJY}@Ww6JugKnb@{%Ol;fg*tYG7ZTrjpoO9NDo^{Up-uJ&=dso%2+Sl&g zYgJWWzckr}sfDg`6KpS4FI5za@%LczL1br&Z1WWB3=_H2l7dNgAoD}s24?Hx34ZVo z(Q}yAI6752)oKHVazh$ctf&bx^LWy-!Fq8Ai70`F&X`gJ=sc2K^sekY7R*RdmJ(Lr zx`tWd;)Ww0rg%hU9Hi_doxeth=qdDx$x)~^JDx9o%nt`s)>Z;+9y1Rql!<;VX~_12 z*wI4CFo^}n4Yr6speoDsq9?pte3iE!q(@Qj z+j{LFFXHeeBttWx+*MB)8-niR7J+Vpkv?ns^WpQ$oQJ!ao1O+J)%J@ksmV6ulZyV_ zY5L}8Jn`z`_%KQdRD%j8QrZ*v&5 z4`z^Js?=1g(o4$cCAe6zr9@ip)L4u;^NCYSO_c@`ArkAt^YPsE+tm{Ugxe?Eweij} zrfU2$GjA7q&P*A(braKaa6?_l^%%LaQA<0InA8u)RI#i=Igq;QSbhYI%;+&goD)B- zp?gHC_Igs;@X|yIA(0DahV37_U~at2SYfWP#83TPR9aNp`QU-0dpw*Savc@-$LhHe zMyXA;Y9tj?Kt43Kp5t_sctabK*L)(IyiG(S75U+9>JBia6;&Q=@yxDP~9rrdO5$E{X6yt^X5r|K`JHqnX z$B@PKwYIRU&3t6Dt=lIiPA{=4>L?pHB{Yw+?bQ4MVcXlSH-?mC$F@4LKH*&aEQ7Oj ztSATGvmJR6yXea`6EA+Hn>UA8j|9EMU4Uk!I&gK$ZZhCH$Rj5P!zD9++Mw6mQ=Jn< zaA`NSTgV@X5PTR9`N*TW*fDT+l|O5p@6LI#Q%$bgq6Z2ifu-K;mTOE+`wTw>qh-YPwag?EdgQYqTfp5IbncZ zzf2Uwga_aE8Y-4j9tnhsj%=X6go{q3Py()hLjZ*rD-jFOhx4;xmJHA-kLxb1tvJ9Q zWpx(qS+kddKusZcr+f_;PM)J5N8qBvQ=rv^1{>Qmufn8W3)M3MTX3s;AU*2ykb3y} zA>%rAfJntfxp%nPTD=st<*R?I{`4!((fsZ&w-}FN5+}oF5BFni4+S4dpGy~(_jP~t zRjP=(k!<{1rSPlpQ#>R351&7qY#^#8bgM`_*g2?$aTfyd$|j_XDpdj(u_xEo)|Jto zEfZ-e5qCF@^LJVip%lf3X$JGC`6-{rje9=<1Y_w2N_&tL2HiI8eNG|8)z&o@?G|yy zt=HajxaXd8y71L0WzratTPfqB9h(MBE41kG0|iTJ@g?j`Iy9Pe#sNC^=78(y{y@|a z(Du@BVeU-wc==3ZMn1Y^w^{Nc<)T`kl8IEEm(|ky_Q$pYX@&z#m*(k7-2~QQ zmhPo3pBU?k-x+oVR)M$P8=%3EQV8kA1=0^wM#`;X@XAFZ2b^a0Z!tJY=}BvfZ)Tza zGB-P=6cUwjocYYx#fAFVT!#|U$vIIimy3UbYZBCUA@gxK@C$ZZq_kAwCKXFr-+(;u$>zf zfP&lkLgE(!n8rbPo5}TcZGHB}LFnNRBi6t?aELBr%$tiol*LCn{B<@@*-NH;{O9A5 zL!WG#b2ck?G9X(Q>vKiHjb97vR6ELt8`*Qe=VhKB9=QuO_Pzt$#JGode;|8EjGIr9 zUC&Oc%fsjh)>6rNbg5#rROh8;hL8hrv@OvC(BTgeRz&FW^VttUnl-b10;NOwChd#} z5P=lDjYoR2qhIBII9eito@Y_Tt@_ZQ-ome=_DPvn$WcArY|C5?Lm?3?onG}$TcAgG z;nrK-C?Fz-!i>op?>VnUkR|tPQ`okhBsc=4%nBPX2?z2u(GAEu&5#C+QFFFG~-Luvh!L zi-Mr!O9K3==cjv*^tdS*hlu$jyqbiyzAKvcO!1W;ChcMuH~u3SmUY z!g3ORp!EjV0W({#ekE!$QvW3erOx+@8l5|VR=STDA$!}#7xP7i+iOlow@aQ1 zoOUmcn`+nZDBqM6gEGls12VCNx@VWMZ4ZQKeswk^*3&rFIyN}lC3XM&73P#P*Dl!6 zD~u%usii_x4dm+3F+t(p}t znAd*&RBej`nwI`-{srOHIS(ZiFL;!RY3_*sBPWe+8MboPABgXCRJ}w-KF{fa9(8o* z6tG|QxD>R4{s?~Rr(+g#+VO}3t+C@Kdg}-3O2r{Z(E8zTVbI%98%hQ=&s>Ntp8>x$ z&>tj#MdwDsWHL(UqXgHavEY!P)zfkW*{u-Yv7e5@Xu^edV|)9{Z8D3jX%V)YP4IBj zlIH!c^0c|scUf4D+ju3Q6xQ2C|4JW~S@jFf=kwP8{7l(Ve+=DS)Unj9Q-`H>$9-tV zuV>Jd6h*u}nyry7+Cut9&BDn-+Cm7&;FpC?#v=A%t-C|zj($aS#zycwVtyK?a>Nk! zrM_>3f|OQKtQi;f-zeoMN322L1~i*|>wFZt&8mx`vH1X}fXMfX{9!0Q6nA^S>pAsZ`3^J^#LJVvChW_sw>@I2ympQw}x z+fZsG1SnHFJi3Ke#*af$c&?(Dxsy=KB0Fp=tqe$jDFPr88xEhh{`ejaEt5%d7Y?EW ze_q{LUySPJD8Rj}np{Zd)->7t(JZwwP3aO7dG7cmM`aTqfRP@Pl8OZj-heM&v-0Q> zBxCZ>PHb2{?#8#RzcRoKN4cZy9N;+-Z;eQX_&q7xh8g;pN{*m8OStgx=e#`my?4M4 z;ZZ*o1yMHCVRveR4VnvQ{LOA&psE=CnbfFiG5P-P$=TWo@?oEsKobrB4?O5#M$BD2 zaJxmL65zyU?b+i%a4LKUIg-WQwD#_DPjvA7R;ZM=zgT%DbV#+s1iS&f&bt2e5k0nm|* zR{Vx8MeVty0jl5Ea{)DkJi%OvUe!+16YAu-#uAfYiD=1kWL=|VU3C*yO8hLXAM@+nuw?hB|#|U(V{UFqJQjx_la?fSLICLX9+5hY;FYqe>&jqAY%DMiRt# z%9C%@%O!4-Jo+s+6z9k`zj&A$@8Zt4N3y*{(SwF)D~C2xCYgF)ZNHBEf|8MGb^RMLTm7 zc^OKjMVBI(h13Vt`cfiq4wQpiqErX?6Q9C+J`{RiM7dFPY#a%<#q$x)v)L6~!YQmn z)>2nUN!YUw*^ftUY?yy)r+s-T1Ke3xL4(!h zL?c=NJyR8%XymN3Uv+t6rE)3Kn~B zxe-Qe*Wpye`={3SK25+*g`1p8YjuRi32;<>T{`os@yK z^G4SYhHk?|B-xZ+yKcb>y{D?$^hd{3PmeJ>3enh%yl0jR*&m#jye~&@_XEaJBu+77 zc8`A00rf*MSnpbdZYrOXI0ttbD|5yG(xT^vN4AbiV6S>@j;8Gq-b9gZRjPxg#f=+D za*KRS#oE~AnbR6TtEZ*ef2T7HD{kA>mL&GB2vAz6L@E(<@h81?qqGWF6>>@VQK4y_ zyo#`su-3!#v7gx~HMd_0-A8pt?VdMyrpROWQu=GUOv8No!fw6%J^lmfob^3A5KD1eO)Uvt*NcD1VwgtBM^|`$f z=7s`~7_t|}SA|6^ciuQz*zrXSHrc@aiwU%zD_NT}EHQ^yem1Aw2*o1n1m~sqx`L%R zq4>V!-l1%wvBw8p+oz6v8v;>y+Gs634EYn3ZQGg%QQ7sXXgfSSuiY1}(cMuz*_q;H z>OR;54CH%rA%ACDyuLw6#XA)E#NgyRZUA=!_uY%@yVkpE0+wP(3VDigN=g{i@AFly z76y$UD8H#H{OWD7nt0MeJuaPoe4Q@QL$4q z<)DEwuJxxIN!H;-`f!?Uim~R}-^^e%n0~uw7{_l*8MkY94kSf67Zo(I)=39kWHs?| zb$&8Xs%OUjGSQzxh=F7`rDA`AZwZ!xC@rikv>-e$J~uc$#TE(qMFQxv+^_Pwh$%F% z%cjVP;kIepd!pZNClenSvjWpIb5zSl)j4M=B|0WV~oWTk9h919V$!AVv{A*glx)f+T=CugyC?hFjDGLpU z-Mm*|QWli7wv(!H*Ak9Z*zBj`D1LhS?h~3MXKwQ=cfMpnJ~%f8Y0Zg&6o_x9H&{z$ zz@Meu$z!3JX|kJ8%24HD`?A_HMPK3O&H7upV@+|lVRHWEBbR-P@A?B|Pn~^!5H*38 z`{vBiwi!puc{Mvx!YJ*d70EIKAiyG6%odQEV%B=ka-4QvysAT0>1>R}Pn|4_BL z*aPjQB~-bK64F{UHg~RitXzz>6E^bK_ABAL5PXD7YEs%#O6kgYyp{$URL|TWBRfj1 z=$yJo^H~;qAqahO#|X{mkDpaJJ%+!rHu~|h4P+F-=a}uQ4b-JDSUl!4PtB@^$J;ex zOEV%)CaL^`*EB^0&N4$&0_2!Vb(Zs*K&+^%?bV2N_Pr2NHyYYT5ZXH7b%`4lheBiL zb*8N|SuB5VK(A_Uxp(IybO=IlWNpc;o5J1|$UiwgxzwB7Ud$M-tVieTBDWdhTFWNR ziv9Jq>OoBpPC>3iPNM)rAt&c4^jWL%%q+ zkz>+ux~dULlaziK&-+ItpG?AQ-KHT2 z^?dj%J$@p6QW%Ma-?9%wo=X+aL|rOfohVtk-7hAK3JJYfX$dgxxX4r%338Z4(psTI z;P?bevGcr8Zm{!p=hWgt`7W}F^f@0#sgrZIj97GyGAzyDzB9h5m*`r{+eJ_ViF`0u zphvw=9$ev12(2GUr;c%zVUsqeRWx-YX?uZtuFHs`{o9JARw4K?9|}_CD-n_EJlG;&=Z?$gM)h z2ojw})j7H~oR%G>aBvont6_#SRkE{ye8Pg$Z9R{XeP*+WuY@&L?Yu+T`S;wJVtD;$ zP+X&Ui9aryr-a}BoXD(MaOraygr_vGDVd%GsW1`Mk9g_i{yD0I32G;{_EZW@2T>h} zus=+22?nDJ-7v1Q4GrlU-K4T6nk_Nrn%{DlG_@=3j6(n^;&TxjgFG<+F#k2XY9eyvBY(PdAi+VA^RxiI)Y zlI~||7b<1~0j~fPKr$=!bTcaoYhv0$Z9#onsOah9U!B681T(`2C=m6y78R9eGC6NP zw*JTKdacbN1_zk+z8L+;+pLopEE#IK&GNp~*q8V!wVP#+UmOw7YfUzXTX>ub28VuW zD81kRVy@WiJX5>zrldWHanh+uJA7=%a&^n@AcYCH?JH z+Ho_YsF)9tlWzgq(-z6;=D&b*%c$qj4~Z4-66AcGIXC6E`-^FULJUGw42i##3e!TK z-)AK!)^0-HnmuJcRPM{)+E%R>%*h5h5{(i?^9MskD<8@Y(_f_- zZ+^aY!I)?M0BXGfQxeKVk?-6Rc=_z8(o8Z5N(j72e(H8$R7v=6UIjIc4X4!Qs1~dP zDoVO6Fu51E(GP|(T&((O!o)LEMd3=uPrf^WC|1oRqXuY5$M1PYaeXb(qf<2Od8+eZ zTu~ZEus=}e-K%W6p$t(A{5X+SOW3w zzg0FMY;J<54Hx%3aDi&XIJ{*Xzz(^^I7+pi(5RE8kF4C#TIjQ0QzQWwX)Etb1 z=L_V;ZYZcd{9_^-7lg+M#)$Ym&=NhKcH5pqr|yZ**T}A!ZCfn^lGe|Yb}~eQP@Uq8 zk73JSb@sKq)dav7ch=dI)OL7zQr30CfBM;0W-_>v{|zehxE)+{NNHq$Y%6B_56^{U z%pP@bcn9IB!qK=)DUBXNy4PL7D;k^MzDJ5!diFB&&Clbg>dz}5ytLZu@Z2sgPwlM* zc9#^It|oiMOQ)%GZY~`W{BWz2r_If4$l;#zHC3-q?p*@R&Ik5xItF)^y~dOZ>zf2m zt&jdY%}0zPh7A>#h86YhZYHeDSlJiZXNq^@#77Pu9q>rwMNqGpZ}>N!6C&=a;(Z$1^Qn9I&M?klZnm=Fp=vR6i)2~)Hw%lDcwlRm|&z|ReC?S1)5ES*4`EF#3BiC&uDtjbewD=!yq zVhd9?ehuaN`LT;r=24F>Fr^m8RbLF`ZcZ+yHw?rq%!#UNu{D~YM%1lS`9hAX-ReW+ z#@?vKCDte3D8EMILf*Z-{N3)C)=~Vfr8D(q~xhB>LQ?S(=od!3|3j2Ihz7gQ+Fh$Ia)H>Kd1!%dY$D{!|l$i_Z7I zxoge@q}EzaQ^Wn$Lt8KJbj@IETb{cUy6rE6r_=WrVI^){S0aAmN9WTqq;5r?E$v!t zQ;Lt}1xl5%#!?pardiJWK%#rT#LFuuRPFt?GE=-Rfz_imHDJ&dwa9+6jl~S~J>Hbq zM-0ps73Vppw!LJgQU}YplfAY;dW(>bWv8SLgOB;^I}m#1!2YMBG>S$!2C~K;R6;gt zEp2fB#18!ot#M6T5ReiLN}IWmE+_$Am%4aiNjl(}s*>BnaPvDtuAg+`Si(@*h6ncr zLHq7r(sqnXMORbLegE(S0e`9lK}x8GPV;G+#YNWo?Nfh?jrInU_ayMKQgzeyGHPiH z(winwC@%Mws73yYJKa4Z$WIZu*{IwKp zLiv);m?6*9lR;~RgO~W&jJmxm(G(R}*&kttTn%--Mph}Bf)OD*Gd-OgmKv%WhAZdp6SxG3 zIKLRsgW2v55M8C;>Fwjx7Mq>#jc9k$iSv>NcV<~0jgzhgb?^!sTvrEF#?7U3Ez*&EUer*^6SS!y zBWi`eR1BLe>ylO$h)I-`>?ZV#B`b~#($RJq<* z!avL4FMiEZa<>t zWVSJj2l}yqquMk@nRS3Zrg?C=-u$s`;Umae>3OpyUb!zY-0s2~A>7JY69+LzA$SY2 zjOI)Gvo2yeu)R+15Vc1+%~GZXF7bDp0@1cav@R*_>NvbzTk|Uii$CW9k~P1JKlg}w z%i%Ek-cUWl-RWSjPOAU-A>jVI$uW!V3EXhUdNt_r#QuQ=GNo*SvM*7pk#_pE3yso&}Gr8IX)q~d`2A&9*<9eh} z#ITcx*-Bj9Kyy@Ci%gv4TkGn;4ILcJcGzJ7x=om_m>^6(&)yI(8ram8ImG;{2tj70 zC-Lo%-Vs3uqCH>EkZ;eiIWQDNn#`~?pe^$kQf#f?rn<9iCKq_WE0zk@pVe@mJ1WOt zqz7a6(V`J?Us4MKdP^!h)O917@RgL6>x|3@6ysM=*(TPWH}7LyMjQLsRCm7KUM*)= zTev#dmzKZcUxyI9JQm|DS}!^Rz=(2t?>cDaOuAu_bbTt|XP22(5-)Fu4>qro)vuI190r=#;_*=E>&C$BqhtUDnBO|9<@w!>aJ5e?Tp9Iq$riLwVWmi$&FTIb|qd+_C zKUIQ`Tp-zpWNm98)!~AMQSSW|cXqd}Y!M|@ZDDJ^uuIia;t!fVuC%`f@V3FfTba3) zuTVoR9)h`}qh$Z?oWucT=RhcVURog3l;lrX+xbsz!H%+%5 zQV8O*JH^9_jT8kWMuvu~JN%H3qdxl)vvKiPTgD`F*dvEiyOO;i6zTw# z<jbzLvbZ;kqlHD}m1(NrO1&~dG&kJ*w_$az^oTvs|z5F>8%NPLKVyrOcnY*w#Bq|a&6rOmm2LPj=^ zn{R&goIuZA)&8sIvc@-lx07$hCxzj)waM0dTyFEF3hQszBk_9g47PUYmB2mnkoCXZ+`>rsJl;0R9)% zX>Y6;dSAp8Lrzwolg2oZZ(+lq8fzl=Ux47V8EAR@yqZTZgh~`^VbE z6F$yXiI#=VP-VULq3gFxq{rpx*xlJF%WCe;1b#pV#ar(O&9kDse1%GUr>70t^7qa; zs=0fQ;yL<5^h=|e6H*y@Z}<*!#1`Tv&(AcxHJTgMt4e0g9t>`@3flTD%EYkA8s%mt zQhyRo^t!6L3w?$(Iw(%1xu}8)6e#V3)l#t#6Q`~3RU#isd>#jR124g?_(Fb1ul1G< zo5$-D=tsIUQ&&B;A|AU`Q@=xeH2%;PtrdY6GV4x*L3Z$Q@KV-`t%D0_vGGy{hS%df zm!J~uJ}rhp`nR#1+wK zPD+dKeuh?Su+Io}EZv1!NcntwM%&p4$rxCc5UXwCP&-#YU8hJBd_ZJWs+`>?8&|SV8i@v;?1p17lskwco1Q zvgTE667!UBez|w3J>}+2@v(S&`C#`j!#(IDx& zWy?HGdD0PMRy4?r5;c1Ar~wA__?=zDO{8M(fO4^ zbhNO$q#Y%z+&|Q;5}9#kiGYy@b1k=$8R-5vw~&bW%)dM^pf?;|;D`m1O?$8V^~In! zX5w!!?>ATwMEvm9b;Jj^*=zDxIBkqcmoB~{U-|K~#b)R?0htZ%eb_`foPD4W$v_v^ltR8;^k z;`>l)pZBG?ml|c@c<5A?4Ct^$P#hAC(z+zBl(`W7Diin z7M=1As-%*zHpdk|M^2(=F+*4BaEY!6xfOJE1Fi5belzd)$YdsP?e5kahzR&SqHD_( zzXtGW+z}AX;E>{Tz;r00ug?ZovK)O#j89mn>c~ky6ji4m0q&zd0i2ECpC8^RHXS~W zZ!dVg2tU1h!Yqbm#<)w+&!H;ScKR2>=N2T7!fw}aE74#w+Ld*b6g6oQRZ6U{T z3Qt%>!pnP+k0{GZ08VOIrIjZoo<)t$D6JSbrKTIxdd@C~W-mS&9fg*bDl<#PQ%49h zj%gdJh=uPkvwUsfd9K4ohA)yFUT*>c>Kb5J2!2rP&BPVR5sVhc8KVsjl8_jy2^Jp6 zNG$w5ipLq7FLExvCEI8PF^-!-s*O&My7UnUbX`U}ry^gOgK>0RKlj6f>Ive!K>7Ov zTq^au3}&F_i!txn=WN)Y`iOGc4w^sIp=tTS<^xVXh)}`0RnLW3G1nHtkLksNHh8s_ z@g7VPdZTy3sbxbHet(U{8zP;%Vk$@36@yX__yM_P%qWSNreQvh^)FEMp z69g#$FcSb)oms`ND&Q{`LD-_$t`pbl3+>sJ(p^<<#u)~1b$M(rs_Tq#8_vyl^NIC^ff3V+M0(QJJikII1HBR2B|R1 zc=h;f6QvB{+lO=SlT)Z zmiH>z#Aq!aKi7riTPLZ?_5MrjNHyA!f67E9T6XP-9ekPr;cG(0_5}kRxYx7v0R&nL zy`QnoG1U7$^ZN=UYb)0Gx0PDJ+3RFExe3-BMqFPkU~u)J5^zZ+qHz8OcwbtAV1#vnI#m zF6dnsCn`}S-QV3{oV1^o-Q!Fw{rNJ*}s1Es)8l{40=6Wyta?Y2-VKc_8Td(+n1BW%OyNRK>eSUYd0mg7x^`IME zyKK6h=iK+5K%H|S7~UtbE!ldONiR7eG`}Tzu|a5>wN!`p;L=p!iHec>dC$sd}#-RrPw0X;)Iu z7ktCst;gOSZj;iJ)&2sG8Ux~SI|^EHZViD^+v%7wcXN_((6e9h@rbE4_IC)2C%y-g za{MqVd+PMpC{1!5&@rD05D1u$>81RhP3?{yZ4DUyi-s6T_IDVwK?(S+O5xj_1LkOc zMKBB*lj+vCX~D1m7=8B7r{JW9xBiuhd>3udp@26U`>S|qgAUWHi;`ycbE3kL4;b2W zXl}~#&0x4jr5r>TI~o<+&w&6(==+5kWfqPE%!_ia%JPv!H39gK(rN$3Voa2htb4)B zKy_IpP*qIHyCl2dSOrF?{1}!<`R4fF4Up=(38z^3A%J|w-c{q}cd$M|{^OK5OE~#} zB%--S^xVxe{!gGE&^BY!kw%$@c++)s6oAJCfvzGhT$_-{; zCKCs;A5A6;#^TsFnJ7Clm~5_#PRWYk^L$cAE)!c`=84I_`xngbuy5dtJOp;Z8Jc9diBHl!@(lBdmBRjXpulGpzbtx;(q&UOX zt*X1%W$~BGcAQ;c&J#@DV~sq^WxDiLoF`n)!JX#dn~&tj4Md?z|W%+R2>g``RYC0yBxt%qG|H1rpor$VJoaP^_&a2Rw}%E8BZ4e>sh(pqCSi;?Gl;vinN4})n;&)K%0d_@ZhU|>hK3Km!DG`+ z%f!)$<`ahoMK^bDG?F(rnVFdZtd9yEvZ+4y(Y>xj36`yN!}lwt3DWt6_i@HujsgCZ zd@%ymPUj)`smfd3rZOTS{vw{YXRfqBYB>~F7ABz>Tf=!Z64SvW5LcD z7%`MQ&O>bAddW(^H&HFxhhsBdL$`qDgs9b@^yvV70z;+|#JeRZpZv3GzlrlhP0v8L zPJUKdIa8wg$(anDqk6sht42qu^3;&U!|6oPj6?vjt9kZ|RGPj^p44}3+pIM95|x?p z@ImO$4mimG$~CvoL&Fz@GWX)C1$F{ za0eGZ8vM{a)I5yXwlPoY?0nC*D+iLa$^5V$2U@gYjsO=*RKdcy9!G9(2NUZ`wPNzXluBxx^0MH91r?$KBOb9F&cm7~3(go?9}t(0t7=r5PPZ z(XSBz#VKmnpU*V78JWv𝔪hJX)CF1}+vmD6X+dL2~2fzNx;(|76w!%VSP8)%T6d zn*Di^>d1?xh^;nw2SvZT?RNz~1D3PM9P6HGHr>vUh94d@WQqZF62y^42nx|Nl`BLD zP6;cb(%0~!OpK(!Y7DNZay}}kT!aWV*j5_SFcoWPZ22uyuhBt_EkRZqX|7g@wsE#g z)xZs27ZakWNj2ajbL*D`A}0uc7j=xPq&-(y1!FQV+29ehy0W#^4N){OyLv>ZWTCdQ z(qVaB2Y%a@>{O-1jZv=3pjl&Tv_$Om5w{S5Ua1DGTcckcxDY@TNt#olMP)*SR5D3H zwciC9`}L!sS17=sf}rdzLAJx-M9n^>F;WgbXa=6*CFQ|S##fN310pljpH1lg>Zjyf z$pc%-Y0iosrV_=@?KQ7I7`HUHGDi+%9m}mT zTdY`A;b>|CmY>(J`CAFJaiomEIRV(BqodjS>-}R$cc0mEOJNnaFzW zzh38)_Y3m8Ad(OCvV?=?e!}k7V%GgH?hzB=|C2Js!pi<{B8qn0hIIfT>d>VZ41H6C z=+80>eW@}l6o!Y`7GB>V0~6-hoQ|t=NZd7Y5`mjrpC5-2&1DD;7beI)%kpfD@l{Ks zy#Sp`F#F(N)g*&qC;RORPouf4@@E<8MrprGoY+O(WW?W&=gk^n-B!!5;ilx@tkITr zbcg3$7C5Y{2H$$WxswB^Ve)bu*gF~zD)D^#d#n^`iXU(5?D!BOst3Mp`(YWAh=IqO zku9s}5{{N?f{(~rqam5z%bI-On9O`LvtpFZ;4Jje0OJDRQ6?0X=O#4rL-qp-Nyh#V z_bA9OCgq0MfnzNoHFea4K*szXfdnST1uiK+CP$+vwjSct@7z1r*h{=*UZ-glqm!@= zqw!mrAqDTCj}cQh9<{0Woe1-z-M`RVcO#?}FtpJ(qJm+NwKp`fH?uY& zr2MaHUk4SPY;7%#to}iO7{4AL7zR~B&Ht2@{^=tNBNII{8z-waAq(3VQuGBFu`qt| zK3_1?7Z}9B#P%Q3)IaPLfRl}hkd2**o`r*rU7L`V=^u^i3!wT!h5l9iKY^+LQ}fGr z;`l1V!c5Q31mGZK%P(M!o|BpBA4`aplkw{~Ga)M*>({ma)yjXFSO8{rdS<{^F99r!^j}gD z6Cn#H8$IA3F6*CK|JX)f4X}R|XZz~)KW%*t_n(jdW-|RNbpL}P``=Bbe+~UB%>RT> z=C{??)9^pT{6BCe77qF^|A^zi<4pe{Mg5n}#Kin{Tfc6`f5({^|Nm4bCPqejRt{Fq zFXoAjlm4qF%NIVz%t_C|!OTI($-zhuVB_HU?<&*3`u(3{{4bS>iH(u@|DrN+vU2?U zE9l(AOif8_@ru*sGU$K}01!_IMi&r~M-(ssqG8Kv*?~mS5fUTHBMMKam^Sw4Iis+S zVb+^RN@y5}j!Qk1*xjV1NFhr{t{KkIcB+#{FwMSamUaGRPJ8s~Jbb<7ow~bs%+z!$ zJDFc8SN{2}rW*$F_fIb&qxWKmLF@0KFF4;WwB`7^ZiYHpRKCHj|Ip**>uN95;VB2s zsliRyFDxo2thuZ_eDC-CLAasD>mxrlkMYnUw7c?+yrim%7Qttu0O?#7jQ7}SaxME9 zuK519t@3$XOZi~*#J3|I_`jGeO`YZ+1_(ULu!x-mKR;UauZdNirh2MDIaFuUSf~$@ zmuQZ&=Rs4h;H+7eDrwEH2@STyO!Y3jx9=`*(x*@ACnd2^#-E@JYa&>LazfJIJVhQ#wlD?A|W$hV95A`72Cr9%J}2@ zQ$SghH)rOA=^^EtI6(U({&h<;K(_IvE=1THasE@tt`~I&E|uV+;KAK9S0hSa_=v7X z0v7~Xh+Fe(OVNk!{`32fmXuX^M)_6Ib98h5d?tzp-D%7j@h-m#(dpJp#0=7Abn;oZ zB`C;%V)b7J#yF6z*qpkErU<`lC;U)N-@9OTB!-~^#}pdA z@9OjTWNNgDf>G^wFr1`)$a!t)N0>b2Iu2&6DX$9z|^83D~1Inh_PRV2n9dt#X{CtyX8tEWi*1W=v&&N4MYIT?B= z_veKVPBMzH@`kcrsT<5|zu8950taQ|2^@F14Bi-m(XCA!>92`6q-y>+HfXe96(Nt4 zV)pD!8jcwq?wfRGzf!HxHDt%~-|K($Tc%Cc)LF$1_^lHR z>u|5pb2^Q{+J~vu8La<&g11NRXYLqf2VbIc>T%MyF@HriL>z~zLJ9Y8sr!K(!^slm zDr6zQu1V4bhZ00|g&4tF2Z0^KItSI%t5tOxCj5&@Y!yt^{zJyMU_7vPwrzeBYzaol zf2oG=n3eL6V~XtCCKYNA=qyBdKt;?zzb28o1|BwsZb6c1!98RATqTJobiZVa9y-Fz zvN@l3z!On}N6yc65eN34<-{`{C`Xdm>jitgu3U4-i&Mud9QP6kJ=gKro7n~HU*fh zY|dv>=!ji6q@Cq)J>}I8o+Cal0nn{`3$(%QO^WDr}fj{g`N?f56CReXDaR z_Hk}VcwKJU3mL{YWeDeIQ1JALS*4r5$bH~7m;J-fUf-NARmkHOduR1oBi=A(Z4Ifx_;o6)~b=TMG>m!*!QTcprw zHHisx<>h5E)o$!m9c_|qCjL3PNS&wQEOg24EPa0|hR00FVxn}gReAW6tNNqO{0VAm zSe~u#lga)uAFLD zO|ISlN8DXD#TB;gf=)t$OK^903$DSP;O^46ySqCfI0SchcZbGl+}#>)oSmok%+!A8 z-7_`c<_~nOuC?x}Yjq#zbt=v89{!<}t z^j6>U-pXh?yY%1g3xDr?-$9D!|DTKT9p_atb2o1$8>XLH?7aP65d2qWyKjej-Y<$$ z4Lt9gKwBQppi2S!i?^4xk0x-oJERBZ<8gZ+9+}|vFwJyjBv{xNl%?RftVk+VyVk}Y<@($O47Ai)`%Fjy1N>`_296#6{`7P2cGDSL` zLLmDNQs|!C{i^?{|E32Etrd!lZvkfSw_AN7x$tGb@C|=V3rUjCpqNU!Jgv(giEnf- zoH%j1?wZ#VZGZ4P-+Jg?U0zf09&x)62!GhLIY-)>=rLizc|{~S4)_`nK={{0a-=z~ zO74wNmRFlMgAdDn-}hk@kUoM~!0k3s$1&p1=QeRo9Ov&}dC*F{?XkxCa_u*wH{wmu zb$I-~cUmVZxkT`^GfEdgjt_@LsO3?3C$*aRId}8pWFyxHauTE{yUcq-zy7H0@hz9= zOB@PSP1i9wKOH}<%}2|oJT}B7b{#!PHm35s-Qg&f2clh=j>GrLz~n3HRw(tp+#OmE z0-!Lyl>>TZ7WcX>@lrCrpS>3Qp|^nrpKhBe4a#1Wvh$mN zxcg3zA#o>^L7(m;%y~Fuo2N8rYuoM-Rxki153f7Wen-6(+RBhWHHFQ8YW$9^-3UM;dTq70!D<4I47J4RAkF`-Ev2y5C^+65%laC4tQ+ zfvf)uOV`fwnMT=&to@rji;<#SzkV7++BXC;_$wZtK-y*e7ej+Sx)3-{Qr($v9p6dC zjOiW*-qWbKLug2~9pG6-7+x`9wuR+J5FgPLU^li&2*O3LXbHkhu2>0%cDTO?aHB~Y zA=~up-Wd}entYBMX4Q-^aU_Tvf#XJ{HbPYxrsc*g=^N0Dq2wk6Y%g-7`*I`5)*((f za;k3=R1UUjhA!4(SzqC0!Jrs30!O}ShTpe-$FD`jHbQ3_Hn-`ctc@+d-V`BzL^u!N z3Pr6Y)DO^eAn!!3nju9Dc-iUMLGt*r8LqLz@uLsyfYqDKQ{*st9Q4X>ZwD+*%t0Nf z!E-y9Azjf5W}knC$jYCjEX-%)5bxEV?HFM-NULCKp?wQct>LSgJOHa5rh2s@z(3cy zyYsmI&IKtQx+`gHhW_x=z@>K9lR6ET5?*tl;q*#p zFrhZ!5vEpMRaVoyu~a>9Uf;dMbIEhSF8gZCBX5P?WoujI=WXC`?_S*lnVU8_9!+^y zb*~0(f#jnxv0QBf{CK3@J68RmjW>Ny*ATqNV~HrGKX7rz_uB~avR#d51w8KAo$VX7jFozlZGq_T(4{n-M4#IFut=p(m&O>6h z>#Blv9j5`kBx%6ME6Jk-WiQV)nZwEZk<_6n(wD3=x=7h8@8aag;tpB55_ZNmSAVcc zqu!UT#OeG7J+TexNrh@E%W5=Ig)CsBIAJyavr{~t1Qc&sbPro>9Nr;a-mfKTi4GhL z#12D3FVtiw_~nGr;zNuu|5=aa&5jftq)cZ8JI?obb@4-K&sk^kSCZ8nJpo{K+9q?VlTEO zs(ad)+FkqxwFmevM@2)HCdQIsNcL0yviA`!$T-OI>mD4{_Zad~@+-8En!y8Bj}ov^ z?%$DZrQQR1cTR!zJKcGDc`zrTt+F0cYv*U#=wAk4ErXu->U;d9o_2l5DxzAiA*)%H zcz3<{dr(!Y-GJ-+S)3j9?)BxZAA{!KyGsOKfjYJ`VYE)TE);GFG z9C12!T7II79kR222fBlfqx$zB*xqVSQ zWHFl|8a?9@uh(FyK2>R%kuRO`H&A`K8af*OZFR6hwwY{`b%R(5HdIC8;_KixV+*8f zAAsS;d^aM`bX{{I*fo=@0PC`ahg1-g`gx-SA`Y)qQ0rhkrXKt3Y#z31(dGBJRLNi; zOONq7UVLOn`00MSWZ+XoReYI!=XL|lf^MhxUVe9_TalOECS2WDWZKwE)$ja2uNZ?r zsny(kE-wn7xrnsWx)CZpEtI~|G3?gudiH9^C=)s597DlP%Pj*88B$sc4;Ii)@Qph%|E$ z!#YxJrjm6D!#Y|GO)fvk9QQ_*RWaX(ZgIlO+u_4aPXiu~r)|UPVD$Y0OBVKD7DX;( zk^YG-sFLxw{I?(ySvoUgtpHSViDmMz!tA*xOXOVRG6f+_*8TEEoNV3;(Xg4T?_2%K zx%S%u%EW)3(Zsx1pb}SAZoeG11r~_>3}&e|q3wXWM+R8lSU4Jqe2wf#2w_IKk>d1MN`RjI>;lWDLnZ^ruC@ki8R;Mls&V z>~^rC6_D9SVyGe&V6@knB!3$+mF*UnDqidPRK92_-)ydRTQ#&qXWgLIz>8_$O?WX& ztIklTs-w+7&v?dAKcp?^{$_47)YOXVzD>j*Ks_^gLtGge>r#@+5%D>&O7hf2({|S) z+_G_M|GTss1?j(Tl=zT6S;z{zkKlsPO*Yx`#cdrh5b78~#`lN_jpXsy#eo7wJ^?E3 ziIxIo-#0fxw7DYuBg~%Wy|Z`&j~2_Zfk5B8e>1`IStUPM*wJu{YwK%w7v!zvO8+ho z+rY4Aw)%5Ib+OFgS9Jd1&{qZ|wW%^nYc1`%rOGBN{3e0b&le%)v!dk+F1Gg({?5ny1u&#H_Yzow>^}L& zJtrW+{J>$(hG6|LT|nJv6(bq7>9TZ~*QCMt)})Be0{ZjMme2#&iDD!g&3P-EM58`%i`j4@@ofK8{a{Pzu$*O7tHkQYrGo!=CHeOpr|{H z+qcU>Z-;S|n^K6EB0RzK!CCKE-7)W+l8_tSJ1+HnL*x@n@}12SQmrt?f`?NVUDOlr zp=3TNL?nc(O?JB;E3B=28zU){fz)7`oZ!3a_k8h{Q&{#DiOea~Ad^Oh5dQe}=tx2G zjU1!tZTMrHhd?mKExR2@;WzJT#8iwTYKo|b%d;E-^$lk1S{7zr)f%E z%>qQN9WCEM*P-inPgl90% z)}A3dX;ZGI<2x|rkp-rjCG6d44w*JYCl;U{EYDK?bx!xFG(-(_%$glyYr)+ri(l3X z`<;Y`R;$6rnxl}lMi>=k;h^A!3W52f>nXoIVsXwPdi)`bZYR)D_7`MgP3+qozb#s{ zN9qaiFx-tLE^=Ftf%(EQKB;3>l1Tb1g|SQ0ty-j+T3Ro5&T7)^rV33vG`1i4=jXn0 zPZBEgMC^rIJ-;+Y}z z>eE-OICbFtYT>ju_fOXhAXxKsD(Ln{H#_-H@cQ^v#(@O|3|`mGhpR};CoSU$2|Ec7YpCZ1G_Vpe{gkg z;yeTim~r2~+N)Rntqv+OpS`sH5M{kJTl0wEKC^6QlD`OU=xV zvn2zR!Ij~jmIBlz`6*a*TqtF~uPVb15g#WjVkRA0*~K6ki!uJgv&Ha9oPQGj06%&@ z$&9wSvgmhyjD1s0lYJw&7F$MvLIm(V+|r;-2UDK&C_wHU2t5S*Tl!l5a@e%~!L($(PfcSW%tM z^HJA6AGpVjG4Mg>cLN$@wngOUa@Md@loLKT{P@bS4@F54FSUC_t>|4*uf9?m@vDMC zC8TVY1}lV=ie4)!p@c*ZNs`on?$6i#)uja-4)G~|7veNzrA zAN8-i6q+=vlrz<&Ifl5E2}GI-hDf>7%D)Q#&Jy2IP8>tCi-^}PUTB$a`SpFP-a-yZ*rJG8jeDfwnS8wxaMR_prmOzz`7L^yAd9)2yCAPQwbd~EQHB+V|sqVlTT{u#QLW%)!a$#|-GF7QmK z%--DK=R(*BqCGBC`#n9`?`zt!m9lSTjAhDJHcOBD^tA`@oyOc0u_*Z5-IQO*t7TB@ zL(zT~zLs~u6uCGLkBA}B$wW@aF-I^**2JamMZXy4j%hoxiQJghSLHY6i7Q!hM)Q=j zhq;_DjU@;kU)9Z-x`!yuhW3xbl(^lxv}fKRgOPEu8>&Vtf2~vPId!nXAOZ6&M1V(= zPUW*o$ZU=KrfhuhWWid0Q`aLC*@v-ZuF`Z%l>hASBfD7l>H`>?phAIDYh#r+?Yne}o*#p?5L z^EFgRa*5{As;1E_nUkE0bGQb;2|b2-ie;Yp@(oy)StU2e<$9ihns|h-<6K|?vUQ!= zq*(>tmS2vtf=>;3sW{gaWcJ;=!%01-Ox{Wx)c*M-o z0XFfL4A)S~qIk``sb+mC?Hz9uFR2W0`U(c*@Ce!#4cXWV)?2&8>Rqi4J2o!~? zV}0s%ZP!1e>&uXa@lHm`b1WE1?FWZq(~=KZ(vWu)^PKTQ(l6eQEpbbL9*P~dCmgCH{ zfK!pP(x5|1e^x9B3vH#KlCE&ep>;7t&NjmCP>$d>9)7NYDq+1?Tf_4b@}eqv$f0Jb zswQDUH&7h8lrj&)RpAFbOA{meUnKT6|NK7J!=3ERH^Lsa$QxueQGc*DXu659R&F!Z ztiFc4MwB^LoHk6mx2mS9R+3R^{x44ecdwzQHsBfXl?6duU0!KQo!sYQZ3N*nvGdXf zM#q>5cf%xqz9v7K-(ujw$BVO6Zz-nmOET&xe-@>kc(xxQAOZ>rztDy(eKNsV93fPA-Y<$b!?Uk&z>Y;x&Y0>R$(9M7~OiM!zK@7E5XB6YfrzAi1C;@HxDURLr zauM^_IlBIw72uK|u$w^c;Ri#5b50gDVyA6<+!upGYmU?)L~1SF@7*OEa^+C8@2?~p zNi`{+636G=>58D-CYQIByfVBo4c5qFM@EN?>Z$F*$rdaMsmRGV3=)1hbB=0bx*#&v zelof|8DE?pNidas$!uWQ$-wbNEqE#rifkhtH(l$MbG%+w9V7ntG=q*CwD&hb+B8-Z zpKEgb<_Kqq9~=7>q${vHwg{@{{JLMlE(HUft?)+_ket7(Now%Qk$0%j`H#k=t>QP1 zer1Y4t$GW1l({3x!Z;%e6`tf%y437yeugzOC3@sBA4ea<#q?|ya3oJK`?Kd2r;+5y zq`OQQ{D|Myv>OV7Uxn%XrTXK7bCZmO1edm2J_4H}`bsm_n#>y2+RruVHOoxg@)Rs_ zL!oHcQn>_1^h!{$Pgh}izA2{(dW1Y01DEu~$()guQ__x=8pWkRMq&CBv%s*TV9sYm zbZD`WdVBA$FU?`Jp|lLFGp?q2=K2w*B9YZTVx_cB$BlWJ92Fc-1}+KkSUIP!Z{Y%> z#`DVpDP8U%$GihlWr}X(?X09|>?~}eUgxL1)PbVxP)(&o!e-E6xvxW{>+^L1QKk4H z2ZSIvu0&JQDBECZHlwRbfX$zPQAm?1g1HVn10@zJ zgOPHgMZwi*yh0I~Gb%5;*1FyM0Mf_mh~Ec%o1#XU6hMu_ok~q)tkOZ@`?$)N9)WN; zbSNEKss}O5A;w!fAs_b@J4dk*%>JWSFuT*Bpx(!f9k z`NH2o*@SYxK1cdJl{4$@!Az{(so{4>gu2Z(+TCpRZYyw!_1LvJ?kDlse%XfMHicqU z)E@G1{6j<_x{x;gNI~gV;0vN|(0kZ=KF^yz6yR>!k^j%EwGa;XtaTpyFzZ{!E>hN= zbrR2J_O;1z9pB!_LgwCeR(F3d?wds%#Ppe?36J{(^doogjr)xq8hEV1uFIzgT=jHX z0`3{`#rvOm-Ywl8R~?OQS&a&nRQpMb%B$UzX~>A%gL+9tp*w~bRevmSEKTX{kC$1O z)>zXRsH(L5?Xfnst%_FRY{#R+r^mjh`Sz;fm+9^m0X*gfl~rpgU=;u zoImF1ssrEV=Q|{Zm6WsGhA1>9Lww%8F z4?J!6F-p(Y1^5Ds#s?J21!>k;=Xs0sUf>$>+{qIp-A2Es+S8vRRyLb11avZ{@_P_$h^iCA^npdl8-6!=VX7zKC=8%f80p+%$yO3gX6eFFjyi9A_||9)j~DEDXPo7%a&#$WraH1@hDVdrVF~BofsQ2%c@gZiVYhSp*F<051A^` z!t4yU^z6g$@eAsk~>p-Nq%c*|#gQTC?W}cfI+BqjA<9&l9m6*6j>z6cA`pzu0x1 z@|e0CrdY0=mP-p02=wJebSn~7|1tuFSDRWo+g2-&PWp~Er{5y7f!=e9ZmPusLm_2Ki-hl7(P2-afSc1qN%W>mX ziSsS$jvd7>FQtd9d_qNG&pwryd0V|FO)^_3pYLPPzc(F&87Z~;x6?7%Z${6;ib4^~ zQQ}J81Ais0{Wp{>%zE@~VeNx>i)ASR>GSwP-aisr3^XOty&s>)XIMu5Hz4sprtSZr zToRTxE@n>uNTQ9AiAJoEqbT_o(R|CMh2)1y`)EZ;-Hw-{~f`V8;l7eTPm z7eUx>dCe%>RC}DiOd?hgwz?gMH$o2_+fOP!Oj!S_(0}3a(5Bfk_uU_Usf-#Z z_jHA?FgIN2@j^?!;2#BV=J6xka4YP26{(U$hLnc8nnmVc7*KcDj-kjnVa zn*Lwtk%jC3vv=g?`bQl(esGYma&dBz@UZ{*FShZ2IS~J+ZRBGA=Su(Gj{leQ!N$tT z$^Bn^^?4VJhuTnk>qnKAH`o+xT5>Kd#cU%*lLy(17XE~Z`BhcS<`@-I0PC0PdDL>; zzk;Tp@!$GP_@W5tH9w;wJTOtt1&92;_XaGjl8W7VJ2qWZEUs$}K zj)Z6kLsWpRSNjF}?`vwmg~A>~k^$LH=0{4@%xF(*47(L2xk4SIOp2{RH%F*o-nU81 zhGKAsb>`^0SmJrIc*{n{>y`VFBH#VvLU+okmp3}?za7F{{=~@~Tp&Nf#}$2%A4*3{ zC(j=bukA=qHrp!I(}=I1^>uV5&)uhhSJmB8N#*);bii3H*D->fcyBl!WXVGQZ#=t!`GUA4L-UOtzGQMz*eN z(7^lylqBC7^QwLNGEs;GnvX0wb~-oPl{|f6lSWYnip?K7tv)1Lwp@BcUsy;lOPG^8 z$1-r)P6v}*DC~7(!%=?+8rsakTHkcBPLTEFa=Pt!Ew497vX!>bvJooNu!Ei@6p~s@ zVQ-vLn5#PLa)X>DtE0XbzGJQNv4dy^{H(C7K*6M z6+0OqoXGlq*ZWlaYP+B3P4q^OOh5v7F(v+V7L1;)%CvEOn~u*=%sbh$4JMZX zyd2eqc8cZx?ARxRTIAY__pSJP-&DAGy%Vx_$1suRK$)(Y5BOSPOaxNacEinyVg`*& zn8zHYpgwzwGvyj2#bz4}=0$7un#?V!yCM8|A?K#$**$PWm!88pd+a&g&ZWBuKA~pPbA`@5hqd`s!Lpb$!*iZ< zqJGWeHern4kKD5nMaT49D&u)shgxt9VkYHmBO_?Gk#_;-N{rLsd@|zkY%=_9E#>5V zcFzPmk?1d22)|3Zt^@xiU3qI7c86{FKxXOPb@&fvsY>CDSB~plZJo292>@&SyGvW} zCe-T&#Z*i0@|3O{ISBYiIt+}{&r8I{Yq$ja6s%2+f83v&L4FzgL*G89i}rfyr*iPU z;V_uX3;X&xw3^Gt`CPCcm(C@DiL07go#+HeeSeV-4@U}qa-0pDh#l+q^c@l()K$MFQt>PGAx(xjrzC2}LR`M(Or zOFXFp@DkzO3(a}&ke56k^D3v=MKXiE|J1FGQj%r-W`%lMSGB5%3vi9=>;=4n&Zn4f z#c(jVDBeipsB8D;Pm2-$RyRdt2GBqaK>s`rG!GF5!jJyj_b$rVycNmZ6K992Gz7*uw)-bF+ z=3{-QSsT_Sv@cUKKAE~Qk^MG4y{c{wH1_t2QX5I5$`m}r)ANyC9U7h+$ZkPujD9!9 z&qxxQ%CLD(Gwd=QrltS0#bh{jAR>EMiDWVh_Xh``W{~~I{xs{Eal0m7MT}rddNZ5g z972ZH22BI|o6azSTPm!dFjKc)fFLo<-p-daq$91Ummf#61_MPzm+v`)UDTdCWvj&; z5$@0zXL2lqh&YW%mTz-eQo;L8hr&UwidcOclZkW|>S)%}I+f&V&1TCe#tz&eHr?J$NuG8-`q{TI0 z8YB-_G>jnSiP=VUSt6aiS~~uRI73A4_|}v6j(S^XCVK!${tu3(_fA%wXHQGq!*$2% z75n)`gi3Jkq!-FcRVx_a>=#kWQ5b)9u8-ARDGY7Nmp`vsj@VB<+M7$m-s~vAnZ`fZ zk>(?Qe22Zf&XCNU^71;o6hDkrQiru^NIZ7f9d41zbPK-}#{||ohiQ%%tiH8T?{2HU zAa!;0Z|5B-YtS)~OjwmDr?#*z#yAs9f0bmBh*v@eu$-Cs{0Y4S`bQp zV4i4C{$2Ond(f5Sx$Fq&$lka{L_b*12XC#DaAVH9{~$ooOdxpkHP*UC3Hc>>h1{(V z>tO|9oG%f7zPvUtJMBJi%!5io}S+a z`CKga{Jt`U&3toBO-~M|b>}#sX(24u*TbG$5_~6Htq-f)oNo8X@~W0#5@W7TNC4ye z49Z3biDuu8<^Fu7s(zY@t)N^J#NF_YtDCu1$hFznLc8f;>w|P+EH_RI&S%{=nA|Ud zD(-LK#{S4ZQ{%aod?`t8=Mz{9(kmbOv-lcdkTRK8SPqNpt{e%Ysa$kiEC ze~YLoGRU+9v(1Ii0lmwP#4jP}AMok&YnS9(Og`sIjXlIC1_?QG9boazfc5B2KjCNJ zW8biS9D_5hJ?o+dNJ)mX6gJ2{UtW+e<@xn@OU6yk!Ltx;Ph9xtd$0g$=u_>ttGW} zuh-&o%^xH)U%g7Cg4|W`JA&C+)88WoDp1Gm6Mp0+{+me^4xtr;7Q2M~edlCJ)ktPj zy7@V>V^f?DGk)%m!FIrYS6CN;LbI+G(N=Ihaw2m3lh z{47}IaLo@2yPIi7BN=o4LMDIYw9DVuO{Rp%+%j)nYl_($_F{x+bsVbH!l_J zFeBt{8tX`;CWJ03XJ%RjhHa5UN_3TrRh+QZ->8Df?SamJtQ5xqr zv|DuH?3RCy8~pthxvU6VlKiLu(v~3JiII+!9>OPr(*N|#yW=-XAQh@KBPZ%*e=OYk zDanN=VP}lx$9;c$7TIX9F-@ag!qbdGcY*y&QAd-KT~TGhvPM!7yumFM529yGHjM^} zeM@e#EVsr~9XK)iqT@jMK3jM2-67FSE3VG&xAUlf9LJnv;-7Z^`F-3AcLcXH_@2q+gXXq9(2*&*ZaLX`S%s-S?r&AeL}+of znN#I`#%+_@34`iD_EY)j)A!7+N$tz?MtvjxE4r#_?(qEhB}pJ&$^}D%8y|-ZPlfb^ ziZ87-mLC}5amS!mA+Fc`anuv&6?!|}QF+#^3`ie$7i|2XlYVHs zf`&96d8L$z=8A{79cc;WX)s=`>x{C`slWHdFpAiTxgApc4&=vD`08U`R)6W-J@T;m zwZT?BidjUfipSbI&WUq%&xqrWvb$IOC^FCQQ2X=Aw5JcWL89B|#_&En-RvVra=F`^ z)0>=v7*?vw!GC2e4hHD^=}6;?hz@n^oo=y^o2=c%pKW(0d8eWJvu>&F_?nrOY9(A* zrL(@AkSyigBSs_;=W_<(I`u}l=D+dK*yQIX@L!b2HR?V^RlB0>mh?G*vet^$gc8tq z%Uv*s)r3}@&URY+Z!vQNjMrk{DXqyOszPY?u__Bhh<;)vFN23qje3>qTi26%yUE_# zcP16j%6s<0uG5S2UPajho4A^t^XARrY3+SgZF}yHMea*RS2|n#tqXlXl`H;rLPP5N zWu#M9=us112Ees@T8#=PM-u-)}>q;)`G~c3aCQtyn=nDelOtV1-aY z6B74vKA_4C+qL^-+lk6BSsUX_j^m!Ms>E3Ff5 z_ngLKdE4@yw^lHqp4z&{FAPJO%dJn#PLE3T3T7y0YMtQ@=d`~a(|2mHY4U3&d$Z(n zP3W82GT>{El4eEzDmneoFxU4(@hdp7$kgt>E^OjCM)Likle}Gwa{rdR2^EA+>k8c~ zll8j@S{-g{xr7c{m$KpXX&m`mpi`>H5it2LjPF@FtF7Qh*+4Fai|BHX_)etb!}rD6 z6?@+Sj;;d<$bEAsr0x@98km#G&k2{G;BoPP>GH)~NwO_L(Aj=U ztb9{e5E}O-Z|&g#2~l0RUcR-&-1z=%+bbmS7J?^kSbq=k;f$fQNAnaqT8&@-V6-W! zrnvOQB)IcI0iCu|%MLqUGK^po6>L%&SdZV3ZWYD2R5O!(z_6i7X!)lkE>-PM4cQ@~ zhKe&cU;rkgTis9r4PS5vv~l*VVM<0l45DI6;!RAG{%tKN?pfPsS^R95vU3d4d@(xh zK+zBKpV*n4)jYg%0BVP;Y*j3|STgfzd8a%=?_CL*8_EJlJ~9Uc-N-Fp(kj5xJ*Xdf z%DlU>TzlHpJkei{Y7j@MtCsiM63@6~qH2%?Mwj1o}*QQ zhYoys&!NLg&+az;!rSB}c(kk!DD}4^oV7Ee}YFf`(J9-PWA# zx*;DcYtI;K8E_#ydk}TJOm%ewbowa5_j`!NL7c>M*IKmNU z@E$3II(aj!wO9@sk7oYw1Zhu`D6}@Svbva;v8-1{eNVo9=(<0fXa8v5sLoGRXd5qY zpsXe#0L?i!G~2(A&p+Eqn71;ho%+6e)W< z!eIDrkFQkDZCiIvEAbQ7d8lh4fXsm90D}dlJ*zH@QnG@9=SzG~))Nf?On>yM5hFlr zULC#Y8DikRM`e2u*t(JOw(mM0Zw$m4TGx^DkGj^0npJ`L3B7nr?$)3Rg6IMC5X+|m z>LYAMNQ%7;!ZrP z73A=sZ942ub-@RP&lYgc{%_0eHD17mo%H6wC|^>$B-*8jwvMU$Tssa=5c?KruJ1CA z!Ull9GF~Wj=XL61H0{!TCC|dkm%#6p=PK}byogO0&cFJ+_c{PU@Oy zWFa{GI%ii2PC%uMpD5!?A@;C>%YJuUV;A{B(^tYvyy*Gj=E(hGz|M5)O$W625R+zsJH2)wvvulaXfk!>~TJGBiOws$j6 zV9_`SKT2P85X=m7s>0+5*W`8o(GTHW)b_-eLGRfQS2l4 zZuZo=&F|zz-lYN92IMdc{Gl{!!WqbqMQi!0KM~sfZz>c@jKc<#11Oe93%lN!Mj)N~ z7A~WnY2}YYerH-vKhCGEL9l^5PqNg@Zx8=Rxi1fUgYHHx^Jxu3bA#P!UClDuD_ROw zZ)~?us84Z)=gehb@$s&Zh8^C10|)5Lp(feUdHJA+Mcod#%qNr*83CG;hla=b_+*U_ z2Kn%wgjyMQx@!aW<#~qJMv8~+;w4Ij>^DI(Oqn>|93QUlOZ*7%d}u?d?V(O_WPVKd z8!`g%hEN9No1ygzBPh-kHq=fR&XL>Ll&e(~^TTdzOp!7lotFaRGt-=3%S;$A&ibF2 z5?`1_+)6sxS2`Zr0hG7vOH4pRZwg@_oUcDWswv?7Hzb!ikI%g2-UG6aWafCL7Hn^U zq3rwAxTYHs-EBe@bVDPjy`4=O&k(MLIxU0RL&1Q8I(hOLNXoH1{@uUnZOyqGuXl4p zo01hGn{dFkER*b(mFPvkCce}$xI3faXNuGrVn39|LS%CYiVR4>%o1aHde>Q`lq2V6l^qh=cs+P#jR8#t0$IASY?B{M1c56<^h#%OG+GpCw*7&1mil2Q} zEHyqC;LK=vazt9qdT`>EeBVV!{KR^D)-+8&QEd;(rt(+GHt41T=dB$z*BNia}TINTl93nObKw zE(%RFnoD&zsX9$gbI=AiuTt3$@Xuc@cJf=L`cd_38Rb;axIe6r-o|`2Ijl?3~C1GSA zV$)u4r$I%Y)7?0@-oQ&C>09qifkV&rG?(0omoC7~B)ibLULC+@{b({{L8wPV;??}2c+ntEI5a(SODF=GFK+O6yk{PTakxp= zUFZY-UTja_a-NBTQ}3qPwV?b1Hqw0zdQ(|adGFb={R0Gk`xImwT-XAca=+_C<@PlJ z&AoGOMQzs4a#?i`tJU8`4!!v=62g4DLp15mbA z9-+J02Xj0>xcnTNSN4Yc5YwLV1*8Tb^Mkw3DR03*;xDz&@`+w2T?3@?t(H`;SXmU|uO(Z=EayDG!e~p1V(Pn- zXrJGZk9Wp$mRoWTJ^8TTm?Vj3dbVtflmME7u3@^OxJM`M$jEvX5c{4o`ab_ciqV>7 zZy@vX4AsrImnVd*P&-jS3ax$F+0xr((aWTXdvCJhs*fbLWAl6SDSj#Kw#F36tjMLJ z{I)M$L{>0JsrUvr@_^`jDqnB;9*w>g5|!b#Bb-d0$yLi|`xCa5ZJ13;(ifbEeDrIaJtdBK7wgI%3jfZ46;Hr@4`Wo%B}Up@KrGm2^VZ{4eek6(rAU@*!2D;Uy(r^Lxo%fDpd)*bc5 z_PmDOwFst=kwc{T$lW_`63%}MJ}8tlE4?Fk8_f|*Se4D2%q>Sqm_0JxpT1lgE-=PS zF(#c_Q;XxhIPwf^Z%7IlaN6MSFrV&`<2QQllg{Dg$rPqjXAT$E;M!UX`+A4Z<>c9A z%5Tmd9+3hx*Jg%h51*#knucEaxeP~>s`1V|PnKD1E=HE_6KrC=)_mP7b9mUqOngOp z-0ni!Tjr%imb1R8DTnuObzeZ+ZI(K4t%)@bQO94WV+nl zv%cKThIoI!%zGF2=#u1GQdb-|uIcSY`EjwLvsInLT4|sqMBN~9INA(-BEa9y`S*zQ z_)9P%(o5A-#ZCXyH_1-^`YYArub-)x}fH3g+^D$7B;z?9Dx7b+Ib&{Db%y zM6-GS-Q_9v?y#RllFH)*oj>kI>I40bFCy}HiRZ+SHaI^ngtZo4|8^JMCF3?P@+xe`;+`9^jp?E)L7TczpsssINiW=ob!r{$yME>Y9))q&<1&* zq!u?2M=Ph&^U7HLs8^ydsU%aApvfe@;ils&X69l{*XM#1!WZm{q9vTB->)q=I?K78=)yvh*+}x&d0ww!WUn=3HZqBmA%_WHQJMJ(!$i3{ zA{)6Q$gVT*$YhV0zh(ciIs>d9?MN=|bpoauzDNK^O8A6P9*0M5Z#WA*l;@eKTytOH z=A=y&jV6odP~O-V$ap4lBsHBmPCz8swHn6&lX9GnZ%&0>s$Q$vG5%|DoRS|H*vesj z+R)MbxCqR=4sU;bK0?o8Ewes8$d=0Vi4|O*Bce`Bm z-7Y>#U6pLlSO0^#w~C7E`SyLI5P}5=?(Q`1lHg8ohu{$0rIFylgG+FCcj-?J^QKFs9M!MX3eU(x@*?@eAjBkY_L{U2T=rUwSpkNy}zxceCOw-rv!QPmnADwfy~=@S=lT6T zNga3Jxgi8DkF)L!V?8%_1E1HL*alcLMLurEwer68<~xC21|Z?s+>>9Vr_T4VNv<*F zNW?cGURbKMu09ym%Vxb$&r1U}eu~`y2`?aZovYh)KwfE=_siKz!KlndFSCa65=!7*aNJMK(q@0-H7f@--z5}y#I=Q;3+{mRg&Sk(?Q&5RJ@_B??kq#1Y=8yZax}t zZBw)|pv!eEsWVRH$f=sGJ2{pR;9A+eP!UjN?O)^mZ6qOP{FasPg_V}jv_7ATgu-A; z+~+3;s#B8*m$qNzevV_hEul-0sw&Y;DRMd^B`@-a_Z`uzKgzX!q6cpfp2cx5D7gC> z^o&OwdFm(O-Jh}`kz zj^7~2f&)3~@7ul6T6$h5-BZ{bE5Hm0jyP6(qXia9Iu0)CCV7_kKsMQ(FSKyxT)vl; zgqGctPRH&o6h%x3?=|QN(cowF-E6E|ygwe5m~}2L-L|Vk7Sm}+Z%kfDyiWMM7K?sS z)h)AP4(r?AI5zg0PlA360b_Ly@M)Osni7BKnVI}Dz@t?5rfu!7k8y%95|MH;yW4)Y z#=Z58@t208#LwXQ#{?LUhwIRfEx-Z7qRM=e>!Yq~QJG+`AO5ue@(Ex1(Kj2Du1|#F z)Q*&Ewj$Y__dV(@V384AY!~IjO?q zbsPI)qptPB_%ptn+R!qHz?8dCHYLJmhv*WruZMAn+WIJZ5mkm`1O5!}DFo233l27V z{PI?A5Zi|PTwN8h03&wv#YIX6u`7NG?@rzDfz6=)^xsCs@dhZ?Lk~1wS*80W6hnbg zEiM?qAQ;mQ)!>a3jjrMLW9*+LraYa@E1hPh_z%3b>Fpox>USozzz~C>cSoMx(5uTP zxMn%m26r-rzyM^F>tTA=|CV%L5P5DXXQ-rJ9b^9_QAed3Hu2(guEm?>v-kOD$KA8 zqAyx8W}-HGI74{y&@$5=z|C4x`1?y5App?rSl`o z*NO-9-9k-dUXu_;VMIrm)R?WnIgx5ySaUccIN&dkXL*9|xn0`;cw|R|S%uyQ`>@f5 zhuM{q9KuG*&{y z`GBJP1H(@|M|h;;Z3e`89xyO&;Vbei0^LSP8daC#az70Fgn2->jexj}V-{Dt1UNEc zD)N8#H~w=@ZLiYn??=;PRto-!M;&}|v1ZHeD{Cxj^Wo{WZ^^_HLG!JVWRT6(fR95_ z%&Y-8s5V^eSKY+~L=^_H`z_+)mD?2^y=klj;yYOX)?L$ackG?nF1*k$y_Je-5!>(N zPq6LC)N?$zzb5)2mj!cOrg8VhuBg~9sgxH3nyEvq#29VYOSnj+Pr}Nr7&l01Cob^p zF1Qh)PPq0n_>$YQgM=*jYM3U*zh>ul0`iN1DPr34s?#spIPJ>Vnz`NQgC|cUU$^$( zG_O)-;`6FohhfX=rmJpnFajScD2-wU<-gUAUlZ@!wESZHkt}b7Ag2Rz zOyGaLwz}x`Vi{AL{QI@b_iwN;@AHZAcfY)SQ#%EHdE9++iK&kdrI)h{vp6i{@u&wI zp4bW=iK7v#w)5&L(PH{*6;D^}TGtPU_aykn7~9(~Uidl45&_(P;o;SUz1{(CMxybOYlRLe;7dJys=d^|O=AZ=EC1y2Z z++_N!7YgJ&xE2W5fUCRU-tncs9U^IbFl&7-rn)vCo(|K+4*?F|wXhzOK8B zA%k9>?_F7df>$uRK+2zeItQLd{>YHq)G)Cj&kt~f@Mti4_f!9G`m%5iA~RTmsKa&l zQ&g=s`QInWbP(Z4R)0W8r(U16nD1bA?rk<{BMzc&o_j;#NHE)$48A%-u!m4OM=vp4 zjbH7$6~JUC@o`ojA7Sz#LpA&WOX*_*P3aYCD#w$GB9ueTiM;p z0_hL;1&FIY+E~)^1AyykdKEclb-et*Dl=M^djZ?*y^}h%OvB4$b$cb}qe?ZalIzm# zQtHBqqkZ2zFp=FV8e+A^RmV#1OTeF=44Su>DGkHEiKjSQGOA;y=CSIw=~j(LH|le2 z%rZ$D{J3+BW4yLM?nGmB;8_>yUBWT~?)B4Un&g7NX6&qx&C0+$2RiG5%T>45QMX2* zYxK%VND9A@X2w-i({g78cp09+8zALf;}R!9e3Z$$m*tB%YxbuX8!VfE>{fiV3itK| zE$N!AttxN9EsuadYacB?Gp+2LXt#p_6UYeVGhPInz^zT&@S<%YS;uK(8K1YKD^pD8 z%e|j9*xnfK4-|}<7a64-3Hj|4~Ou=Qk z2j<3P7M{>Xeax@OnfsMeu@Z~Y~7Kk>B4bN2uPWCou42KG~)%?S>7^>ww+zyq_XP3IQHzG6jIG1 z`uVn}b)WethI7A0sH1TP95A&m_a*#~l8iSvTXkPJ`Bx_I# znOR#gxP=|*j2fi*boQMSYG9k~RCGs(V7cN#QROPD=(;z2$+IvWZnp=aox1Lzq*aa~ zr8F4yU4Mx?rrZJWlSJmXiu4oQlh#Tk03BXS7*Pj83;eG|B-DUc6+r6xtbljlu7ULS2oZx;WxUqH@olt)qnhKmc*ZBj|H#{Wx{PDJ_v^z*1``y% z6mqc$nZ5gXd|;3(cwJsip8a(+6Me?)Xz4UI5RUBQM`AzWHw}*K?9(S-z(cU26mStC zSSG2@Uq6sm=Z(I$F31M%eepX<3HQUj$L%I&xTWJ?oyOnmUpmw%i?cbluf;oanOh)_ ze;q%PDR>J(w?lPeAh&lnPZa38CO8RxyX_O9}ey)<9toiuI^ z=s&^bs&#}TA}minkD51mV>EUdQTsfx3s$>V|JAJlJ5HVx6NSV5`y4@8dG5!^9P6X( z>*Lm^{$>(Nu65U*f{pfY=FPG%^4`E}y%5u(QVFB}Yuk9G(ebU`xnqCsyY@Ek6U{H^ zaq(fJ&{c&;aMYU#tY;k^stb360FqiE-kWL9IGr1ukR0mHmC+d+< z8cKJ3ZujS7$qgX?PR6H-WfO$34+77G+@0lvmqYVMX4aQE*Y}UUfu@t!f2<&p90`4` z3V;xQMF$~p@6YBPL3c&N=UZh)*(3-AbIdI6)0^5j{2l|Ir0aL^4xQ`uzE{WWz8n+l zgKEs?pywQV{+~X~+Hkp8O}f>*kwnAJbIGR9Q=|9pgll}goaV)kAnlW7)4%=wplge! zf)U9Q`2gE!C^7W5Z-lR3Pmm`}#v&Gu>RJjGL#k^&c8s^tX>cdHGUQYWfag!HGb|(F zpx7Fv=)#y>XWw*x1sUSq`|F+%TdsMI9Dyy6RyvHg{pRgr+f!gobPgbA%iB-;Jat|Q zH3>2;T3PWbB8)>)Xlmw*AsSFuvgP!AOhjJ|PFjejuPyyFqytc$mC!5C zb9#RPQY#L)wR5LhArzFAJ(XTFxon=9pM6_da1jE;ip^aQps&oaCl!BS0<23_=KNVm zK&DYCa7YxsHz=j;d{vgy@ThU=o9ra!-S^vNDu5C}>Xq(U*dYuN-~~i=Cd6zLZ$y zH8EcGP_KE?ow5#x*yI{EgTh^sAJ@)o9E=|nR-5nLmjiZv0=D4#)mQP2vrpJ2moWcN zyspcI%{~5K4SYY>qO5;TZpGsbJEeh-3-b2 zg^L|O0B-vVhn|lrZ&%ka&T&i-&r&ZtU065R^H(rlMXIiW0`!%muqx1bYya_C7f0DK z`}}=}%^VVcU+9%YcGHJdp*|(P({yo|hvynARHZA(;H@C!?r)CF5wG!1Q;ff!!dLWp z`7Tezg94s~;7fknR>`(~Tl}k0+P%qdXML<d+ckDOS;>y}&1@I!mSu2O6M~bIZm#2lca_#{$8*^z|`L@9x;xQ?=M~*TuoR z+ekhHi6!mZY%uXXMDDzZo>y9{>#KTVKkjw2>BmBzNrYv_okPaW?l4V${Qe@UF^@4W z>*pK=yD=uP1j_yxHsn5$0}6gb=hrer1oPsluNAlyYEmqc4YHH8f7`+J?d@Oa=#N*q zDv+$qYP6xgae95@NM!N%fJKC2n-3cAKk>Hxb3ojdm>hE47};sAq}f^>69r~2Gwn$f zZJq59oYlM^m_%>lhGwTGht^gis$%vy4c>F2Jh->|^tWK4a5PC{V*N?+_V;n};Do{o zq$X?!hcO8kJ88TZyT9yr3hoe-6|NP?6uoL-;=pnFR}&-# zA*yLI6xgmqYOrIq^Ks~fn~(&Zd{7Cd>*oNl8>hc-*zDQ^X%N~?0opOv!H@A7)iCkq z_v-s-zl(vcKHrfsrDG+Zh+WZd-MKAzi&Ye-nm-;h*p#qYRmTZ_C)#%gHEtVB#JXAu zJsA$D7m-@MOst+$_&{8^Zhsu&Uz~pKc4efdAhhAPt``uM8wQms6tlQII#vZ3Yjjzx zSxt>w;g(;oT`Qdd&zNrbxNg!Q7=^%f@`0PRTV;-Qat5vdPs$jswQt>U$Hv0NV8wK$ zu6wBG)-P8TkbBWRUs0#$!*$06T!ih)??pAA=5*Gr3AsCl)^WW0r~divoDVIrA-H>1 z*7q9e4dZ9WcItdRK2Y=e!Fic1`D~HB)mG8JD#bZw*5mIGkAgO4ZI0mGN0D+%I7E!{ z`W_OhRqR?^^yDF3>_#PQ{8Yu*UDXM}Pjlqa0cKjc=N!lgO(5n1xGG|7?oC&b_hlB$ z4I^qg7Q6|0iZQ zaZ1nE;#eD(^(+Fv^~`JBB@rd4xW!>#k*0vUH(;-b*ve9 z)i4|Fq(62GJa@gmYZOUdzvh^)<)0&XfbyP=f2g|s+43s~d}dOEdn|`&P~AhQ z`KoLhT)+3h`wMxH$G+e}at1l%_W{LZbY-(+P4BP?=WxR#1tx01>saE%&8k^25pSaGZ&vr65N8NR$;U3?>6fv-SAO)$A|g(kg13_bRBJA zdi`s?*6^o+#^T}40XCReK_%OQFsacfp|%cAMogwkZ_;A)l^fGM5;Za!%Z0Ar{6j&&V_G z?X`5@U^(0HSa+(mKrix3^v`%d*D`!re;hj@yW@F0xgJ*JC_IaHJKl6C&!L~((PJ5- zFJEx!nF)2&j<=*}g$Ql#?`-TdRHv4j_uanUJ3{hZOrDDJO;4JPzIR~+e;2Se-MgT% zsa7tjDw$3NPSgP>YNqpqd?~6TQuXoR&2z&A0ghDx+8&s`pK?@W#v40jV(APl*JL_m z`mn|ttm>te`DhZWm$lw!F*1{4m@hARW3f%-tRW zE2|H7_1wlx?dkoR1^K$5Vt3_P3q2ZOv~LCA@4sPBBz+m7ZtzPLWi_kE`to%PjG-ipZ7hcu;Sk zW$aHitDES2d+>8~gp+i>GurY|SIPc1hOVT+vF*ZYCWs4kZ8pe{_4MEBTYbvO>P)-8U$PsQey3 zip21!=S_#)#WwK@h<$MQCUSb>z%DjTJUR|Vx|-hpaOIlMC&tzJtZ;jM7xEvQE z!YB50zE{%4Cr15K&-0#rOs}N@V+JM2hex;_~l+nTY8Q* zElT@u1{DvR4-ic4ZYYwC$v_~;NKY~QrOapKnh_R2WsUKH;qb)}(XO*|u+KF@#pi;n z(+AGi>X`H_@_@cXaD8y>>^{;`c1v8heWX34XLoZie9HAL=gWOepB0Vqh?T~?oTFci zpTO|SEr*4%)mjWdV_Y8OmvA%L`s5&2yUe-f)1_>&n*3pJi>cOf9TqGH;P>WwM zF^_ZJ+pIpOvD9oB^rf#Uk_gYZb6Uc1)gvpbD8?4%(`t8+DcG?Z`yv(M`K|ITEarZ6+%jJ) za$!!(cR&edn>==Z@-(>jpdSu#4r=IKf}GTm<*^Q8ck{_l1$e$T#MFlfVsbW*Qpkbt z9V{l}Cifs8?`DPj6rV`{(px)@%x50kX^ZibV?U-hTUm>#-^aro*Gt2*sswZV@-Dan z)aC`m4s7Yh-7akduDGVg3c<%0!Au827EAq{#bOE7!=zEhzjlQSpq^Orwaf|^j9@dz z`3rUg@ejut-1Yjg#oVDR&4mx(Ytrq+Ug6Ar-=K#-@P);)?K#1v9H~d<+R$|YJ`Q7) z=VbDN{lfF1!TZ6&b-EivjR=Avhu;}3f{#R|L1&MKN1ZyO_Xg0&Mylj~L-KP~u&++;-DH7asPj!|2 zVO#zRNUq>M2@a>N1iOPq?^li4p8uPdJght@w+}pElqPkKcC91db*?v6&+SgpBUm3W zN5%ys&z~~N@>bOMdL(O456Hj)ach3z7eP z=I1gT*U9PE-NHZei|TY;(1!XuS6^icre5d-VcsFZuVb(H9yYm~mb?Rjr0$K` zeEUYp!8KFpPojZW!}tpqitwr@JT>w;1%lOsy_N86{~^&YmVOZ5ig8?+#$tKP6`gDR z-}DOU-THpo#7DxLueq+It{+s1b!`pLx!?nzigL6^-R35%K_?37(K1$MGAwT0&z=eH zbk)tc?ZQR~WeEe6hDoj&9{USi&_}`=Hr@b)36;b01no4D;fgi^ve_u4z7LQ0=s>a@VYIq`xE5%!Ee>T=~rB@8(ycrseaG$UKrO4P)li|JAn}vT=ob0 zwdHHgM(u{lpUVx{hVviYKg>aVZ}5@G(b-T%NguS&V>a~v@cdB;GDZSpo?$NLePLjo zVVdEXp`8(#VV>c87Ukj8AQKekrgbwo=qzMTs8-NRa|2Ep%6ZbSrb53=naPdQA(#)F z3R@D^RNypr;fXyqn_8Pjn|Yh@o0sPtn>L%eo6OzM*d)w#5)iAeCv5Hf^=5caTzG1@ z@p~|d5w1UdC*^~)ml^ChX^k;)dQGuHu|mEg_6=1JH8u5ad;v8MwK}!7RCUA(fi96Q zo;Pnj=cWA4=s?Cmf4n^*BcTnU64!gqFe{Q@9Q0-?4QGp1mg8fCql_bt8RVm4oB=M6 zZ|_jw6%zou=7Kd|F z8N7kKe|Ycgks8rkPA5FX7cc9zhntfGPJgiy))o2=7Si?u1S|&I6!uu!3YWA=OWDfL zj8PkT&KF5()APrVc8YOpW-16O{V4 z25s$QCl8DOerez&<>Y>D2;Gg^nE#U_t~Ed|k8TmN6e1Yj9_o`1KuA!FKlhS57#@^M zr8i7(q&Q3|6q-y0CXy-oaNF2SUCA*gYSDHDgGy_~GO=SrmP77C!^36r0Q4a$7{;+6 z16$~>GIaOxTw+tETP5r%0@N7eXw(%`g$5c7e9=U+{=*sB)=y<|u&ND}?BoR@dR_xBl9R+_af4RrWJM}97 z)6bxL)FSqZ+EOR;O)nMWSU{#W%wV&;I}R1hvZC4AA{%LmJIbb~;}$U4ZpuieAza?t z2^-i7?H!IZo`Fvp0DMx8QKF=VI0z~@HQYs9!6tph#dla= z;SAtiG{c0RM3>quZQPd!z)>hkW@9omf~(tD5MMy0mZXr;;KQV@b)A9a$Elbz?FfYGsx4(RZ9>cI4-0j zG@NBIraq=-^>cdo@MuSCXS${ssML~iGdo%lRe^dSq3rJAuH`;`BZ|LU>x*3i zEHrmPLtV+q$v`Gg&9llC)wZh3)S<|ss$_j8KJ71^261RF6!s-|@{T z_)UNutoJDj5e4rp*U@p7=+(r%VrHvJ({`5YU5OOH#n(2_aQ@tD%sf_;X6WqDyNKJs zG*-@Ew)HpC23JNKT4k+0Sx%)l*_oe9RYYYFuYpx5yVCA{tfK#%Bsra{X$Cc47gdcr z_^%W*CfR&^y`>+ z-eC>g_U)?X*R9=J*In0W&C|osICFsi!~U_q&wh!f80+1|7Ww#t^Guvb07JGePlLI@Stv_8`POf3-#cg5e&n7TlGqt{HMh$;3}k?;b!H3nxWX;KiCuy~ z&6{6~j^^NOg#67=rS>X<=@IYQv=0 zNE|^~KZKChBJ*>+O)z|@if?0kZd@Ydj=gqp7v#>i(v#zO5I=JXf2 zZ#DoE+n9@hZiE0?-SFY*Q*j53T*C6j_E7-I;%gr)%ab5K69N*eE}cRvF!9<3O~#-s!l?0LW9pzrL$mermbU@=9VaShD(*8{xl3qY@Mfg_*BbV!sXJ9k(MEV9jTVVD8U_+rB$_nH0 zRcXb%q|Y@B45Xs$V>R!%!p~Qt^gVy!_nyNtkNf$SzdE6MZLY-P^x~u_IdO$&A4<-C z8_<{$iu1irr79DW%zD{JF*8s5TPI{jmn$U&(*W#7iLT)~$AsohTV#W?Nm z9Tz6bfey?v_01d*#+w@AA=}43(n{MXvv!>Doh?MyaTEDn0gg!@;DO!5jq2fD?HKk% z)OliUX6+fW>j(b`c0nxvMcS8!@4&C3&AJJ8gxFAIsODd7r?S7it5bO8LHEFDM3hI*oH(Fd2d;N!B{__yUwTDP=b~b3Z9-+Yk?07uN1h$y#89;3-k6 z5GS0TBn*JGiNmeSATp(g$Yw zV^N~gwU`kH9wCp?1TB(%g59lB%~*ZKk_)Ju6JW_cO6pNsEaLdj`&=2=mMokTCDE_C zGaEP;hB*nsDYg^-5SUNIQMAVgR$GivM%xd5&E#4z4yxR=+;I$Jdt;-|yPfD)YzwZR z>_1toRvI$Nbe$O8(4x8CsfC2p#-vFOmY`F*iRSR`G}MH)ro^4?72s3e6#MpZ2h`hg zT@r3TpuQuIGY5>$H}9BE2E12!kB4JvfNHJ((lsl@K|SfCnm_iof$UxAU9}-Y;b`c| zTe$-_Ns*MaKFDD8qtNA#SZ`ipg z&6VoF0zH>Q=mBsuJfto;w4G&-lF*RpuRp#6edY1W+i0sxw;iKWYTx=}t&es6blWN) zIkKg!a4fSva#Tw}Mba-L4_vX1`Ml{9o>QV?gGs6(q90G4?0@=8mZl*n6@ih9>rr)F zCO7M*ORAOgw60ns1Ro3LO^%o>PonB>O^%q0AGV6BSsA2NqH;XvT{%1&(nLchd$$}7 zx=-SI*V{r`TH{!xnv~4f3My-?v2Ag?qVt1Qlc+-dWv;#>*h59QBeNn+anvWf817}0 z+T^78^0y7u+QfXK6_^l_Wcvw`DjzH!3KqIF+(d?aWpo|O4!od9sWXm6*N(C`ZP>o)8qkuEJ0=r|NySV9(ap~8aFXqC#yGNy$)a*< zF<3!&xCkm{SYpDbuqZZ7SfDoN#K=oeN^C4|tdQQ~)>r&dNKa1!_7;Jnk*jpr*rGH+ zOPW)p1X2(@E**W53T=_n>7Gitlk1;ru@GJIpio?2o+@r$xOa1Fif&^rICHa8CEQKv zid+@f9wxZrgfeuCSRBX@-Y^D=YqU-`p5Yk~3 zb?OSSo~JPEhf7*mAy%IWN;_!Bp!;AtL4XY_>lPc|etLA-o7>a}GN9 z27~$qz_DeA`L)3LD6|r^w!Rcv>3_D~KVyD<3#!7<9kC#F)hHPzIq;K^osS}Q`esNM}kE$z6pbL2(HJn;&}WTo{GS#=lLp8Z#zPc+svDbDKH!_ z^V&m@?|Ucy`MS$+JSG8+C&{JB*PId3xCi77%ssL|gZD7Uw)W^$v;-j4rNvjjU1b>% zZG z_v6F_h?}H~^;JGG-fah*Wi`Y80$$4flthu2P2ledqm@#lMGwY_N3)?)qE*kx8(XxS zZ%gRkGx&|9_jhm7Ab;xBwiZ>AVc&w}#whb)GL6>f=VQxZP%fJ0Wl z!WwAeBj+ciAqqkk3G-64#7$0d=^+zNdFf#S#?(4a<-Xxfmk<(0oE-)Mz1a8&PPy3R zK~AaSA!nD)Oo|1jHu>rl+udgpA>LruWv(y<<{-AW4q1G_!1iUDFr~_ndc8)t&EIfO&`3TUU#2oo0VPj#ENe0i?_J6Fkh4PDn<-KYYkouuIg&Ov|ue9IZeI zC>b#B0f*Rj-=8+9aqgm}o2ewp>`JvM77rV%JS4I%=5*>|I(`sG}@e+ND>SttvWZOx;@y3eVJU4{m2uEOu$j%M?+I=>2N4;@%UY}2!~RK872!JNlsQz*b!iltz8R1;N7<5W$#iwt9J zSjI5KV<5U|tZDdWOXW`$$OK5pt%<&(ije9tN(xaRz1aH9M(1%(MM?Ds8;~GLGqVy! zgCjTON~B?@v)6mF37>mKOJVV|uZy4+GVQ_l-r<9fkNMsUC81c=~^oDzqy)zNxTj;V^2+ zSMr!f+&g|Bx#~`NV-bu$7eL?Djk-KFMYIv>hh5Kh)9pXlq~){u{Y{m<5x=S(tr5T6 zQaPg$KTFT4A|R8B^+cxADJYL3ayJZZ_{f*I3eQMn; zQ^+(%Y!4UjF%5s;5FM8I_q5g1#jMrE!$DyWdvdB)Ns`tJ;tw)O5@)g`oA^%OQBz5< zuwO;IHlA%HC!oQ|opcK@R5wJ-&d+uoJB5m7KZ&19HCM!LSg~*C)tVaQ)54cbU=x@a zsfT!ot=jxy+U;`^L&oP5rr9a->q5TqdkSEI2a;8AE{Rd$IGgX$Ic>Y@N4e{Z2@|5UT(NyKw_KPrfZvWz6*FLVH%PEb=Qx38~%@!Opw|@BIh(ff(vhWd5 z_bc@qexrt9zRw2abY7l{*e}+?nlNpZanKaS2Iy%tsD5sU<^@jJ1}Tyv_S(2(gYk^# ztdohn;?#?qW9plgkfix4+`A5cTrLq*R`F^*XVT0h1+Qt_0edrt`8Mfizd3th3w^_U zpXx(N7d~Ld*wD#Q1{;CQieDBet(G>=@j>e!`7AuF*os z&2Den;!;UX)E>R5_`~;-jM4>@OI^peB?I!eiqbBn_t3H`#YY_w*=gebVOR5qW)zMP zXBSJ=-Nv>=H_^JL1Qvq~_gU4C>_0<92Z^mqF~~aOIH-`?FxHYY6zXi7NjzN_#|>L^ zKTX4Oqlb*(NR3`f@5sC9yQ`bwFu=s!`>CS{IV(o@~643?p9)e4T# zZI*>$e#qt`dhKaZVyUKCx%m*0^9ZS2U4f6+UtB&W+oxDR&F%^%J?P?u^a@ z?{nO1ks*5L^3xfbiGhO&A;B3xq3^eemx-5ryZyU0LvhZNXL8v%SlLchc&O63_`k7- zcnR}*hNy{ZNE!cy($u|>bd}@5iTa!`T!EV8@?LF7J%Z#< zl(l}EjwAsJxk?D5JUiYPnPr1J(=Ja+6C76lN3COvk@F6-=;mow_OO7fMN^xfRN#?} zvmrqWja-z>Op;tMtdggZh1KPzDE!BXoL09AYrK|Fd9}lqAF7v1(?cM=}eI_1@y6HH@Ds9s0evf;S<~Lq7uf^;+GSU{OSl#enUfo3*|#?kAZAUem{?3 zXb;{tADb)+(`@sGQ^NCxKPc_(XZ({Dp-I1`@$Lr!(VHJ%QXemW^&y4Za^o#dM3RD$O>V@t+9wPH7-wfBbcHCFA9+&$toNbG^-UYOL@i0pfwm_kkjomq`1cl$e~VuKPdN9#_;}*rVHe=}FAu2XuHzxLwpcn?fU(rtrvO~hWp*?T z&)MqrX}bed>vkd}pWF%vxpdsgkJ_T2AVoZysAM1eyASmkafRpoQUQS zvE$RdBP^AO>ZvE-Ar6da5qlJS*j8ay?0CG)!SjZ_4tP8UBRHNe$)BzqpP-KO&*jqu z!zR;p9NEzWY77)2%>7tYBya=PH>XS*c;Y|0euQ>QUIDl#10MU=9}gjtdXKPv@Du9_ zDk1lm($lzrhZ-=#cpmra3DGb1xPJQ-R`F<{O(ry&{6?wOUp&gYWKeUeF^9?KkPe3b>^DlqZKlGq88nPyd2w58B5^qyM}J(}-YB7qb^Ud! zWoc=f#EZ?0b>5jGw$IB|eQ?XD2=gD&d^6hWzm*)^5nAw+WYEB;-MOWLYU6br8Q9!5 za}bL&q!L}`Gfyv_p)fVjhCQk|?9?yU$R>A{^H!-^Z2M?(IY;NmzDt(pHYCCFVdsir zX37@M*k$gi%lZ(#kRha8P~(;qyvtDSdZHPV_*XL~SSQxm3!Bm~_u@`2aE=xhub*~c zOJ0TxVOEN${zkm-{%9pP9q{dAPAbd|+wU~^tU*XSudVNnkvSW6^sVX%39$U4#JH*b zUj2||=?*1@!CLQ5Zu8+y6f8*{{DZEQis||x@z;IwRHVNT+Te1wZ)nb!aljY(Qs2p& zcSM8A7~J?5C#)jVv+r259=PQsmL+g}j9ruuOQy`%;`c}nDH+aq_g8-GxwHu2Z?$=< zz|2>&6D6xUrxXtbwb?|~1V?hH1X5O@(vaVG@#SsO2mvu-@4ek2u_DP^E;j!0_|@?= zGB7#z+NzL>-lPiXVZKUI&l~lT-)*mul@?<3G1FU2jQCFJBQ|n+HmYZ}_&rz9I2Wl8 z^VUr0JLCVy1bn$_(!>Ee_1Tjvv%1i+0O@~mkdPqc?2>;wMzvV3Nh0* zry}jI$}#kSBcuAj9^8sieJ$!Mnj&?0<*$pHCw8>#n>(Y79XoP&+p0qLqQ5e*pA72T zL5Dpu3suM1vMOgZ4zbRKBG_LeW6Vex7Z-#xnx$JnA9>@7NlmU2kL+dKj+sIfT~Ubc z%6@9Xr;Dpht`u)3dF;`DSta}|yY(2MJtb-fm6e>uPMgTgY-G=WC!K5n;srwkrjS-e zLOA`3t`v*9V_eBs4S~^xX?qoI+Y{-$f8vCr-X*qNZ=6!vPY;0!Ar?g_$^MorpkwpdJ?Dw!0Q9VaK`&!#f-IgTM1v6Sl3det@PtMDLGz zM(sjhCVb~@QF+>*ht$%lTyM|yg>Al2opgJ!=z|b;7AQpa5tAB-8DLVccnfpio;-bX z{gG_81|xJWni(y%xcfb>nzDlnE$ZiqQr1G^(miZEg+p@@c_NFdNdtk7b|RL#ikt+Q zvZ>no{E3*tC1ox62ks&Byb5P@O3gxnSIi>rw~@2&P#b*aOQvSu(MRysTNnEdWdUB7 zqf>eoas;W9gIG#&J<2PDXQgk3+=pJ>z0pWJXlj$ETvjP-C0C}6v0O|o)}ecwZP#?^ z>}l7r4_CW0>9_G{1A|n2cC{P1iVP z@z<8rw!ajrqt^2J=6-aEvB$pBr@k>@q{y8%sQZlcR|UNUrW!_Y$WVY3z$WO4jk=R)vEH0xyhAQxS`0{!#I#FCM^+EuzJveal;d zkuqL3^pm?$C`dZy_1m;<0bVi3lL65xMLorY?SU6VI70P~m$syIDagvtbJ~zNAlYbA zc8uH8;TG-etKOuz`SL#UW>gP=_RoJ zv)p;smB5B<*Y<5EFOf~G`J0CF`<*1TL|T~s57apqD4R){n?aCtx*L03{fAWYK&DuL z0LwIb#}dC|xq)D`nKv70lZFDY!1@I@PTvMRaldk;Ga-z@uab9Sre9b-a%k zXpL!n7^}55Rf_=5i0K>#Q4cxscDu@1V;@jYAja@62J%u%o0h3ns~SF?2JzYSlb#H5 zKhrBI5`~PtWlncrMiA%_hjo?306`yz?LmZV^77}kaS;YZii7`n{W?F z*|spMP5md&tm3$~YA=z!cmeHBva5m2sI}Z2JPEgcO7tR?20A@s&o}3cZObEHnr-G5 zhJCsP05OedW^FwR7{j#F`m=G)`R`E!b6lbeDtR`t7i3#Xf;fhM@>``YqkneoJLG#a z`pC5P!Vbd$3;1hVNR`y(hw8^nqAYQw>l1VvrKKsLHyCLk6yOPJZ8YK|<< zPs<1RN-W5#?5Rgk9#k0YK2t1Zb@keArV(xQO8%2Zr1Oq<{G)NBB4mlOp=eEUrsT>PCsC&**k`xP%kfvuu5xGW@Ay%%BSdI^ z6=8g)uCsebP`vle)~>;zSEOp_!ux2(9Ut8iB*Et=pT-u8ZZS*!< z;Qk#F)ccv;EE^Lljg?}}oQeylYnBG{ZGvPr^_wq5=<|`CS|q#d0RuySyCCcQitJ$0 z=TQ5>5}X6%CZ6%_C3BV45EnPgnZ47n$Ek!d-A^`AR37%pXsfH$KFRzDahfCZf6U^h z%0IonlSqh{YIRtDOgGA*E~+qae96b<3E{Z1u1dZCN|8QFYSJIo%EwD7!3vFaV=3j} zYdR*xY&G!!yvb)=pM>7F(%IJq*1nJL=Hh7 z2-I>WgISbg_+!Z8#ovYPNFpbjN9-~gicmAH97lL*qQ4sscPj8FIVONd03wZv;Anc& z>p9DFEO|jmb{nqo>v(7vp}KP~;>**pMdb*tLFhwZSqqUyfn!x1Fbh5>&hV5Me4r&| zcqgItEWPRw714f+%Nuvz91!}9mC=4_tj^sedC2>jD&hEtEbSM7v^IJuVs(CrA#S=V zj=G>Cpmy^YZUr|el{2_$io-}o&MV+JN|DDI3kKe5N>}{@jvbCGv+!P1l0Q3sdD)<4 zRO;H$Wsz2Ti%_SWZjT8<0gjeHv<$_X@5b@kn(0oKtmyb^R%i>mNwaSA23^4~mWZsV zl|}X{bjG4LVpO_LOOix69QE+%HkaU0d#yNjb%=6FLv4(yR$Gq&IuuXGiuL=Ado7)W zOW140*q`yJxc&4go@F&T8K~PL4491Kj6)~?E#<4x3K&L@MVdTYD;NqAW2tMmW_IgA zcA^!z{viC*E8|7Q2mC`o%5Z3K7RBQ%y*A7za+#%`9%y2ljrV!_;{Mw z!;{z#37A&=yg#cN(c&;mg}Ggv=t}PChvV`<=z5rJO@iCrp@vB4;qw*an6l|O+|DNH zZ2fO)jaw8YT#+(b1VlxP#+cHHBgE{ZMbT>Yqf|`xKKHUfqMFruJ2B>F^M)QOUe5{o za-kBn_0yFD&vxffjH9dOrzddZMopv!5_D881nVU!@}hp<-zWtyN z@>vfl0+752%LYJyi|xB@XOrucO#P7{z9s0!M-p#-!;2?U2M=J$%#U4?y}~#kk|H)) zT;IdfDI)aD={QHs6XirgA-TfxE;(>|V>ga0vTP4I9>&l3;t%ANhwXN*cJoM~T*d7( zA}roXRoKQn^L?LhVyvAg_2!b+oa)bKZ%Urz@uSx3`;(%~*b%7< z#>P}dFi-PuGCPe7^mr|N${%G6xr+?Fs@f;cxZIeAKi6+Xth^=KhTUcjmIltw zCWIl0%BRYyzW9FxEu2(H3seHbOJX`{ZR0{3L+ns)7hfP71Vj+`-&HVSjt_3&SjVZx zQ9;`0rj6@diTw=e#~=miK2=i&n_vQ|X!( zQ(}o_L_~!RM`n}ry}zp*XcO>v{h`c4)GV%J{Syc$O~`3gSZSBMi;A=d-UH{PNdu%E zt{UDtyem4xt4+GRcX;h{`VIC85c?LVlT$a{|E|kwFeiY-+`N`Y3+lKY;c;vMY$s6~ z5AY#TZ2`8Zgk~f=QsMTqrHiby6VWn$Q6FfU#zz=WelzFLnX3p)pykwcFwSc2)Qwrw zCrT5lFYonVg-l^du3WWGeH`R1GG2bMtB?z(=H3n@PWmc$~wVv|t7ylT8VM_?y zaRa00Z)e^Y{(9cive;oGl6C;S%OH;mL$>N^`R^ z=u)}zy8qZyhMcDA%$h=?w}l0R4Mj_evOOcx2p=#E3*-GaLqE9&Cd<6hE5CSd$l0E833zw!o>Y z1s>SqOf|@a{W9xr?bmE|zGCRsUp6kTGzyFGEE$mKd?|^<70O7;horl=??)Pwv=KW} zMRh_mT$$5^2+k5$bXv2TFwTrwE{Jyhx0?V-GY-ZdM!A?Cj@ZOc>A}Z(SNro!QUM1mqqwW)#^#^R4Vi~(#;Ec2 zj_6merfFri=`d?SV1Hh>U20yBP+!!2sQC=A|Jj#`toMkclSI~e$r)Mt2HQiEIZ$A@ zv=ENGqCI$+6bJl-nA74wD3@*q#BMw$kSH?#(I|g5w~_LPZYdMFM0tkY`g~%Iw^Ohp zRHunB!p$PzdDdp9#%r4=*&*%o9p2r5*xyIO=&)bc>}MoG>DeDcJTShBmQD{A>>RNW zgOKWSf{g`jT5t4sbRdK^^Po56ZEFIY zD!!jGokTy`BC$H5rBD-}vczmN`tPxif8A3OKb_+JReHMjwquvyS5QrcI2FM#y)-WHi`CoRmH|xe8j`1xx!cN%hx_^<+vPA{nV~tT{^&{E^Cq;^JsI z1EQ73-bt;U$4ejia?}n#R&@Zx#Sgcbibs7p+vgp>A~I1i)H!5bafbTq&XBLleowD9 za56t2LGrTD)43FS)fU$8e}EQUjm|Wc2(}sGuaHd+A76be-e2ClZ=G9xs#u4g?#cf4 z$)%a23~Uw?80gGpHoPhXDOF|jc3<2j;&cR6PJELkkGmrACJqsk!`wb7EjJ2s%7jSv zwG*&dXH(EuehWUd-KGtF)KKs?%a&S7TPFxH(^=O8^PzJkraFZh=4!D0Pq*y8qVdVrN#<=_~w_4 zQrVlG=9iAH8b7yUAvP@WPHjSN<^+2;LuOOmA?hfJ+GdYm&C*vib|W&Zzx^ygBBC>k z)#1~pR33$0CK!-hmm+U;NkXo%rT-dc=%~`k6sgKQxK65ZMv)0CqS>9_?Z6`H;fNtY z(h(Em^-V$%v%7}(>#{!TmBEZHz-h;Ur2ZmFom%htQ=ZF|6Y3HdhQNE_@4?gvy*{n7 z#(~{tZ)UXbD?K?*isl6#Es=5Ms?)_vr2zGVF4B<5rmq(H$-a#Dgc8uu={W)@JuZbDyLZ!)(|408Rs+U`+*97zpCAj z{b4P;Qs;Y6{NzbSW#^qss0sq_`EzFXcSXz9dDdAFvjBtdr}apC*T8f_fYQHvWppdP ztnQPh6|>2}G`-LCOd$~g5b*U@FLTI#oZ3}#tq@1~4oiqxGw}$-2zGI@$(qO2%G-|X zBWU{U&}P)Q3cNVyY)Q=k#tWf6D#-Y_K?E=E6P#Z^5;i<*1`+t10%IMc$l z@{@b}#`D?7D$AqeRp?GJW-(^loFk>-`@qTNXlV_JgbC8Fs>*D;Ayu0?HQDp^kH-|v zp@B^VUQ7hx)7R8t2k!pI`6PJI>_{$i$O#&pwPwuVTGv>Lbfp(oRgIPCSq>&O1;996 zfVNvFe|jlXQH^A76QNNe61=TrQBSWP9rQS!e0*&5(zoO)LG_kjdye=kA6@c|!^>hMO|vChJHV$;K7L zIB42xb3L>42bV3~bO_b#zvYQO!6xss+Hx?$Sb~Ab_*-PdFA>fO-y?Yq1-) zm^Nl}vRXX|JKBrWH6nCoweal%w$(&pbq zK>Ib!fNW867@Y8m#!!?}sU64DBV8F~L0K2|BT1A!d*RGTu(r;G=$l*tZ#R%5suIn+ zBEddAn`A(muu&d%w<}qGB#WUZBw=-Ccvq`r{mw$f;$asnkN;2c7NzQ7vSf6CM~ClB zl_9}tP>Y(0klF4>1O62nFJPnb9+mRt>J)r^5`TOyR_wB$FkpD}umUXwjEi!1?` zP>o58hTMxP#`z^Zp^dkO9$&b|AL680~JH8hJyGk4ATG7h=8yXHV;YTyF zkkSZHREb@usw@4&6Sr-r0pvE7>+e|ps-KA|8Q&=R`j}?V-FM$j94X4`#mV}mL=zwG zsxQ`6NPV8B`?J<0yL~OU2OuD9g|gmZ``xYq%a@j%jC#s-CW}CfpH$?#+#NiWN+7b8 z4L-_YRFx;k@z9Wrz)BaO?9P-7xL_5;zih5j-Qpr6<3Zwnnp_t-Pc1f96EPF2N5s^% zWhrAi5tL?9%d{kOt==V3+jDdM+<_v9)$tBP22w@YEKuA;Y-%Cof>hjsg+*ckZ16%R z98pP8byXQ$byJdb$m*Q?AFTn?e?xYR)H}TNVLqqj5bUF@psFmH<1FrfDzyTTMFk_O z*zf&F+0IdDux?WO4da3Hk*0gDrg5ZLDCwGeU#UrO|qDwWy(nQ|<*d|hsw8|LtDiw9Vdj@$ayrDnoE0DE<1sNIlI(M^tN z`A)**c|TFh_?*HrbLqnIX(v+~5J5I3Vo9s?wG%5TCQ-`)TA!=O)$hsi& z9f!(g>TqQCs#*;+vI$xbXa1N8I2-ZL<)*41RGbz2i%WQ6ID(shh{HU1PNK$AmR{X6 zE;;hziuSwCr?G8|1{wy{MDJtbsM=!hC$^YYh{5U00k!Sg86x8t`C|kE|io0k53!mE;AhthF zNh*Q@cG#~{we3jx#la@125=&lAVjm6qg{N64U-M9KfiImYFn%7Bi#Fh=r6` zQ4O5=Ya3(0u`@B%u}2>;yB(P`Ibf)Sgg@Rja}&{-1N4x%X_^!>ohg2khiqlSo1xnM zNa=BOi^TIFVZ$<|2uKEQc4EL+b?M@rd`1Jd1BS=*j@b?5i1|vzG<6?;t<(sn+3Usu zOPf%VkEXU$HfqLjM{0K0QOa z@<;?ah>t!R(=y=IuP%O@*2uW4Btv!6|27_2a0T~kT>LNt4t`ROADcc0GL6KErMZoA zDf>=9K$EC+*?!Gc)@k}MM*sJ|sWaxZ&x>mEq^Z|s^p?Sf!K+QbqemPn1Y!*`sviknz_$yKZh>KCN8rlQXQ)u|m=9`WfzP|& zivOoSm7-glTS3Cad^Mw9Aa&(!+67}@k5ut`&9}H}U4P)d#k^{kD>!ypX1I+190(r9 zEr6bHOXT)fX>ZqHZYuN>zVBYZnP2Sc@g*>5Atk#NeAN@{7w6vISf1N(K>2l{(sVNO zgTO(Hi4#aXRahd+vv^%Y_n6t7iOLXC*xQzQ z62VHb-7)3F5tUXZ_<@huML*j&lP*Td1J;F8D3le62t*jd(%#gFVY8&EF|wD+x~+rU zmG#@O-{^90_~Iy=uUKK%NOirQ1-(pU7l$z0QUPa6suHQOC~G2SJ^-eNxR394RPOZ} zOFFTP{2oB#N&tkPKhs4?07@@>&Ke}2J1a`EW0bK-NAZ`%PsZ(D1 zGfIjEZk6wiCM81rl>NBMNtMvj)Li^Vl~9N9X%gz{%iI!49DnU&nOCZ!zLcjI*<7M@ z*qK?+V=`s-3eB1?m2%}#`=M=kVH?NAGIMl&>b>Gp<~#AvneAd`{OJ9a)`EKeEZg+p z${-R6xZYuUJX11SaEIKSnw!$*Bw3LlxUhSSZcP=;kny%E*6$5>G*}iEx%^{*^0{xB z<%=dZeAt`gF1Z4=@mVCMSqy%@LANAVOmIBwl`r5V5>*3XM#9ppNEjhQPzf72tWbAI z$*~FY&12$Dg1BfP{G67I5tFqf$#h*-6VUKDJ^7v483jciX9^xQL0p2%ehQUll3qP& zoxYX~_7`{UKHk3by1V(aOCD9N_3UBb!x2{U|&IKBQ?968#FKt zS$t+^7ka>Ss1a2t$p3gwoRLa4))s;dnqTz@Jz{Xkm^fm<1{zkCty<=!zQL>s_3E_# z4h=72cegb&+L)!O=@~aO7-P)PZ;{5g=7OGicVhF$lFIE3G{usBvm=6+ugzCH%rn_# zT3qQ#CChmK<->EpsNOx-iY;yi(Tj6U<+(JXw{;2s_~Qusg8@xQXNP8Q2-rb{#jptC zkA^m0u-RteCQZR&Nm3C5b|1uDvkrFY_kuaK*RZfXyAw8xUPYKpGC;}40Avdzo#7rt z1}NhLI-0q!7y9mWGj2aczwzMji+YPM(IJbYG$Gj4S8u{sHZhz4^yg@$uG3+ z_vU(o<+;rH|EM9!=-VGW`8B<=FO|7QK0UtbkLsy*ykGdwXK$VIc}>^hzqN}F5w3bH zt-GpNW#7J+qFzV23jW}rm@!T(-@CobGn6hPPGiSU$Z#YmNd92MBAKrhtZ*rncdd3E z9;c4IJ+?gJyp98}8)Q0y%bPKFI0@6^rhz1fWQ~2k2m}M-TeZe*0apTIry|TImJ>l* zCo`%|6*3TN+p#f1gpfY1)V7J*y%pyR@bym%-SWhxS6 zE$(Nt!g5c{IA}S(_UdB(9QqA%mCCr6JuG7M+&U?seg*areMR&x-IK^vf8LP5wGM$@ z9kFq`0z>BtO^x~ll_4chizav9|aZk)hfw#GRxk(Coa<(l_x^h&akoA&jX zSmTR8qOJC9Nj#7JW13uA>k^EW4dCTTyrY@B=6gMO1rU4l@N+F^dk)jO?f-6hEf^P^ z@xUCR!}hLFU+BpC`OiY*=PJt+)!2XaENK`I6>mPeV-XG&?7F8sqSCNTAnPNQIUoxA zZNUDb{3QiW@g*KukvZDei}_?}7#U0TZGXDGuljG|-LmGX&4A(oidxgh03lq&mBX{z z=m>L;Ac1JH~p13iFWmhk&{jsGjWnih}(K-eg9~ZTbQ{n zJ8PHSodFDa?l-0IZ1^+RLSP>3hN9Tqr~YV77w^rgR_G+IkRV%GlYKO#)Ez`I83A)S zqgGl$Q4uq0934!(z~-DO&BD!wh*QX9dK!b-Qc+)936IBoYT1oy>$IRE&P)a?*|Ro6 zo<+ZBhQNt6^k~q(_6Zs#6Xv@TTRlt4gfd{ywO_2dG((7BT@wVQ+A}^;4{VWs{6RVL z$0jMQGcg;&LFVBSK%VC_Xr& zfhZn=7pyf_wsVHA0cqBw*v-%Nn}U*wEzg-qAP4%Lmcu%)wkFIvD5}amxBmWQ2=)wD z*WUx40nLd@ou$+qW>CD^s(REbhZ=N~D2x{hQR)93jU2jT#hwZHE0rQA-eEc>_va8&EU((M7tSb@-aWg4T3J~U+6B^ynq<|hE7`@K*(NOt zRcscRjm3}hJTf6+9Ak;Dhgko;gBkWsn1Ns6q1I|>F0um*` zNBiP)Z`n0Ry(=_+{Og)DZv}`otQ@@G92H5-NNzOuw|I%iBtB$R_Dl`u(@*-B6D#o~A`8@~%@gNE$u+bb96OJ+T<-Xj!%Q__ zX>7T7g`4d*f7SXMsW-W2A?fctv8ieDz-wfcV9KX*$Dprf%r~0Rlr6_UvYqA7*34BH z{C|H7ai{W{TW-_@KDgM4(wULat~|cjd3*Q|?K~~Q-y_=ilt`I+L<|ir?9nqMT}&5C z;OH^kWpK{$kQ!=K2b%KEik6qPY|DT>uJ&-@%w1k6QS-t!HzWkHd6;FL*fZ}h!sTS^ z*ztFMMV_g^p9q`Tx|o9t!$RzXMdq-1Mmq)(_{vJ00N=3hFr@$JX=dqA61 z&Vq<~VwE@#LUlki)_0RxA}Nx7i{D^Rm5*BK`{gvZ!fN-Wd$>wC`G1kRzz; z!)Fs#=I#q}K>Q(M^e4Iv>IZ4Fvmtp>_PA%Rd;ScWYtbvB&x**RQP*?Ej@`dba+ z?Zq6%J4WFJnJVd_cr{iPIKXQw0n)SBGo?8d9&*s*N|Ris#M8uqm}q#>uUPLUNyoXk zR!VV$H%+&3L1di0?8JRNqST+ktE12h>pjxTW|BpANedXviqlGEBaRl9B)Uw|^UjuX zNRMJgb8D=F&2p?+&om1^VcXSm6=lfIBx!%_f5C10Ac96&Xla~~Ak#|Sp*q8t9Vujj z+tz1CRSAPiNzn9f)L-<(+gM_aGs?-G0{BjhYIx%hJiK|->U;$O=bJ?$vwota?_L$Z za`O59`J84OD#5*|SSSK_vZa%FlhR$(1H)ies%M&rW9A6dvPd2OZucw;>Z-&=zz8MS zN7mvzwzdUKqrzYG1HO#L|Izhd;YQ+0!doBFQ>QZM6@YCpP;m@wcK^t&+=YdAVjjJ5 zdg#=E(4PA;qGKmpdq0(07PRZN+W$!FiQUIt>Teir$$Y9=K2m1+Pi2w6<7~6o65W|l zK%oF;f6yVg$ ztJTqd7l3$mvJS;WiKcc#6a{F;)Tal^qs{+ijTUDKwnj6smlrh`B_3-?^^d>RHN-_` z+PgzPuW;A6Pm&+UVTYckryY)^qRyVw(uUr%r}So(b;dCb@m2cMc5_=KKz;;UktVkU zefKa`s(Mw!VKX*Us$$2ljNXvgZP{r>f?rk#Z#>yW1+n8Fc8tK(?|0)8k9jdD%zq2B z-_cgoQgJL1_^1(xopyRW`YZO|-CyABtf#5QN;aCPbmjL|=MPHV)aTZ$!7p)N&Ffx2K1dfcPjp5kD8C%! z0T%pcxcOaqXV!H|OS>E7Hjm#Hd1$v~bT2?;#3^#Y&!bv4--@Y)(X)Wi^m++}%7bKs zz0kl2t}BoZ*pxx(gWjV(POZ}g|(?0l`yv&uTg=CAg9u74F&N48`R zDmvsU?$zY;9cS+}%*MyjkOIfxKB9FmcK*iVepOG)?Vr*-w@wVJJGTsM@7cHo0f|1K zcdo6QC7eoa)hf{OzH$!8dJ-9J6rI_~tCFp)>A8D3xn2)Dwn|J(djAq&|JvPNiVf#O zeO{YF@SxG=`~j}Rkd+gIgxmD2-c-LDXW25&_#D0Go1tTov>ONFx_QE5z#p-Lb-b@S zJqK3LJrwXZY9s5C`L~YX75%t>9;04s8v2FW7_cPad1p86a+at_#Urym7PAbPIUn}_ z&d^TC_AVgK84*iJSM|c=Ix;721;)9JWiz1HSYiY~v0ma04erKPj$ZlPvGj}4_6bg8 z+!u}{yYED04+*t*a!MD?QINWQ`krs&e4tA4AJY*Gn>PFm=XeT8eY-qyC9GNkZb|D* zyj9%p@7yfc4Y42JH1a$W`PRxigmnYl5^X(4WQ?}8If=0UT}>ZEwl^O=@@(hVxM#Hj zh^*bWwHxebokOl-;z!UE2*(qhAG&_`R_cV1-MsCFFRbVs)ly_Zwr+ux0<*8v+k+Fq zxQPln-Wy4f;F0n8o5rbP6+;+nco1x1?=DwEBZgbIj;;qWu-+r^45L-NEiJN*2gX;f z@t_}+0RQL`!RshMuIPa<#p0@#+rXm1*oeQG^TBKDUhabpC0~8TU+)~ zC~ROS;lS$`uCxxBwALrUvROFR?ly0>HpC@?Z!~UCpZcfr_JLHBpH-IA74bkHSs%b%YG%F#}NL~8>#eSa4R-V8OY zob!Qc(v!sy$>@Ux=L<>>z5XJVONC*GWNawd_?#%tPgKVC%`Xz!%^ z;6fCb+u*GM;Ck(uqAI+7jnf!!bY z=zji#nHZFTj3yu_VQ0Q^yz3MQMfEaPxf!1uw5VA)oB6<;?SEw&hjyz6`<8m55`yeU zZ&jm8B#9D1>T^2!yD2-(brVfhahEI2D!h8S9C0VF27D1Rbt}(3!AUufb#+aX`~BAv zxU-rEGwY%*pOJ|W?E*!c99b2fD;*SZm_Hqzuw$st5V4;V;LDZl>kw(cA>4AT*80wH zXFHZG%Qp$NJv_hUFE2f}YAWt=>u{TB8SQEQoza-~jCOO2-e&Njg7EQD!dsJ|L%Eu)IAd^e>Rw zXV==lYFbw23v7K&POR&YITzSz7S@2IU$+5x?_9YiJuI9?&{Tfo0hAi@;44#P@H@j9 z`kSYUo7U^;MLPicdZuy)E0PHRuc#+f7DrK>?pX0ZfZxn zQs8#nGd;X_*lE< zw6|QnYv+?(cXx%|jFpC^h3`=&)?p0+a5oGfJav5G84QFZ7v5B zwXg$F3>UM2!!4$V4`PpMK!KRUJ6;}`~-bG;db4cWX{UDoHnpW{x@>QX?fW{ z`1{P-1J^FEko_rJgNNzj&!U>V6=UcyxV0F00{D$0Td-Jnf{w&gQD3Y8`qnX2pwdrG zPiRXkqpzR6Dd>JiK(Grakm5u|ji8g%VN%Xf}Xhm}4<_qOa)*0-1tAnAbTHTykBDE>Jp>scRb@G&2 z&|jZgF?*o-r%23p*$(D3-tv+lHSfBPTF!mBC*?3gUg5Lsp;&jFjbiIKFD4c7{-?;^ z@V5<&_HDyW;`HG0If}Q~cb}BQ0QW+4$li$}yoLGUd*+3$?u9k;ZB*Weah@n|iGVbj z3gqw;&xKRSvXRqLXy_Y(?w!syYx7C1ulV2Ru*FoZj6GtxTKA5^)W=o9!f?LEB>=)+d(=S z?OJ8^%yykVrb|rfJ-r6Es;S=Y;k(j^@!_4^+=(Z=shN|Pf1#!09td70-qY5vUI?mG z*u@Ekj$mz;bn!tSK$u|rjE>}_u;8h>czEbfD5oMDedTG8j{#{{2Cp- z-;45Om4)q?jp=3;2W12{Z)rkepTWAzW+M9>l+i2A31yFGywE?~`tMSd+0KBFy>S4s z5Lq9OS;CB<@*i>21V`fi9tH=fL$N$V{?`nm)U7curR$%1yi|B>ZYj^xRHKm9=&0sM zHu+f0{k;VQeqpfg(#`~vLuYPjXHt?{ILob?i=AAacADFwms!q?*yxGQ&q=!oxgHpd zjM$u^$6cE3i#^YXBnRpMFDnULW7>ODhwYna9)_SWkSYd)e|SO%zTYn!h(XiU{oSX$ zzt6**1~#(At1Oud&ZA^1bd%9~1m9~aS)uM|A*9-1i?$Jv0^QfHZ z#Caep#5YFZ`**+5RUxtHlhT?3^(XhK^1_7$OYJ1|f*wk`m*#^7p^r}$poe3h_h+*A zbM%+|;6b5>==bY^_orK*%bx!s&t9PC-k>~PAa}g&cf6$eJVg7vZj-(5KHXNmOZD^$ zgKrdkg344-Ze70+3t3p8y}3p(_Lp6wOSG-~xEtg>BBUL#_&h`JaDPSGUHJF*Cefsh z_4UnJw&U&Y+~*~kA@W7v=k@g8+rxf`<_?eTj`|C7e20(jRZ)?KOIil}&!5?o4*x8M zF#xAeDkAak>G?z1$Cif#G@sMH2+>8mM_$YOn@;LbMrQ6w#|6);_E+Cw$*27(kZ=9L zw@2^)El_^wFr^G2r7Xx4crx@IsO2Fw7B3%p8t)Fu@-)nY7z*8&A*abY)*r7T4>^Wq zlPFyn6Ue+Kc&oaeCNf7Z-gQXET|)X3b&|B7cW}x^{$$;U zp>J%z!(e?VxiyEOt$4_e(ef@{D39nR7dPPWbAd+eKB#K(#vb>)b3sfA@7zyErSc}X z>7~SuxzwkTz72um(X}kjJyrX;a5_cH+WHdtw-9aDl9n~C0YqjJ_W=4~@(gcCaLxEVbUAnlDj69LE%Lq8n zWe`Tu_;&XjG>YHTL1a{p8BVc+ZdqNlYG~AtN7Nola(}A}+YD>t2hYOn|EMEtIh5_D ze;&jCwrk5!M@GfDb)3z&Y#pz%7RP54ZHCS{BsAaVaBReAe6Fd>lz0 zu;@J_?Z(;Mr0p`H>FS|4tFCj#ZiV0aG7|E=r-wDzRRJ4Ko^L-6x+yj(=ISBaI2SXk zq(AfL2hOrl*eV4%f|P?DWDZXWIa+Mx{W)j`Frz;aKM9Qm*`}w+RFD?gWt?ZYeEOwJ z>3%hIfpr_x=9mJv%C^=vnBVQKRJ_DT%;S=W*tbjgCE&2dZQyZ_H3kqz#!Xn{Gr&KQ z`XxKd-DOvlm;JD1pC{oecc5q#y=9EOO)xoU>I+=jZ=E+ilsHF=9&L4spr$FlqcM0} z;i?KZM_hNGN|n6(Hyouj<=;yyGe5K(k-?Cc7_TJ{zKPNsIb$u4=%JR8{7fIM(K89? zUeQ-wCFMa|{jlKXn&9fc4pMu$*7O|wN{K@ z!SHn77A%?ljWwz+gHm&YuF0$%Sl-rRYx~pKZ+}GB<0lA;OBMkA5bP>ral5s^8AXc2D#hk7cF)Tlu-u!ta(6k~hVU?iN!H?L0ypE|xGeAJ11#hSc&DV!bj)qGYbb&vjbM@d?%^Qm{*VO>6dHnmv_7%uoq;xLMoUhQgdFsw+5p;7J zPb5X$^bwR6QRg>m;PadhRQeW@Gi9qxj=Dzz6#RK}8?Ss=6oy7XiE>gvQ$qm@wQJHh z=CUixdo;1_ntj|#fyP{K{P>6I4$K^Aa?$y*?HIg7Gr&d`G*1G^?#49q7r3-uzl%ch zgZ6X6q6EiO51X~7$BJ@`AYHhGXk})?5VLumkAJ0{5E<%jHaRgshG;iC6j~w8o;;p@ z3v0DIMRlz8OUk3eCn*wXdo-`zB7twxxZu%;adwUX*IJf19sFhk1 zDv|MN3?8F`U9O-?JpnNN@~x-5hOcvdM!H3Ck@Qy;@&=+3Xu=D>2;(43=LP>mR*c2b z3X9AbKpksep)ylt9V+jk+02thMDub_rt|4`1VYi&=gL$&A?T3%${YT6yJ=bRVQ;yTaTrDmBsH4n_8RxXggPDdh#9LaG8Vi zoS@J*X50A-YRQ)wgj#!o$<&CbeiWMJ-|`n~w+Gj<=w{Ek+y^Xu9d zQJ%2AfT)wG06Rlu1R?s5%b3iL(Vyy0h86ulq$Tyb!lK#RwEPTtA~Tu4B#I8qBboQ& zv|O5|mY~6PjYZ|+T3I~y_pSxVDLs-T6d8`hrE_2}SA2?i#!rxIW4HS6rk6iX z#C_^i>#FVkWYi&Db*A^-@|M@6*k1-_s_>+OG;86TMSeJDO!&Y*BKE~Z7`JgccaJn* zH`9k>&Zr#(5pgJ7+VoXT@t=MEaC``n1h{=m?1XoOKFtI^1AOn$JFk;;3`XtPmPBAK zLcdZUSw|8^U$KjJBywH;2c#TzG4?I7&nNm3ti2(N`b^MWk|yI)@4B+ zv%H(5Sen!U|B*nkTpmSr>Zjw{7;8U+tow33KW^SsIthkIuTh@ze?E|L%@U3`8`?M^*<@qzXBv$kN>m(7ce%h#%b<;2R%159GjpA5obX7dD; zOA!Ot2_U62o~Od%TJ4l1;oWmf+a;y0Ou(Ig$9q)9rFoz-|CP41?I$KCH|=nkvyag$ z$`qoe?&=LDbR4jHY4N}^Oqk$-9cz5pCXq?nb7e?}-`$)6sb3oq3K+qXXBSvsN9<3A z2FD0;?OO7egB1t&W){Q}eYzH!4kVAMVhtA@tMN#-^1rHL+i)|a#wQxt)myXAL1vGL zp8P+?buPyj8h3m2D4|AwJ>%Td%Y(%MqGmO}86tZXVbqEy1iyIcbj2vNGAL>L$}dFE zDZ@Wd*7OXYtN=Q@{jth{NW`%~e*I2=_FGcF66!IRsXyv8Js{Eh$SXL~-AlHr%{RBX zsBsY78=T{}IC2{{gTN}F-{P{?gh~IYzSJ}Snp7rWp15_1UYs~ek2@pOkd;Yvtd?BLGGJ1sNHcJO(n{rdQ2l6LUx(w)GyOB;YqgCO?7CkhOk z@vzb0enDvq|0{{|O5&m*mN3zYL5pdwAQwMAYCr_+yxtlqXQqo?oZ#N7}%;E2>4>u-nTl843Y-V`?RlgQWGgP4_a=8L$4MS++Oh2jQ-{EvrzsBqzv z{h(IhP3QQiwghi}>uprd6=xK$jqqC*fR{COZ@8^xIu}35#_Hlv6pU`ou`M|EgR|1Q zX@B_`y8k`cElb#P!D(1CO+Mdgx6H{_1~Y{$4w&{e-^pI9qQ_x~p<>>k2>1lU zp*~vu0qBuw_@*?RgG2<)DfUXY97;O*_jp;7yE#!tcW20TH9+7}|5(q(@Ww`*_0?4N zmML27AmL6`mM>5sJYz-`{Gk~&5a@Q52!5~gV-sLAu} z5To|$!A02mx6=ar1H-~~O(SyFa`$bk6)QMP-Q>Dc4`0-{PtP-GFxaX6-!uT$w>tE= z0QUM&S;Dc3j=wzv@Ts*n^D4FwGZTdPp*6f#vbbw-cPF^(;_mM5{@>rdxw^W#s(Uj%Gqu${Jv+P8-Ou;)eNxoP54Ntg zSET5j_et4WS=?s$C*Z~gW7g`z{dH%?76)Rp?i`?52ymhr6_sspNRRpbQdhK@-(~dt zr%PYhOdhSk=XHJ<>nUQ_yOFti1tHsON*Hgg-uXBXG35S=1hwp1JRpaIgxPXxD>Y#_ zRrB~#cmfEa)x2|CG##?-VJi`EVL01AQSe>-YXl9oZf*nay~O9LaMR16x&|!sGIpr! zX?=4m+z(&Ql?SYkvk46y)I=@ysJ2~7;&iv-x80?C$t{j4gZ8CEoP=9T&d*Bo z@$TK$joH+sMbOuxE&7j2mfjHN9lStR9RNcjazh6e){j1NlxEzCD&6P0Wh(dUZxL~dj(%FXR^zY5r!Dd`c4O;Bcmq| zEGe$IUnjRR5<}Q=G4W_4nsXwA}HMz-(tSu-eRGJL1zRR zlK#Beijv+niPG`jFhK=jSQpFdJd8+?3_T|{niH8i-mEWWnl(SpS%Y)I9#@fFwiHp= zAX{Zq9y?k;f@^w(9r4LpspOlIM*oOz+9kP~rc41MO#zITn?*hV3MP?rbo#p^YMFg) zoqc||!C8OfR={1GR$}rrwZN@UtxxMiY4G6_d)}cb;MT$#S&o0eems%@Mum6V&!IN zs&0S=;*hRt$8;%-F(YkA^)*tHL;m)k@F;xo!Q}Id0Z7=-gt}}XmcA#W8(_Dt5@LY< z6U1z7UTI;x%)7&deYEO=sYkj)iTNN1C4A6YUsxI7ahBh$dy*JQ|Yk_vA5(GL0)oJ{w=OE9arV+L7W zj}ljE*hFk00iCzyOL6j5NPx%)mnU&bYBh1oP;V21l3| zbjijs!M){=IUF5#ifb*0AR>-{prpQXRvQyTB_!1D z!TRs3evIT_<~wbl$dBtkX3xGVgZRmzmW1FXo9oDQr@+;UVPB zcKvv`(!YEdL+JHS{cEr=o>Ehjdtnu1jr}@ub6|GmtOemTR&&icHWsf6Qh|dxoYbr^ z{{}SiYeVgmU+YkiH$D#OCsGS#jrivs&aRvi2jx$J7`6;@mQ{!*YGzYA5IclV8sy5l zahJm1vs0$AB%fRh-Wb@(Fh>cDUitF3h0pVcf8aFomQSTpK%Vdsonk)l@l$mc!R&cE zc2F}YZ(1uhc|_0J6Mq`-q|Jm8Br>S;)Ird@{`M#tDYW{ifb1qO&*8c4KBLuSudhQ6H2LtLtB?Fk7hBG{M5SQ zpA7xU&brDX2e4RP_<1e^yW<}M@8%XTX8R6H`bi&vQJE{vYOlb9RVVs}+`%xZdTqX- zT!i(=Vtx9%1!5#&eR9_IXA*8&Mvn6}oWfO3;EQ6u#{mo{t@zZe({*+tI8NYEtixf!M4I0S(hEFICYoD8o;y6}%uxiX+_7>%H#~Qqa=lT^IwfMl3k2 z4hSFD4jWi^QXIpQn!@?2zhn`+v<-D!5n5K+<9yWk&Bl8&wd5a>->SURz0`Sd?q=m* z1*NT1rcYBH{TUxL=1$!@#uN@Y%;7V}S zMq}~C5N4^&F{6syJ>Ao8g!os{x%cyl=;$^qUt@|DRozPzp9->(^F|F<<*)H`o_mAJ z;yJqDsdnG%38giVcZMh*$orJH0=kwa)F*yKLUJLBR!Q+_%ewZO0d>!5Gzt%_!Nf_pc2DPN{Gr`n)d(uF6LtO?DAS%$;mnw|Bf-A?gwYPD#oXHPTN_jIu+;ADnG6q&{0Iarqr z4^*o*!2C#nGyl)9hUL#cJes8@QkPp?>NOsIZL1K(Ty94(ObUk3)~h^Xub)Jcd1&>) zY0}9PZB}qgR^lv^(;k=eU=aUKVtc~`q1ZTI*(y?ff>DOFscO-`vuXC^U7UinW}C2H zfY@0!8ZQpT&;4ItW@1qjz$d^QR&RScw*uh+RtCX5Y3m*r>-Vf&+0YE>0*~$v>vOJM z|lS2{ZF+bK#U&9`nF(ZdrFCiDG^5->G0^q;&`zG$0CK zySn$bg2M$S#WH1;y)^g=l7=WC|30UnLE#v}GSiw*MQ9l0s)9g1FiYpmGWTHzyR?6b z>-1C@8X;gHRJ{o&k5c#ICilU8tyC*d+cIgOU|g*ml1Wu;$^nEwfdz!n%T;^CLrAG`N1z2c_;cfk2OZx=+S? zl9*C3+cDIxmWHLOCN#8-mR0zIjVH6;5~lnIm>jXgYIgG5jynPq4i5Ql&uh1$<9k|YmdiwR@GRADjf89F7}#euKgXyJSJv(D#HWcxhzAw?3Tr-l zfVD3%KSTm3Z2iTsI-+Tr%ttAEP{%f#EycoeHYW5TdbU|TG6c-P)9ZNoR%QYKwbyr$ zD9^?hboxFWE>e6HJP=a!>K=WQi6r`_%BT$!t`v!5YEXh1pTjdp*gD4u_gS?S=R9-He@C5nWZq&#f_jjY!f|EPck=)&61;N(zljf{&rXGWWI3TUprR?yGwJYr6Ag%_{(h zl6e4rKZ~Vrm_z+`lzozVGAkN_kjn6i^B1&ZP7CE)aM{YrY5l*fU@s`By7V{Pvk2{h zwW_IiWenz*+!)*>lD=|1@!_Xe0T`YhP6{XuW@wHVyOZsDacbKOxCM*&GeI) z(X8p8xd`67Cov{QGehRG@+ocO<`q!UwefBi`NxOlYptya*1*d6$lx@-fRPE+$ePpe1eNiYv0&Z$l{Jt0 zha`_uaIOTM$7x^a=P9V@R;i#Hx7*lJ3$})wt6EVi!F4adD@W)f6Y$=bA{X zY?@cQ(6;i4J;|7F`$}q8l=j3$8RvWf65*PldX7oD#v+CypK!rQi=2>Fg ziq|!$!C%c9cp@9Ux-L*>|j=yhfR*0fephlSzfhR`#Lz}=P2%ILP?%hbOk?Nrc>*QvPe z{4!RRA2adv$U}`Bs&H)OR;d`0=r+6V{j|Yvh=@*_jJ7|AT=i(~ZS!w!SNXNR9W~^h z>~*hChod0gwViSYe!^CTj0^vULRRG9Pvj{2u@s&T-#h}|d{EmzV)PWC{0I|UG|Xaj zUXO^MlB8;@xh|MSdZR@kZZGz~W&acy{C3Ui61$;?>vI3h2+E|ao}$A4c2ul1+LRon z{rP>l?NmaWUlG1AX`@_;TM1s?*}zD=v{n1|F@LJ%i#z!gWC0n0offpT9VXvhSZXy9 zv=2Y?>m%x>!o5dXA^q<9@5`B@!gz%n03QG8Z+KGor%?s0%JGLIkZW^_-xm0fuzwjf(1U8^OBhYN-r}%c7*Bmh3(%%O5)8? zCc7&^_(R_Vk5$MvpHBOJR4t^ahZOsKI%SY5zxD=%xmjQw@7v1m4Y>y~&WcP{Ofws?*d#1~!L+>EtXgB_eOHWjwdTC$=v zqB2gPaZp!^L*A2nPs1!aB0q7N~)|kB4i^tvXOscjCoAl zr<&30PE&p5*MKf6%^3{<$yu+dXl(9ODFByMt{4?3t#11Ra;sJs=6|rfJQfM}?x$Ke zrB!W$gW3+uS|%(D^1fd`7DjRMzuR$~eWmRlJ6N2CIc;0P+BKpWwMi!kd&4Xvvdv8HcBJGV4{$TjU$}qZ`gU<}y;@Jnc`m^IAzkYt62v@`d z%_cdpi`^dx9mI!qyD<9^wKch`w(v;tTm;oZfY)|Mr@d9eZ;hFD$YX8!3hEDk2c~e} zLGV6H3JC95F3Rx@HNb6`b@Rubn$l)Otz6gW9sG+Qohyh17_Wr`Ps>L1nT-Cc^tM4- z6Um}48yB<2FBsFcYS5T36bG=#o&Oj=zugr@je31&PTPjV)scOGGDu;ZJq37*UNm;f zTAz6_8bFnw0k?`mD%#P_J(<0>|IpYgi~((W=7HYX{7qoxvWbc}dZ->V!Nu`4<6D#_ z$C?YSIF6rPAk(~nwEzWhkO&G3vsY@XqK%;qB1|{oJ%(cOSJL59jf|_Svig~8HD*-sf zo(#qJk^9vgZxxqDc86M9gX^J9J<(oG%|KMEdpywRfAuO~=hx{6W*-0yL5hB6cL`B0 zEk!28i#*-7Pltqp_aiLLed~0a2Bu+ZW|W(W0uw>D1q!Mr^B^Of*Ufwh-nx5bq|~j6 z82U`$n)-zeLrFa}i&9w=>S6w`_DvL?t9$>IrpLRHm1z8158Tx?d4`Q%QT@z<-ZGj& zg-q7UISb=F1zq>Q%26g3MXx#?gB{+d3CXO?i=X--%o0ZJ#Lmh=7yZP-yYklPeHYYe zmhQNSXrS*iX($alI_CF76PiKd(qj`H%=>hBSGc3fo+Fur3z`f(5Ps2W;&VED=kLsJ zoT`lhg|J}R@dAedSIj?VXL56%W@m4e)k`udh-&Qz{&?W!;c~h-v11ftRj}DCLk`MV2hGbdOOE@;kJ{0bNhE56 zSggzo$z?loFg;?k11_|r7nGNXaLtXQUdT`^DHS89WD|2&qfbA90XSnZ?R72~O3AEx zj>u+1>sw#@XSrr=_4ass5C&IXP_XT}!@KCsU9GEDLJdWiX*66^+x+2%X16a3N!tvk8L$($jVU6JB=5B{Yc zQwPi?+jD8s1-OZI-xIGBq~3n=bH#xdsc?Sm#Yb^fSLXd*35U}019jdSNm%nCe zEN_sU(+SGh?OS^|M}`Rs*XWxeqUgWvTJ~({245)8MTl9^O9G|n(fu|<&2Ck2`uDkX z57V{zlD2w5+CqJ%IQ**`gnd6qkcu@Lb5r5S&T$7e{Xt^S5mWF&ljbq^Mzbk@xpxiU zIkC7$-dGoAIF{BKlmV%mmg2z`4EB(?_0y3BCeU?&#KxlW)p0&5%5kt;{_o+O<`|(7 zu0f5PC)g{oB{a*cB*3uXM zSWqX8kM&OR=LqMm1OgnPh^Pa$dDTUA6;{YF=85?rb@2cr=sQnxXI&@8Q>$tiuNL?c z+YC$ww7|F}#b5(-MBdkW$Z*-s$Mc~)N6Vh9Ksgp^7_51tG{2>MxYeB{ykbFrJ6l44 zOkOoV^3pgI4atEw$L`6#l@BCoCZ@G zvon2TsV5eO`*dY%6}e}8jBmD>Hl2Tb&BlHTc~-4cnBF6b`E+XXuAG$CAJp~Nd3}7E zNTJU<^~6@QYjl*LPp4OSCyR@x3{q9)@uLZQTxOxzg}ER5Y`VxAJ79p} z#**`=o&|AhO{Q5+!i%IW9o~H@SB#iBV2_3! zP&%wc^3*0@Ooylc?CA9auw>pbo1JP?VC~&Jxa3SHy^sW7c9YOvuu7B4BWJTy{LEtG zoK@TOIauL9`CEtyw%7p^?uroif=I@q(v%iUL|G?mZ4=H4Z$^rxp1^EU%a?Hudry9j zH@rr#Bo+sk&s!puuC{uLDZF*>tzKR7Ec}4zvScyt69txz$a&uH6!?S;GuP8YZqB(O zCQ15BkFhDY%`!~!M#=%%D(D2Z5Le3L>|oppEtr z&NDx?nk=h9M*Y2xrN!+>R5UHtlz1JHSuGh9RuNxC$zga(e;d8_MAaj2t1ngD(}Ry} zM^CF|cv^q+uuEm|e@3^l)l*W z6;kX(TCvU6Ui=W^a~65vqjHKKrT`HNkn;}2gMv`tdf;#q<-&<0oBe3i66WX9JW&*W z)do$qX+5(lH;SqcMS`X}htlZ{g%_;e6KHOsEoSPQH7&Ueg9*~Xv{dad)uOwct+>V- zL7Mo#$o}*)*?ZzE2W0GrgD$;$S@$D+=_X0cXO-g4YeD4zsnIx*8?-f9!eo9Cckpp)W)qh>%hCh8*&No9&zIQ&RjW@~`Rh z`^&Uz1GWMrDYm)%8}s9MAmy_iWZV-9{%hoJh>(oCJtyAQ@W)2dDBd_0ARoZyW`^IT zfGu-AR3v00H>iyOoc|l2&AJi3Io2G?@~5f{w~b8LJOQ8Qh{8KTivd74xc@b^o+ZiX z23jjm8W&T)MO75XlaUjyR}}kSn2L#~NYqUM;Xe7`)91zsH38#dYa*1MPja@SID#qP z|9mesRd=4#uW$12w#WBbHqZC^!h}DV&5jEXJ_#pS>y@g@yUrRU7^u~N1#uRSLe~892#tBFP!?K;FHCj*VoeH}A{1D9C@!{P(rI(#P&Y zyMxc;5~&f%3bu0VxN{AIe4#EfQ))27d8DJJUwlCxP1S9S)WUT2Bh zPq-Ot*x4k8R=>KYOJgSeB#54Kd7&bEa{eLP6wMr`=gC#1ND|F_61^gaj<7JsF9sgb z((;NtrS~BtjBlRObbWvBerCMC0SCh# zgf5*384M4J1$xaP-VVp;;Cw!Fc9_w5S+B;tLPsXW^%oodY8yl{i`0Vk8*P7+KS+Yp zWPIJgjF>5o4}PAug8A3|pRVAPe!?>z1!zOc?lt=Sd~kyF3%akjf-IayVw~Wih_-X0 z4T8xFH-}EJ{$Z)whmxYSrk6d0gQP94e%@R@Mk*Z-EpeK1Px;nAXQ*IJ5iYKl*L{r; z!8v{86(l;lNg(9q;+XOYn1lYlqLV$==HH3>qww4bx*cl zI*|xpbG-)#&o}$ofcEDVZ5n1F-UW}yJetM2)L+^z`X=@Q${_^5ojHdtssp?SXZ)-C z14zZ+GNjFJlT@N1w!4WZ?)4sZB%c*cASrVZJAw_(0wV-+q;~ zILT(aX?@*aHbNPFu)9zj`zahn!g)gDn$aDL3d}(=bA0rJ@3yB1$#(HPJ1ZMA75lz$ zD=nK1Z|1l!RNFf%y$041o;>v6TkvOrEbXdZlyh2S3pcWU0r`V!vUxLj%Ye9#T}n5* z@rz$|+i8aeB#^raU~v)hLhxOD`-l6z`q5r9GCa4eVVJP|#cW{YH8ZknriMcMo&R+~ zmZvRO86XIxMeXE*{~zG}ZoK4Ǒ=H=6uGM>Xvl&Q}RM7rRbY0n>~^JJJV%Y##*X zH9h;Zz9asn{I$Tma_ec^T=w<ZU;zhUh`^(0>;t1^A44F%|I=y2^~kJjCUkA@nrZJ50)B% zIDXYLFD~%oo*Eeu+i`SrXULAsK|DqFjLJOlRY2TT;bZq#f-56qm@DjACf`x|d|*J8 zVwrQC&xlKEKZ@&I#9-@>*91h~%zi+kQk}=4&!c5|rR!S}eA-!D#}mYRHux%dd6nlY zjTx~HxP}bt-d9+)^E*^bA(kDPw|~Z0&3Bh?GSe)wRg~+~OH!kN$B=^!6Q}SA zvdjaGg7C&f28#=7MN#3CsYDj=fRoMAPS!b{J@Rny}%Q!u6(uhjx;g`17h5MMlp)FYyG*_FXf{F{4$HDzKUclWv zsZHP7gxZj;G;}U2{xvz3&DhnCT=arTlm##?Umm;h8 zBZ1N<_R7)F+j|4zT-m}ngbN6R=P1{_SrjSHI5O;lg|LY4n2e(d#BLl3xV3zX88gKh z*VhdD?3^b=QCw^D+fj=d`A#Nn=tfi6rE zmx0qP0+d+*4uxKECwHH>dqZ0m-AtqQ!{h-g=i3*Yvrr1+vB=lQYA?+0%@ zE5<^A^sHrXjx5W`3N`oL#W2HRHf|xQh?VsxtMWnjkQ1+x zXNOoB6*x3dYfRa{bk(ncNm{XbWs>5Hg!0QCcY__A{H<4J2ajUeUl<7|aUdkD@h{~r zI`k@QloRgj`(4c6sGbMRs||DI8GJFaf4d@DvB)RvMAhF6G6FW)(_ z!Hli9!#Y-p0$|H3s{QTJb@zW877PXLOOIxFgM_&S`n_g`m+jgYR1;EkL;jX%wcHW! z86VfxzDVcxbX;kM8R3C|Y105cBHTNq{eNDe!b+1jWrTEA@=6+dS0FL|HF~&t<=*z$?I;#+* zgm|MAs<&TM*92QHHLm`*Dqa632vndHSsA#+6e>Y@)HR@M`Yo=^>lwBzTl`!0D34@) zaDBW{;@GMxO(Y*yn!7_)sSWm1TaL90+F10*++O2Nx>D7suVwj9o=#AUMMS69>K5+Cdj<@Kn*X1sY?`eL>u*ORs z;`t6x(72Z2aOhwP#uc}KQOM_PZpGC_2NCxU0Ya#@Ij=R)WUxdt9Dm0I6X1Y_F!&(R z#iZY!7(-zM7&AK{EKlw2$Om0;Gmu94nFMqTVQsC*$H zU%_qRfntd{1e+1#aQ4gNb#guX^wxN)%NX4FDQailMCi2IjuG_-r z3hYtm4t|7J$zufG)u@yf5hC7Il+oy-RU@v7TSmLeJ?aNl80|}{bLtG|*ECP&X4uyJ zScpJ{A%`Mskhz-v>9zT!lEJ3&)9rq)IN9(_`Rb-c;|!n=X%(Sq=EgiG`HOfJnOUVA zig*v@Ql6h102Zwy(z$XD}58k{$_s|36$Z0<01R0yPog!F_ z)PJihQO`KWKeGolJodK~u~=KZVIi^=H5q3kar#a)Iq!~HE>x%NcB3H}R(XVJE`bKR z+h~qII)_9DlEpT2m=6O-_4oLzwCWDtn*8(c@e#0mzV$<1!Zdte#b{<1j1qWRM?&6k z6^FSqw9D*p(TA7&zQ%XXNaXFFU)E2V%e*^YUA#(=QCE2!@qk7&wfeas-#I=sjw1xp z$N6pGqT^`cyK>Qil+%@)H~ojKUDDM7NseD$e~)uSseY4>XM(1o;+=D02F8*|9CLey z*9r(pI<$2vDpU4%kUvUFG6EfXcPb%K?BEn8;JSA>s(vu0LY?uUSd5reUbJT^d{OmB zCIBc8Y$Fv0r%0c2$i_K+ZkeRP?+cpz?!Xp*uoc2-QEW3B@_mitJ^B=_Hz=7m$y|IG zLU{bbi2z;i2$x|7gZb7h5eVp!pQ@6?RjFVE&+q?BFXi|x=)oYOvGUjkD zXW%ej4P-`J9~q?izGY2idFPPp2ViPe-C(sob?8ZfOY*`G3k+6?I;{a zTQx&rM}$_#ziPV=<(spLUc_gy-0<4Pt|tPg7i{rGa~TpU1)odcwm4P}y6D39Gfxt$ zXiwQK5(ws%S(kgvgS%4-2e7kJtVH~~+^IX*yeaS2y}Bu?1;moG+vFLdOar;`AUfzv zpN~*1;RmlGX!h4U1<8QF7wMf&iVRk$=YwC^SSwTh{Q5``w)gJv8J(sor10WP^Qtla zn4Mf4j5YXd9!~{ongYV?%T&B&p5QTyPaf*lMw@<21B3RfWmU@oA-w)B4xxRzSVHVv zPbdDP1nr9t&XS`c5#|}J+@hStEy}gsI^c;F#zu7y2a`~L>)#Ka%wM8*q=vPy%3~oD z2{yD_Z5lGDG%tj6?L&7m^9~a<`?Y^w_=%+Wvz*J*p9%|&<@mp=?3rQr=V+vUA)Ii= zJh|Uh8YGCT-n$95K5gMl{7Yb_TZG$tOI-w1rh$^lPU6D>4>aVtzuHgaPoiY0h`XrQ zNkA>StE%U`r%qu5Xie=_6o6@|pEt)7R1tguJ%so%fE9yh`%aAm3b`3d>~yjJ;S-_-)7mJO0HE%CUCyC3|OSGQ%vk>IyXMF7)RNsm~^(6OyVA(scC2`r)=k8FgG&sySUj!&&hV{D*xXk72V|w@cnnt=8Z*^2 zuyDRKSNtQQU=Uu9C^OCtBo=P%_@$ zpi`;np8udV?{JQgJbAZZ2G{p>0N7x zZGo4YS&lDOj&No?sOpbN7rMU4GGeZKpE@;4=Q_`f0E|FS**|3JJf zY;6B$65I-}lTwraAl^5OyU%#R|NxMuX?Q2*iyJe zo(>ao1h8p7Iezc25JLEo<6-nb&*vjq;I&_p@pfm~=k0Cd<53P=@S6SKZkEp-gO?$~ zm9GWTK!}{hQC~c^%q0<))P{lM9@Eo~C-=ZeP`3zfME7H-W6HHfe-pugC~W+1xo8SL z_}c_%x##%q-N)e?Pk?v$NBw@Dq#QNT^VTobcS=~Lh_3ke9{seJFGSJqe=@4(`MHwy zuh)Bvg4OBP1M;}&TC{*L$pRz;PLY_{kwskPErdqSC`tls{rXX&%=7KEKlDf@c^CF& zI}D%mH{%)#Nv2*Ch%_$(Hm;aW#SQOgyd{S;6JY9XbCe?!Ha_DNoPh z-!glP-_DdW{H7X^IrL%fu9hNH!pD!1o)a zlaTu22F)6-Ul?lRHFH%VqzyeGhIJYE$cfw3g+7khxZ%CkIcQ(<_N>Pp5su=8ZGLFw zJ00d17aHe_u&NEH?J=Dxc`=c*nP^rlk1H`l3T0VP8Zpip?^3^d!3?$hxQSd0VK5~R z>vMA0zUkL3MX3vVH~A9|_tYl%*k)I8L+MQkQaz;;Kt?DO!Fc1ecAe2IPP z)Vz=TZn?edqs_=mUi5P z0{qN~{^O2*r?-}xa^(A{XC^m84EdglH5G|Mtwk;Rn(9Gc$`@udw?V=Dr;tf;>Q)3VkOlfNgxE&b9 zimqcvZnpQF&>nU5ws)w0Arxd8rf?VN){OXFn<_o_W4yaJkH;VoZ)@0|Qea@4_KTa- z)n9Qpy6@|n;Q2II&6$`o>JXA{tRB^2S{rX*P$M3{9=K9j6}@^d}Yrq|KNB1Tw%#pzr-O= z3~b+^li?3p*1}9}S_beqe*+y|9q;~T4ZqmG?b>xY!gvB3gzg@faj6!V4BBM(d*oHj zw8;Vj^v0AiSk}l=SNDuFQ4{fN)}`uQ^hqnWiZ046*LUqcH0KPdSKk{v+zKSJRLuv$ zTSaQ&tg6mM1IBSuiq8)3v+~n;WF2W`NvuUOM!&EHSu}Wn<*$0WZNFg2+GMV0hI9B_ zN*)@iW0vtE3ip1eKN8>`BnE;l1iyL{#~;r6?*aAu{7xC6AN+>5k5gP!qrVb*gZ2%m z20f7a?Zl4gh9Pqrc)`@b#IWamNY{Syhkc&}JQcvK8ytsn0L0 z%Jv7s2>l|INZ>pvi5(SM*#ve8;|U&MWX8}?i@0>|lRF&j{VqBKEj~wYW+FkO{Zg_J z84{D-?+D%fBZA5?ja@eF(Xcq~K7i~^#Xe5Z9icw%9&BxoZJb=@tuLQKXpSso+p-UWyEKPsZSMtT$5q?k5iVhQb{42Kw@w8B1#L| z=V*PzT!qx-%|?6AoRF1j&?$r=3NDV)@{S7G2>w@{i^V;j3eMP+o9iJ%*hvylyP+E= z{2ukKyrGNU=Y%@lDAqhsT}CCD1aFRi1wiUy$=sv{W(^_B)t`j+$t91^I6gF{nfe(b zLI&frxkYG50h;vsg?jEFIDtYyi16Eb5SCi~7F}c&jz>fh*f?M37O1#Q&GR~+@;WA; zQqnyfcIBvRi#>4_Q!-`5DhrDHMs@X-@SJLEe=R#HxJ^042Y@T@Z>(Jkz~<+Z#~Ejf zHy#q@W}|5$mC8Fh0vCRD?A902|Lm9URyyjeP^=f+8E5w_KU~KCbS7-W^!nrFK8Fz7 z;DhX|WpflBp?7?QmO`-j;s=zG0TC}#&xF4ZMvgFY2k9$Yu=u0=&>6QylDncvwA>(m zIFOWkLX+V@y(dJ7K?Rj!Jm_`Mzqs75C@%G&Rr30b5J%eYmcnepad53|;zvK!Lm?7K zB2K6lGp^AI3UODv={xM2e)Ae(j6`C(B>0Tzs1BX9a(~TF{1I0oYKLjtGHU>rKf&+o zLm__f1Zw3h;Lp6`7fx^m;SJquhX=WqjoV^r^l72PxXlV)-(qppUhxRGgmer(LBld&1bjEzvh@zccVbH#YL6mOwQ5IOJyDc#+|z9zh$;-Ag zXO!C?D|>%7Wme}TJo;@=ZayrE;%%~YRBood$-=s-m!nj2Yn`RN>&>vBxe&;uQ(4ZS z*V}K%G>x2B6TD(wt=onlQK!rBu!4=4lMToN(80ad>o0%9`2&$Dgw;y?Y7JIjr1}UNH`R{dK5xf63F%q_fj<9TvP&@NC3$= z!Mj%$D8I9HIk&UHL{hf_K%o0psFz2PpV7m4BFPnKlQVv>i0hM=$5yBhw-416>h;xL z5ZV1vCTwu~{PW1S4_MBD-6snqUpYqn8oARDG`=AsJzY+20cuwJQ^D|kydB@(t2!2U zU>funl5|F3-G7a|5*yzH*#$wWaH3ALi-eA<_6ku2=Fos^?}CM8orsq{sifVVfG>_xkrpZN1MHq_kSIYS7!dN36|7A z)7~PisK=t8w;N9uG1-Ah?pslBEKCrtp ziYR_M!-G6g^daS=D;+1a|4sc4ZSc9wgBMKNJo zSWyfW02}-Ret-`9*J7zNrXDts%}3O3mJFVwmnHtshL%L<)w{N{#&2n(@uA2OPm19~ z{NMsY6HS+>!CxNCf9`Oo72PMkfxv&9y_E>+y2#EA^YS%T`zg|hqC$!&O*&~CW>^l+PRf#40q^q-)giIgGc)7iJ6ZuCk!4PohdKj8a>#CieVn-%*W)NN5umsG?65=ROgS_ue`*p6 zhU!8FFqNIF+Tg)WfZ&5foT$$6#C8|s|4mKHH={-l_4>&p^}j`u1JTM& z1s?Ya7&q-$VaWj2Ln+R}?ngIjTPqn%{`*o!iMBH`nv`zYV-~ioAv2C1`W@c{H=<3R zI_y|PFTZA{r^EKPv>2>s;;A^Y9|TwRV1dUk#>Vzm$4Z_IF7^#FDc@pT$=HeQNM-{T zeMtvl_wF0i)15ef*Kk;sk}Xwtz}Ug_dccV`#(rQxI0B5Qg92E;=sVTrkwG*R9O@tZ zN%lHJX$~ah06Imwju!y*b((UmTj2H99ZZXK0myU^>ZI_9 zl%d#RW2qXRRoS11dK{3%++2S08hF^0{w+N1Q?Fglso?hMZqaSfJRa@w>isq>U=g%? zm@xjJZE4GtJAKtU3)D9r?5(FEfFXPfj2x`q5FojcBaB9cdcJz>Y9owZsV^SPOW}TT zd#$i&i5h&JG20nv@?f_;smKK3g+g|ij+crjZUF4Zq|X^3)W`#`FQ38nPN6}(cwx@4 zWO%1rujLfOVf8DAD;6;_QH#9cfxOn@n%d&A9is|aSom59-sqC|PJ|qxuzbdH6Lmvf z!uR!~XDO!MIsU8D-@-YbdXTpz$TtqRW>y`*yQcJ5BR8J|z9(xDQp(=Ur^;C*dn4o;-#m>QGARepT%7>O?48a*+b_#3iY^aHDctFMH8rq)?QwxB$s z9mL2#VSMg{(kplSLshfU@1TQtRsS)Fe2Mwyvy7h7?D{w&lb5W5C&zi?E%n&00c2(Y z;mZ%qMLyH;S}({|>Q=s+?l*Qov*dc?qaN_?L`6<1oeEPPa0pI!OS^sj7%49y_;?W< z4NizPY~W2Aht3POqEDgKe(5UST1E!pRzgy^+MmJpRsn_rQvC=6oL2{35{_+t`Z((- zpk8;{MUG>>33{Fuc2^JDrnq{ZZfQ0RcJQ&x5WZxlww%^*u?bkm@y+yFRY^D`DST>{ zgtg;G7XRf9<)329d7s4O_sztVspre}6?6QfvKvC)n@ex++B2GO)BEt5^+2hPupCd?J})v`g^KB|uueG$oTwm2S4MX6)#6Z*TdBC$qSc*rR>mFZ<{ zxb4UGv8d0teIGc@ljE0VHb0{(~U%P zxw6@V51;W~5D4+w;dYns^8FNqo43A$;zvta zW*7LH>W`6-c{3v^QFq7T#X+}HDAv%|VqbT}=rmuj zD{z1U1|oH4eO6p!SQgIY?Ps}F6*q^|(ff*~4N&8z8D*I`>^s^v`F>V|I0>uzZurh3 z3om{L6$)DWZp|=6F9_PWx5O+)ebkLLZqmAcA>>g?K26>&6(C;>O22zWt+=z0tXzO!`b1r0T4m%lIV^lW;)IqRFBfz4y zA@Z?H>|;AH_!@iv_O-g#e&)M7I@IdKzQ-hujK*~7lPEL0Z=)u|aU6o0exo)ktbYkR zG~Ba#P8EA7+=(1qz zd~A0fb2gMS3M-x++n{d;_8~A9G_{Gi8(_MGO<$Ru?6e+Vb+SjUb5*$6r#?MR8DWw( zPjew>S=gaV9V>bO8L(%zm&lQ1Njy1HVOv>K@sBzomGkHlAX_(dxwM+2etKEqj=AC= zsZDbpFgTjf(!cxthp${&(P;d|YvY*hpL-T(Vle->tS?b6x?Sn_c>k9w5(6dtAEOpf0cY%1W5l`f<~E z*o(Oe3T

60T{U6Tlg?C=PMUPDdFTobF z;tSCfKc;(V5VmS6I8u@i=n8G#x(-R+ki}Rp5l&{;^+k5*oBo`EvZ_v{v29TwAI?U( z#s$pu`B@K}rclRDY;M_Ej&17IVP4pun@sm>=?`YD);K(u3SFzlN18>0euZY6A5$#tCaDZL$C8loX3`RSzCe|3xPtzoF+A zXt!TxW8{E{?h?oYqoA4MkQxljSI=V_S0*TTbwZtjEVet4m|9g>mQ5-C8|#%)TtDFZ z4+;$@B(Zgg-dwW+q(-Bx> zZz|nvhMYbcS(2lUWe4`=I!bCC%6PD;K35hsGL&c4dxJgZ@`$0+uD92SoJ6N+4!T!K zs+x8X8O)xv=wQINecKg^L=4FgWgjnyI?W=sXo)AC|4^?&WIjgX!XQj$`4J;ATL;n0 z)!wD8vE(ko4mVU-KOu|FzLTsv^$h92BQqt)D&DZ`G-Je#yuf{y4a#Q_b<~L@cOlRH zAI9D?Dvl?38%;ux-~?N2ad)=`5H#jN6D~TP9H*g=bAMN%5d`#;aiX zpPSCuIXA=9kGx!mD2kqL4-!6Oy<6OlEwW<)I#4Bw+MqOb^kxTHX_Ur(9(Tm_2pLEZ zgz-ic?QQlEWzD1S`HN9f=*hbLG8r%gKWt&11HfOW`;EYwNHtJj`>&}BF0;pW9Nk)N zN4JfOIIp!DOYxjDdf^t;m}2EkwsF(XMYQwfda4HuPWh(lgyCDVuGaQ*dE;_zxX~vm zk*Ks5Y#N^{34%8>EBmE>9?$_F)=7h8Dq_YY_OlxZ5D!5ObT(O&N3BoS3tRUczbT6y zw{Uj-iZjcLUMX6W<7au75szBi;(Vh?v9`Qcx7 zn+0xw_lI57UZ`DwEsnefyLNOuGAw&e!H8xC2WIpJXnO(q_)`mpw65bmKCzb}SnNLK zdg#O2o{T1ghR+ZZbyk8+a>u_eM_5_QEQVpe7HG}K4al)~aR$2R3VGRXyY51wzKV82 zJA=@Q!5j?1rmPA&<`B6n6Yt_FZtkzzJd^0d{?H3NycB|MP6|D*AS+HF!qjhz6arfE zxS{A&$-4p)t&u*>igICHh{rx~xqYjQQO*GO=fT$d7>B zFRs>E>5`dc`*bRqnJQ$$4jS-Cd&#^$^Aen6A*k3Q#JE(B_2Y@;y0;2*|rnb z>j|<_Y3IK*s^o4EE3$zSpJy!rYhHt=V?I|H+3Xb11T@eKg2QJk*J?AKIhM;Gbv6VqA8IY*# z%?+yrRLi!O%tR$^yEfL$Ko7+>YLY&C+Zs>4$|dE&A-92_g!{*gz!&*oKN<69=$W5t z%_Q}i!&kFROwbiHr=H2s^{vE$g?iUPcZPJQI#=`{&s9Sl;|R)8&*eb1XmEaBI&@$t zt5JV)3ctG=CIGar?!qRiQao&ro>0DrZjmX!(!d#8KWRk|?s<{@vN&@R`R}JaZW2Zp zYE^S^>xbLY0#3sT-biSOr?i4`z39w6g}wjYu)-6N1(T=AMbV)2r?yk!J9c2=%ItTs zx~HPFCyNyZwSCr3M;$zG0kf=onI1cp)73Os9QLyjxF_=CW#afNTekV{*IA_wO8aTq zrkRORSjU9lM?i#k8b0mOYzEMI-TGwGG@ONmA2`LkaFnQf<2;sCa^aomo)&HN=QO{3_vthJZsF%Do2jD55*seVvsSvA0(ZS> z2gPycPjfFUSTu9wLn1!1o4G}!$Sp?gvJhd@#QyhGS|oKs))LT<+R{r_1Dj^*e1Lj5 z@+mJY#U2N&U*$(6(rp(qy7Uo8861svg%R`v;jmSbUvudAqmKl&K9l0RNJpllL?DCv z!67)}<`bkMY2K`zli=(%oWGwqD_DMCGf%=;kCo`=kcjM~j8=-$^??9y*r!|8FyRd? zLh}gx%2p0NokH;K&3uE0IO0?B5s%^2EOmE|paB`7D@OE#^EO9b>EY3m*$Cyg`9A8& zzBDQ~){fF!Sa52zumf09WNNlSzDTgG%qCgsx>$zT$pryxYi2t8yA|+O_7;RxfF&V{ z0Hc1vo#|2h`HdXS>}X9?iOYKHss&j}rfEi6*c&NFdIk!?9H!2G$=l42`(I zNYvl195=9*^~sqlz)esueT@8GxneZJ{=IvqX?>0_knLlzqN`dKEP$|61)b9e7ox>>j0Ry)8)o8 z{MZs%yj{>aMe!%uflI^s%^9mN#g~i2 zxcbCp8e`kqH*wUooo1Qfd<}I_aAG>|99hIP{(|KO^phkT?p0weBjJbI>9G%Zhf4Om z&TEZHmKi4k#Id=7wngT0z0)1Al(gs5hDhVH@9U?+=zZsRWF#6?R;6%(nx@CBfgMMl zAaa_usl#dc%plMV{N6w|yNWuwE&nV{+9PE|I^;K+bMq1OMv=R;Ub?3mivlWw!%u<=csM+3E_Kmi>6BE8OfUYPWzz+sDZ|bES=A~oZ5sbdfAgg<=PDT7 z(dUi;(4-)bAP2Ll#amz%^f}YDa2kY)CL?c_xpz|a$)AnHl3oPijgZIzKZg#}v3m~# z3;Id>?%vA4U$NN~^o8*au-c9&)vKc`mUQGYkxr@^q09-N)W zi1cj?X_{Kyj!8a%`oxKeo)&;aB+sB)r$LVdy=iYl*%2_w-aN9z1~I+4$uQ=c}&H5VO=FxA8+;2j@Ul$N(J@G@kV(9R7w6j`UAhsY$188Chz z5qTN0(vb77M0d(6w~@~ThV=2cs`UyxQxO3-@1BnF+4U}?GGs-^sT^vDA93s-+-S;d ze@}k6$^bek6bhdeg(1C@;yx=<1ufM*z4-Tmd6)hbIzRrL1qxt#J=h~ez0EvhV|w}O z>Uj(FF>Ee7@zG`ECy9{VzL4)E2ROw9*H-RgfLz}ww|E;iQ)~?R4BZFG>_BE_yOwh3 zgAi zPNcxv+~<^UPJR%4T!a?z9G1z{(DX7PnM)-Iu-dZ0XDZ~0RMQ*FLb+M$nv+!rrYv|R7tkbS!-Q*0fI>QOc z;s(RosCORym+g4XKLrUMHN9E1NVh_(OxXz6c*j$%FjO{_Om4Hvce_n^>ld*Y6QnM9 z2q3YDXY~b~iza4Q?3rS;S^YMs6;`#pXhLLhI7ZP@QJSNZg)M=~Sbuw9br}Z%+h9qU zQ5^ZaeDz#j*T1My9_p_f`WYqQaQa4T9xe8ryn9zBg2#5+P)%X{a6ov1Ch`1M>!`o} za6`IRvMV%r#AjMk#?^Xa5?TTOJ90m~r94ygSnAAOB8unPIk}&nz01$4UvIBGv@TOEuwiXdwyO?(SY=^8 zwRTbIIZs+H;@j+5mL5KxKJ>E_8fYC`7@xii zINLvC{zUTR6{~ml+Opab8NOj+#PJU6uwcxfyV07;DvgQ_TTBkNL5zl^e?Ps`l zO0NQfo+hs)m5(%9ohCh;ZO1Zs(4pTsia;%%84wI@PdsjNJ$i4H8gG9YISG5^j8+2B zkLZ{uldw*@rQw#FAyJ=aT9uEv;{8!7y7u3WJrAGq+EtDI@-xD;ftQ4)!eay@&Y?-k z!RcSKTPXnv+cFqiJXw5(W5@L4wSb#7we@$7_n6d6oom2!R0N0eTf;0tOG5HOnY<ngaYl7>D z_=ek_Y2;1Xs1@76-^`N^wF$Q=VO1_vuRlA6S${9`23Gki8vKz5QR^yuA-wr_LRudU zq;u+0=m|kI`-_8YAR3-nLE5#SleP^v`~-IAMS2q;4yC!%p#u5|1}u_Dd6IfAn=tUmuB>R7)WboQUMt1qMWGP1Qm}x-@mpla86ge$9C|dBb23 z1r+s2XUWX_!-WhEV;0Fg^J=NY?wBR~m`P9as7T>?`GB%8fV_~HRU$QUyO>h=*pD^@ z#-p^Lq`CBMlF_Z#5aF1-sfatg>r5ga76Cryn7pgNy*_vrEk$f3dS6(D*c$$LXnSze z1Fu*q3{15|mqscdV7T9RPsW;?dKobcX9@i-9fw5;?i&XD?NV8o!}r>`)>*tZmMMnE(j)}Hc1j%P;#5JW7NMVstio^A*8P7)k*I};Q~yq0&Hl~ zq{mE5wQ%TIiueQT_6G-(mY6NEmtrz$n#tM2Xe|>c{bf;5++XR8&omH^2O1dp@?{7` zD51#*F;qXi_xUVqI5bZC_Q`H71sc#^v4u zCbjdrIcYEPQCL1;T0t6^sR!SNqJyXf0J&St0zjNlH;wEWm-{?Qydxm?rO}%PVO&-l z{@MS@_ffX*CFjBCK{iNT>atCkR)UW?NPesCH;=P(_WS$(uPK7NkJbU4BGlxp*}^iS zrQz)r>MXxwH^aC9snO~d-H8o&7qozg8~y<{kGr!xh_NmVOXot%1O@p?aV{=Z<1> zf4m%aZ@p&!kq3`Zr!)wbuW7cNBwSUA1!iptV4JD`WmNe&c-5`yTBHAU6DanmG z+nd5@0R-hqw>gWoiqUFuSjRv;$ACspG0Hl2iuwG$%(rP~daw%wt}o z32r<<8Mo!wqD@3kA;1JtSq$mvOHK5U?Q&I0+0cGvjVD5B!}^swULC@vSFbXBmLjHw zg6mjM=6wyK2PFo|JVcm%Qc{z_92&cquPVRn$$|=WS`yA@jb!3m5k704Am#`n1o{jbm9i8yiT!m)dNO0-(a8 zb{gGc4u)S`CR$B0s{&TEDE^HHxl|AnLzIUny$PXYq0#NS4Swa}#)%3LVJtPfuhF8h{Ns3^=VD zfvW?`Uj__KI}`rRXr_JP&cyYD4g{3QT6_zeQyQZoF=jE^ZBw=Y!vlVex`>RUKHeC& zw^OL`ZG2#1mrYayzw?7#kT@B<&uS_!1?CO@@OyD}IIH>?S_M!Plyvn6U+-y#g4ruH zkB}!nL|euskiQm+{Rtf;3@Xvj$htr|T`4Xj9y{ zrm--IvfMCkZzddJNtA9LG=&L^;bl@;B3(jTnM);SN?L-^zGdnX&D`#xCHqI>9=1!5J^)GoLUrHLDI7?N21Rx4opORgNB~B zEe>`-zJ2jp1EcQsD&=PYxs=m`1=3}|;N?@hzi!I>P32jT-$>ygb1UgOcZm#+B>Hl4D-LO8m=B#K`s5>%*2M-ea z`&li5Hrug?HnVUH68tAlR@jq=aMegMyH&_wysmV%u^n9)0QP)cl?e%5ZgqQMnQ9^06R&ga_?(?%|z?&d_UE&@pmXmoDgYT zh9YN=N%(dJ8^ptvfcV^B3p}7oriM;1TCJdr4Ha6dp>HsrNLX^LXDPAT4`;w`i+>Tpzdfz^P8z4BA2t`8uN(`(zOQWG$QcOrfco)PCKSb zXlM-McNi+>d7Twijz6`TsyAv&DKkgz8R~nwfmMdNxbE45UC_J0@L%39L~dY1fh<{? z@nVRQavmd#LQP08-fNL(A-IJ(>6zty&0cVM|M&3<7HU*L*)rVK?)MLCAJy<2Y48$G zz@1*2J8pzNM#{&#$d=V0?%bOvh;F3)BNbstTY^@WxGJkP{BtxER8=(a2%LXHc7IiS%ni}bj^amXLlFPH-`qCIw0@Z_PmTCg~Z;{*mb&FloJBi zhb?ECU?JME!-u>N-NNaXTt<_S{iv_gW+xu)o&3OPZdWuaJ~BnR@)~J|DtDJGoFWoW zpC`?OT|MS$x4dNjQRS1>IE)_Kcm32)b$y9d#nTb|Ptph8eP{epFA#*M%y7U1zU-Ef}vebcjDo_34E>_<|}^T+SkpquGR+00JXZxWUx>Lw+) zI&h|D7dz72L6yr&RFqJ4G(U$XNHubh{~@I6+eoe~7qRE{9D1I5kH(MWJaw99^Q<93 zSgHkwx7P-d3jk=%xN~c+QLYzO9d*8$z@fq-ZqjQj2MCiYMcdaLf%P$8<@0AzjxxL- z9PIu!aOf66td)jo(3!UXacJIxZVTx;aTmBAVCu&A_s^aQcAV4JO4g;~tT)Jo&%|b^ zmSs4OW$EDC$LiPp zpNswMe@`}lcmMo-Z4!H(eSNzBN5{h$a8vepHQc8A>*ySTzjH<)_MaYtJg35d{>rOr zu-~46WzPD)yXEY^j(@!W-ZuUDx3|uH`SwrL&867SXY#T88!>qFLowgy;dP#i1U^T? zqXwVT_l&G=C@rOG_iD%z;gt^8O zkoN!XAHa``t=^#u9n-&knCODVeY7&_h5$`8T)zZRJB`fNAgjm?{$yDL(xYz?Zv zJ-!$H|8OCLG(DB1Hb+#!l)yPtrH%>bUM$tB6!pWs85_ZucC5;mdPKz3sq)!(FnQXpGiBM$DJE9JchSblL4yx8e7l=aqj z7=#C~@k(5&UR zNdojG5Zc3X`H~9bnOZ%YNgM$UxY8s7l{#R2N@G0gBZX^&1*-4J==wFIpx?KYs|vRU z3pjMSYHmK!f8uMBbgDnNUGeoMAx!=dq%tC)391l+TxP*0tW#S%-Yd&^iI@hbI|n8} zLj#$iaRL?s=W=2| zEsxubMV;%|W?*l{2&{OmKC6OLJ1JgmN*!kl&MuWwcIHv9B=`|D3!M7G)nxyL2L!>n zqmj%{P~G{r!LcK81FW#3{YXy!8XCNu5lh;T)-G5j+4ogQExZfi(rQvSEaA`*omNvm z4Ry4m|LK>E$ly00Dg&_xX~QzM3wET&wGDN{GV9tp#KzJ_hsG}TSwJ&lqbtb6@Hl7^ zK}T(xNC*Bnyc4mJEv3s6xUXZ~-@Zd6+kF!3@|`!6^PP?=|2oH(w?qJS7SFZIulEfN zlmSdw5`Bwq-60!4!;84iJaywvXqPmD7yx(a=k_82A8S|2NLa347G9pn!6bm|);{tB3#f}&OD22l_EP#_|x%ja=!19gVFhUUdFQVKETgN( zeZy#}r{!n;a#}Jd4IBYORc-a;$rf58+kd}>l_rIU5=)irPkTlEjR(OH9K3&OKr6CESLn} z);pzKwiEEyH$Nv1Q|t1ux~iq19h#5Z;wW(u+`PwmODa^W=eU=a9Mokg%DBfZ6GPAK z>5=UZI`vZ_kC?9{PDsr-<`K6gM8DSrWqL+J2R?nJ?u^Qjn{VFRJW+<-1U*7vF-B=# zoc-3O3ir7S_JBs!gZ1^#_&zKEC^uk=cQHfyf-@lkKej@^gj;WU1q$mV6L*6-g5L6F z@s~hZ(TQS^iE)sqoc9p3hCVxs8Su84>~ZxVX#CT4wQoosrkH4vpts2Cn-IZkYCJaM zDCvf@>4~cE5eTBwAyI3hpn0*|b2XT{(_x;-fy8kulEo)e4hhT-LyM<*_)i zsko#Y6ng(Vr;sAn?$lRsp>HbP*gJEbtv8QKL#Uh3yd~*RTSCPzo5*{uWe zdQHIW%@J=!$r%3h5Py5O{_D))V4%V*PL#{}8WZ-4eU2s=*(~|TsJXBJmI12Xzyf9P zng&Q>cW!U$zp1JEZb@dsGNr&qYS~laY+dE;AA)9bWhZysKk|EhOXFyhF9Bt3y4;c9 z4qb{*7I>fA_jNShI+{LSYopua9+jHFmZPW%-+T58W@Y&gxXg&sUF;m=))VSLNqvTp zrDV;R*EoIG^V0%R$V~XcdnVgdRW9pbEE4E8BZbI;+`PAbJFuPN#zwT$f_Sf0IR5a7aOOXKo!<~N}1 zE?V@+HP2RT$MSy1HdBSYq|C9}z_U6|0xhO#-MM6DbM4Alh(&b~_9!*7vO%RRpREyi zhxLRT+!fw5cCp#93io1q#4d9RO*0qWxz4D6^(o}{+aE(W=#SAt`> zS}QB@nK}r*$MzEI@nKa>S#^Bl#j29Slyi8sVtM`8BNpr6 zCA(<>qSZPaqz=?(-Yk1TuCv8KEj``*#8IUW@Im_ zAnP5jrf~JZgPfy`#r?mKZZ29!+Yt*z7Q|=D|HA)*l4j?A882aT+wC9`UZCAJmbI$m zhy*0LIF(9+4bts7K!hUA_xAjTPd}OndF^>ogU&)LcrFdcv|bO-%IW_1FIgmm=lc6N_LkgjyAk-k z*apzWg#3Q%%O97)N7tpmoj=3G3#JZh8*=O~C>}?YfmE?LcRLId3+k_wDnF%BnhwJy zeXcU#R;Ajb@1ooD?3kyD<(Wp)+l`T69mg@|#VYFpu}XJIs=k_qtP?mrq6AWPN@pfO zjU0&Hi3uh*Xm*aJ;iDCKTWo9FCXPm#{ck8S^XDw-=l9dKUDQ&wp zH(rG5OC_RsQ#UdMJ=pTH}r~@V!TgKw?4o~*MrfTH{S^)?H%D1FgI;!6k*|qD6 zaNQx4UK5HvnpGK_Wsjy68f-g+Srk-9;&;%9Y~o=luU;2Gr!MOU5ZBP8I_X?a@Xb(h z$=t1+lW(9AYYJ$Z+f6Z~U2cD*BM^0kC}4KJ{XOxZu!{)VPFbTW=XEps7R4>8y!z3{ zLF!m1*maE8;2};;=tqmMV1!L{nud>cii)Sq3@lM%uf|WH4}T2BSbx6jJdYN_6Jc^{ z9L;Su3`qPbjK>e${F+mZ+*D680>iM~;lQE{;*vn;7a<>te&u*&yQtE+h#wWR?l$-i zpnE~Qw(VYTpHu{QVv;L7I9!^iUc?KE*}~ROJT0RofNP8eH`4qJG4}VK^iN6?{ml1*JuV#& z?LusE&P@kjLWraPijzbzH2iJ=yawLzDvDN#O2O5dD4i7!nLx0|>vz&x$+-$$gG{mJ z@{Guze4rKUF(cl;lk+06YqIFl5o(Uy(7R;WUUX*I=j6v3%kq&sihHIdGn`{k01+SS zSGC8ku=~qPL!pX?p*u`=YN`%SRn(i7d}BsufG7b=ajbjJH4&{mYrXL)RADW>sxsZB z$nC!OyO`;2%!j3{u|f>G3fs2`%8_Me(NLiF-H73Ppyv8+y-yM>Co8c<%nJLs{N<+z zx=eLuJF{p^mcOR6S9S)d^GH^c`2BSdPO@+apBWO&r1~wrBlqNl7VnuIKQ=HYN|r;} zH#nBjHL-`}iE^qh!+` zXD@LUqwc=eRB>I|F<{2WoM>W!x`U+O8Dp-}5Ve$uWKU6l#0~|1N6=_wo*!GL!Ue5fz8Qqj`wZ^(s5Z@@)dR1Z~tnQnJSPymgO&oRr17~EXYg~AYgqY@)1l9|DjyK zBWK^=Zu*NnSiNpOFAfkTDecn)6Sk!GFJvwj{-Uyfomo&^^$bt0={>L1{W}y@40hiw z7v^Pt);5nBBj;JPdYlo>J7C;Z+p_JVP=u@UYhreUhDzLK-wlfeXO!Ri@2<=7njs|p zH07whG}9Kr#_;me6v1}$s-+%xN?<-TG1EokbGLW7CbSAmEg=W8-#1%w@#1Abv>r`j zB!aY^inMf!ogb>ApAKon(LV zd9F5^rFTa`m4&~cqt~z`a>a?(#omLHy=Pj9FZ^#*B^nPGLag_>Es(*o_E3;G3qa^- zq^2mv|L%dBw6dy4_FMV>Bm2j{A6FKxUkL4}Wx3Lt>;kyhekzgp>Ez0Oi^xq!E#0^) z3Q+xg8UBHW%lrp5?(;x}DF0PjvJ?JSV4!w%AQ3?q+85Pt;9QoN1; z5blln;NayQRb$N;x>9ZIj|JX z+Bri!W$B5Hupp*!C5tc460nY>Sp9{xFG^^`Hj9O{VP!8j+7W8!KAx;XgO((m9RxnJ zll;bBtU?~|P_KjLjU(_^ZnXJ$R%$*!;v?Yv#fhBJtDo~05Q7m<5gH4K<%`=ua4Kkr ztOe{Hk?(=|;_~abZ|9DSCzdWkJP@|Rob17p4xzPMC4DRDIz(7-%gL~^N>PkHYd7}3 zkGmRx3h|Y(hPD7m={t$a??p5qq#5AZW+RjwyT-H1i=gb)W`opQ z{FZd^?eYksg2Aj|pJwqY$*ds~@+NTB#l>kH+sYxaM#l9Aj7(qP6k*H^&(i1DXL|l9 znP!{Ay)tjM2H$6>Fh3tfDh$(AnpRpFHcyDZ{;$hCw5j9uK zdei)jY|4dgjh8V$f*96J`r{5zf+ax?K!lb@wf*16@;5apQgTl<0ipzTvR^DHza^W; z#&yPqOW+VfjD+0qEICu6MJT`OP%IcOFRO@0*ZQAHFLtR6PRP7tewKDDZ{2+1VM4nI zP2_q*^gqaT3d-T5Byy(~ZniL`KX*QhUmOJ_}-ArXwDT zuHjHg3UYXj9#N6$i``l#SHg!umI3VKKG;7(C zA-oVt+E8u{`|mb~0b^M!^iM{DAmJ;$3zq}cO~wex9w)tvNp@O1++diN{`yCGs$gn7 zzgT))i<1nz0d+liQdsrh%hp=6d`a5szvYkg)PFnNm6gk_3wUTH!a1)}>>g3|;ZoN` zr(`uE4h^luO7%1N0HU?k&B}Y!)Xlnd&D7vdMQfS#?%^epL2Io<(HS*$+QtnccYv8% zE6ci>nhuZO%@cOlNxb&vOEm`f* zAKu#`+-SVCC$}CEKcg%Ee_!&Y&BA*MT!lAWyk@5Mlm<_bM&YCC)Q1nZQ~$q`F1%Eu z3ordo!dX#PK9sW$pVo0Ayjy=Q_^Wj7L;t%U8((&}`&!@+;RBovf)kKy&o@!+L+FCx|6PQ|^I9gKI{1nk*E#Gx zwv}7wYyDTvEm6(Rqy$a}5%?xTg720?8!!00kKrp#bOWc_Q~OyhQ>GaF;eX9e&A-Bz zq7P1*n+_LvHLjoVVSs1g6byzhYNa>4F4;JIJ*y7k2^_xhzv{Y*!n=I}whxy^wEVZ8 zznkH6Y4~r`fZ;0$5rMzzx6f3zbm9HG{7A=Qmuuem`Lj-iRD%`XS{&#N7%=;XdzZ1r#VqgPil69VUy=If0|+O#VEb=PS}Dsl@w+mRf8sp>-$m@ z{>KPH*;jCm9^swUepdQ}hY?!;?Zh)&y4Ru~L3_Pk3y@!EIj|VLKq0|M{kyFKXM=X; zpDjp$e@$l?=#0XGmuHHRHc%y86=bl^rWCwTQKhEhM;tiZI_jz?M}#xyLF-x*Qq)w{ zf@>f620mv_f-RsF$aXc?B^W(3!g;T~GsD!g_9^GB#f6hsO9|DI9|@GO7tuSaEIe;< zDI&RIait|Ua@nO_^$2GsIWo65`xqm3RZtksE5)YuI?E1{5Wn^P^PNCp)D91NM)HE- zlsuf19FRz#!>BI~kWE2BuTO8ak}~fg7CTabPSW}u;})iq+U&L#H2dqp+*dm7S{6H| zV_4+zLO;$-uChqPO`o_xwX%Gb-A{IBFDzt>UC<1II6pz+OG}hEzy2CmSO|$84FbJ9 zb&e$8h)Id%DpzCvanPpLamW@2fank!i>o*&D~kjiM>!;wEa=}=FHZ!-W;uD5*9lTG zyMmnlKrd>r4~;metlu4`%9My!HQT4${$3bk%iOu?H2T}>42-7sm491Nhc^D^-_GRk zq<8i*$!j!QqOIsD9#`L0hp#JlN-!eRK=7a3pehNv$GlbeSOJ>J5u(Eym3DpX%OF_= z@}q&6UBCa|1EU`}Mx>1%Itl$S_467h@p)qne5AU zaZu)U>FKv5?>m2?^{>Ur@MYqnOJ~qxKB_Uqf6Q2m(k31brXO53Tx(G!0XmUZtuXp% zPcNS=k}~CGzF9tMl_^!{#&U4M^RnmI>ggL?hxG1UiV{LRy|crkX`@c~!sQ}o#bc#K zo3dF>@~mz(749hhl#)J6^#H!k_f9&_V<@u14Tr1bycz8EH|3)OHu)OyzI-xC#^5@lUN}fb-S^gw_V+X>Yi|n-?p8%%|7Ng8idKPf zWn~g#L&ei^P_z|0FI=Ak$L0I)1=uhyo_Jt6G}z3|LEl(H(k4De|B50;-re|Nj`OE-7RbBlBV=>-C-PYo$J-@fG0 zfWz`>iS7Rij0Dr%$938aD{h0PonG=&WkulOV6I5Z-P#)7hVT@0a`D>(7D+g!zELt& zZ{m!;waUT4hxR&io3?$IG*(laizX2v3D9VJeit58fV|Z;LOy`xWw`NC zdpw(G%z^3M`1}W_-5-bHVU;f2>%;zs!(VaH95fFu+tAYpp0*~&(+>D4V>1;!l&1Ok zRoY_4GAS{RDcplNVA!JocFkjiRVOx+a^|t$@;7#Hr-|U|a*Q(R~?<|bv&_%iUr#T1L`3G}v zp81>Bfyrd!n*7`xKX z0KK*Twx)dc!pe--W*La47ryT`s{VxPLLBz_f+ z-Zl3SX&sU}IO^_z%FaD(XOn|6apKmDn%nS(>L2+4u>l0DH$m=z*n;^&{(6{W^oNR* z4UE$pP(!T=N1n2tblv%sqSwSQCU6=P_IvO3`8PN$aeD7?(BE3c* z+G1n{U0B6@o>2E(S{@>?v^}5A;~*^UxRQ4z`LWo{yaCvu zUiqOdpE*C?2Gy8H1WI4o_h!nYYBuUV#^}jL!Wjq1>Rku+4=!3ao;tKb!w7wMOIQ1Yh{fTtY!BF1;LNt0ZQP;QOG9)W(S9n7sEA25zT2_wa9c0x$j?*TzruAY~_BNYx zw2oW=cD;ZFqOm#Cs5;LK&buI`3^24Vp2TXe$sE~!%BO4=b89~c;}=KaQ@HvV?Kl>L zk>*cZ#=zu}ZMf0M4e&hovn6t{-rvkII`%wZ}Qn!TX0@)B{C=U z(Q0m>4QC}il}`S1ud3~$Hk<1++~3->19s$IS}1ZJ)CfH9sO!O*pE8cxq_^`Ub$ZCL z2_}f=vF^C6c3q+cr$?Gkn+rXf22N!A;4??jnx!^r5I?yjwIHTcqt^ zQiDjurf68>zUb>-(F=ChQ0nx%#S16CuoT#9N_712Imf%k*X!u?z#hw3lOF2D?0$d! zIyR|PeK!`%S#lYqG8Bw*mHzkF+UfMQiWK&vex~WHwi1`l7u;ZbR*SAU<+j(L&6}>W z&q)wof;n6*#6DxhDIDsV=5ak5=L+O77N9%7L?ppap_84B3On4)>{R5ww5Y8-tB%rI zkT*d9H3lUqJKm#n6fIi@o6S=Yj?~SgU2~at)WN!bwAp^uLL{+QkTjj*_GHci3!h8~ z(v%r1$PyCxq!%uN?|Gp>9x`&oMV&OK*=%~lsR@>jW=6U`4EeIJ^gM+VPwNUuFBTR+ z(jXad(hF}>h0D_a5*{{|jnX{h1(BoJzQPrH7^eI5cW_T@UnAIg`rC?jKJ|+J=hE%b zQ+!Kj@Bd-!t)t=!f<55`2<{TxAq2O3w7TgK$8eD?AOTNk5 z-S^&}-S3?Jr|(SNmg=s$-M8yj{aRmnxtQ;Z|BwFxbe1!EB?yItONakEZ;%1$o2oVl zpb_oCD6Z0HSN~d>Q5IHlA(y9m;GI12#RvvhZ z8#uZt;5{b4uk}>R`f+d>8pWk?cCC*U1Kpbg1-eMXXH{RB!l4t1kjk^&0TkN4^m3BT zaE0gBUiTWkair0!w7-<#619ubaFQ0#d6waWu?&;AdPLri31OLD{|!l+>t1JfcA!CT zvXEKq>pRz#+GdHX(H6>z+caDZ?fmBDkgs;!6269^C1ONFw?>ZpU#!~LcVeHksBc-a zOrxX(+qCGlb4e_(VkBK{I3EXy7^!^{Wx_mrrY1vi#R>aoEX97IUr#x5vZWRKUQAC} zMqrJsJ8Piny3+L^zNrz^364h9?uwJ2`guVnNE0W$ydW(2W${v~iXpq9_95${;bam{ zbIt^br>B-p&6MHvx9O+$)AIS#lj^DG6bsAm=Cs@jDt$@H0PHEAD52pD6N1)QM5LsV zOi8+#LSo?Lk9pIHvL=-pto770@H@-5_`Su)Vv>b)KLMO(sPDxe=2l50ZUA$xM^yaM zFdF+Ev1J$+KTMoEptzan#Ex`?KDgRD(6@v=v4w}WpUQ;@+B?~OkyT#+8UkRlWxV0> z?}Y-f^-5j7*+PbjUO->Amx>QZuj&#b*PmHXKp|MQRm~r~`%30-1Y(Eb5sw2`jau-& zukmkkdkZ%io(Mc>R@RK{lXLYsP)TG_q#HvNokFnOf|p4vupH~SYSqY(;pARE*@k~; z(h~nf;6U!*kGQHueOA(U(ry)rUiOR}vBeFx9i73;uU|Gc*KP}@&m;{$Gmu}CbTibw zo8|td4sg32v_9nt~)&{;mo@4PSQ(uYO;Dvu$TRw20BJYGGLa&EA4Sp{!(0piqQl zCx%A(Kx5XXvAHG?aGA(V4mbcNs^M-@W)r$4w4o&DI%jhGwrwk8yGyEj&0K~TLj0Z2paJdO;Pbp$%PbExgw@bKA zC*FyyiG>&e$jSLY@IAB>EMEm&`yue3`O3y=ij(@VT>yvHPQW92qxR3+F1bw|w@Pb# z!F||vcirM5-x;pR9{c?Gxs=T2D2_!abvM$~?`Vpcvjnm%OL9*p7hP#3`qYRR>zNtK z9G=VhHkh1;d}26d>?G5eirvk5>rl3Hw(uN!f(`vioq0jj9%| z~!Iu^uMR9 z($n`!2*fD*dKAsJ+3D44A;V}d8nh#C)@3ZAhxHvj!pmxy}*M)D>d5~ucp?lA5eWltnZuP`0H{QhkW{L@-`Mqql6_;EJWxW z3(1^qdvBNHIhY_nWY+)Y1G@y>taqIIoq%J@ExV#iQC2@Y>0pTL;9TNgJ~Ad@MMui*?X8 zF7P<&jrEXsDgV8Vp`&@{YzxIWyKY`|ogWyW0}mMW_e8}^F8!3@bP(2MyR zf|q~w`)#pF>BnYp3Y}yToIvziu2Ok^P|#c_u%n359w+kY!F!ivY6t4;=Xc7*abnL& zgk)s2-F5d`%vhs@ow@g)vV%RwOgNTRtNLVzRJTy}Rr`fIL|o1iuABb+g~`?MzWTqo z!gBsUy25gB0yvO`g(+M?F6PGe$euqc{Z7m5TjPHJ^^U%V=~&qy$Ar=6QGXT4qrkv) zAtvHaIJ=Kns1aIhD5S6KNFEPzRT?ihA8lg7(5~AJ;Cs5p$bKsmdU-O0I;{SEMKg@X z*y-!~d%p2@Q2jR~zB&Wp6nbhvzX8dgSJszD^1m;(Z#x@*_nHG&hY zQw`Vmf{WgSR|ZpwEO!DHhg~YneI(lmj4H(| za^U;Bg5uDimTrE6$`P!8mhL*Q8j()H*gr1?7fu0B=Qjt+VRP>W*#Dm0ehUb@`EXhX z8aWSmy^9ZMGZg;&bbQ|RC}-)FTJpRe@Otam^@@L>@>t#V_Oy8V_wMZ*Mz9E;{Us~?TGN_~0KD0%VpzB;$a564zw%=r=$d9CVr5|%@KVSeAo6`+AV%+-sQg(< zcbefAodjEdNW`!NbpaC%Q%zpP3DXEtYX|fc24lRrbR=4TSBv2C#j*Yeoa-5MbQqk< zDae{vfJ+90J6UGipEbr?or>Rt`j9H``h8ySqp}Z2}GdgFh`m`n_Eo*-AZxit_H~>i6kT#8)%Bbj?DT)i;&&c~TL;+=# zt~#fuTBv3XX&3#I;D&9xo#v(q=^>PevROaR;tUX z;Y4YHi$ZCWYMhcHvy*)GWTk%Qz2SsLOh+arUxeU!vuSpTKP4}wA7?1*|cdk4<*TkB+Bh+j?pE@NjHWF}3;fkk%e zlp#Mv21)#rFRR{@DDRxl8f1nH?#SNp7DE@ya-vZ^u9IwXcBX+oCi3c$d?)>`HsiYo zXsn<{P+tt#vgp&HuV2{L_QtajoXJAkn4Db!(TFuBe=CHpwwP%mpJ0-^de--B6cMll ze^#kP2Pa5B=b}oS&|nGS{gtK60kjh)Wlvc34^s*W0pwIvne$oSn-f)fmkyBdrHicJ zUTZxh&rYE(JQYxG0G6khkcvtDkcf0{H56#SJC9&_*L|mocwxw~3RERPOP!PFrByG& zVkoA}-BvIzLVG%j4QI73;v6YnsNDPY0dqW4mPzxTp_P;=V!}FdZxcc*Omo7TGdVsX zN$ru5J;5#Dh~P=rV8SfvyoEE5EZ+r=@%lF35}9Jlvy_gw)cy-kwXTOx4{lkE2$sb_krK~| zwn+2v(E5po=!?|GSS{UA%mUutBm1`Vey(oRLi3%CT`ezmkI+|V zcTKFwA`PhOCcCOQNk|`X1zo$o^>blf%rB!iJHdaCfyF;O-OR47vmd*}Gva7l@gZH?_qe*96$A?@|+xG0`*?E_#K!^gl~cw>uPg(Msr?3w?==< zOQODCTT;$u!oqTNSZ7nt3R#eH_<0$8GJBe6M;#9JdK7I3O(J)p6>`?5ct^`00Gz7Q zX<8^Chl1QGfn){C91yIU7=W&vc=0vi`X=`6USD#HJ>a%2_U9^A0z>sEub)=m>q7mq zI#qxA@|%V}NY`-f+lPBydW6w!Z05ekg%G~+1}`BytxV-Z^YPa~aQJN=I0GzYpfvQ5K+;4{Mt7Cr~>$t&b;!zxf$Tfk2A;8pv=!tl$3a&m2 z9dXQRsU><8^umS1j;FLawn|*H>Cl)>oWM92j9Ayry_xePeDE8jF&>4|WkZQb_(n{t z*eOFqcnxL$9Om0!O=*r`GbDtIIMgGMvQ%7^^?RZ56U9FKe~=t=)D&VS$7Q& z_{(8KpnbOpgcJtvy7QoDt2Y?4+iRn~k(%Vy?e{&w%hU+lbW{(9{TiW!>4oZ*7^<#7 z*6d9f0XGrZy3{8f<5RPBMb|pk3C^Bqd4$FhR}7piyYZSsqzP6~U2Pv2W^qxw8iO%` zQY*sJ0kUOm*ZI}()GFe5`OATn(W&QIEV`Gz8{tc6lVB=T-YR+55I0q1@%%%2_DrT^ zGD1-xaTHs6uLK`4(m4_bFU$N57ai>5H=y0r0(v}Mh4}ymAtb-g^Z~osqGd$fa_Hv# za68U+SMVhis0r$vbG~+;g*zMf@S~*kBbMSfR6+c$g3toY_Hca482V!flBcy5HNFn@ zE)(+t%L^78HH&vQqHYvbxtrb{1IGw7ouTQUm_S>G_5Jj~;~QNfKT+SZ=EdOt;?x-? zWl6PnmE}B-l=b}cDop;^#*6!_)A*XHQFi;#X-JdHgy@xJ{J;n(UVY z8iwlA8O+?Wj7;=G{}gCRhmS#UJx1ofwf|JpZ~} zJDVxVqpd^*M?UXgQMr%pzzt4&x)@)IGnCc2dsC_9C4v6 z6wO>VBLpg3PCG>4%GpnW9as;p-Dny!sy<*Yfgv+SiS&bL@o zs}_dSgDcNhn;SG#KQcUjkiBFAwXoI=Kk82pMqfA_JWa>du!*-Zwe{|s)<5-{PanR# zYr;5xf$B+!qP5orPj+iPVy>^}1zL8{uLy*VO+t;0i!w()XU3F?0p4I@u=~nt=*(hy zADQ(CiH{@WJC$=rUM-|5t!olNrX=?(YO8m@_3H-7tbV7?=T}KaI_WJ~j1Y`961G`2 z!0OtCk{n%g#Bq09|NWI-M`r-Wi5sRh7x92$%DEOXQu;0oof&20Y^m75i{!;M0W5z~U z0Me%~9NLP28@>F-j!lE3j&xOCCT2zIiqAyeE=AGNV6Airsg|nO`*3<> zdKa*^`X&^b6v3W zbRN$}A(7x>-FhL>X~N5(>%8P=${z-`%c@IA2d%#Hd4a!Zgra6^=t&H!&3IrVK2kqs zB`2V|<5vb@?9hdqfRQH{&cj(cSGU3?yq}pJh#Z!>zht~&xH+Q1j^y}=p^^?_6hDR# zi%2%~Z3Il$%19lwHW$dnXxJLCtO;_84p3{X$NA2D83v^3ONxoRwV9q(h5AYK5xxIK z46cdaMmFJI%A+`2i4>{}9sPu>ia*q9RtmwrmlgpVs1 z3Lyo4RQ7^?M?Z%X9anQrfARMSTp&*Z>{18!+ z*&_BP4!1EJ=p#&yp}mT<&;{nKE4If9yM5o9J*2*oPkl&Q2j(ysXR013^f@x2obK%Z zK*|Zo$cr$r8lB%BJpE z9>=UA64pY4$W(ZH*S)!E-FS6?zG>4?k3SMzl2;xH(+{d_;V5C#4}UFe)24EEr2}BX zUkdk-Pi|jl1$>qW9bGOBY-w6Zgx52H52z?A7VmS`07^(KD8*aR#Rf8bVE2Ufq}dDG z1b^DSHLK8ot6$}f(@p`oT~{{t*VMl8TQiE@hM@(=7p#Y-7}C-@?IY-<-PXG?t=6H$ ztc$mB=C)GO0=1mkb#`(L73adRytm&1jk5@mnma};EGn6dH>8P5VLZliP%ly8#^Mfl z=tUFc121&>25%DNb1`;gax$gU7)^XY(xM-(M%5W(~0MED_*a|H^(^PW7wnmT(=IWD%Zn~b@uyuVUP3-5jZ3$QCaEf?ut|MCaf{0png3{X+LHJ>Eu{F=@s?>eFr$k zl{afLv1t4C2v6o^|B_l-m%s^WmDx%M#R6$aDZ(G9PQI?2aeRg8yfctgMZ`Q#{-_E>;<1jqk`n#_-OS`O8{nh7sx~3b0|^k24HF&i`CgU`RDf8cUrDkg?5!y zzeEi(c^olVIHHl3cxhK02_yw&sB%1ubw*C5;YK&Bns=Zv*nDvfHH&SYgP&xtY@XAF zZU;Z{x2^?h9j(?OUJ^Q%f;**sWv>RX#c4v)p#N%j;7A<3V{0{o>vrrDrWIYK-NIsvlxB=TOe!E zYIEkWEailyCGkRqS7qEhAapp#xKd1P!YGBNdlZKTol4XKdu!V zOHGoYZDv)MH(Yb{UaJuS2o~>mHK6ptLefY*B5#vPG(DT^eWmVpm|7{P%U46>hjNSE zO?MTicd{nz)CjkI~(7dj0~0{aa1b_KXWL#$R>}owwh+Nm3m;8 z_L$7A&xe-?Ps~N@v6VzKL#1X%u$7{rggi#Qrja9&Rx>$BqPe4L-FwSZE|xkmH>|WZ z>Nzn>C{p*PK_9+cxwXQ$h7Xe9a#T>oL87A(1Z#3n(>@$b>Wtt$Z!u-b=l#xPk?Bx0 zyk>WYzVruaUHqbc+4`a%f;UFY<1Z{dKs^${>88JHzQrpV!WqwyE0I3(KWf2$%t-krn1VEoyR9_-x17u4BSuoKRiB@mJG=DABIHB(SoEy6oa++U6$%Fv==`+g};)JzS!|gd)aV1sWZFGwdz%Ez90YuE#g>S@UrMZ4LpmalRZ%R2u4t-`slwS zKBu%|ok{`+4&oQDOGhG`Vu>xfvu z#u-uG>*8q@83=tkZCd#V1$lE{7?OBXg(>j0QXyzKD2!#i^TZ6oCPyEVyzU*euI*N%KQtroSW1?mmPX{PaRLL%{#(2a}I0 zY6_^+6E)P%;m(NAf#W<*W+~^qc<9kBHmv9*5X7vofee`!b}X>*Ajry zf*t3|8K*Dk>vtwfBxEqa@Aeibz~nF9g2yAp&8rECq5euUKv<0hl0*nxuP^`pu*uPQ zU>EJ0N4q!FEzYtE$p7nxs?3oOqgPjeF#R_OiD0O$VEb66j)~>f;$BUh+?dJ9TD`AO zNQFI~KY1QeR*Dwv-PCt!)+#%qL7nl!K&v}nW3r(^zYGrT+jakz#I+>)-5f}rXKTQd zp&mcx?ze8`ppOKSi*(%$o0sD1eFx;Gf-usUkVP4nV>xAObb5v4FsgMM9gs5AOgkmW zw^@7&+E~J3dE>>Yb)bQ1ZcC4xC-79rr%Na0ssdfNc~XA&h1(?3f4>~}N5}N5U)a;p zOjxK8xi=AhZv~yU_jdH{ye1BxAAI=2frSK8EFzL$=7z`MlYh&3EDzDv@2+hNa2{-T zi|gz#Vj!c#0s%h@#*myjgmq0Q9)A!h(BebJ`i;k5Vm;@WY%wjz|Mw2j%e$2}+=^SG zDef$9son*~88Lh{38WjW+S=*JGOR_(s%dm8<-xF;DpzIg{CJgbPZ{}k60w(SdQ^Qj z>AWjchsWdBgK?k#xJhmv$pr-)R4^hzp5flwRMMMg#|5R&9(46F&~HYqA4)JyX3~WB z8ik!BUh}d}sxGjy4zM7^X=0cjODz~xMcq5uKA*rdA7BeVVEc}z6dbT&VkbeE70-qjww zq{1E0EN3;t$T+lg^ZN??91YF_RycCj;<(gI*y@GsX=$!#`N=422lr6*pdrP3$O}Zt z-8an)66sR{OSzz@j=2<16t|fC;jgGxiYW6F(C@l0t6ZmCd>LWZW^kCxFR^*4V=WQn zi8JLwC*mUhCGlW0EX?5=SLUJ)_Lnl52wR^=K-+=wNl7TNfqQ7_t{NH+d!03w-*m8p zY(-+e1pukfs)eS1=#TURf{C%_F<~7g^*-->m;>kJ$R+mH_)dB^;#p#o>+|9edOqnF!}NKe>vlOmcY=PA zdXA1H*T<(Tmi8);^6wf?YJLbC7%Mk<@g=&Jj>i2i5b&=P49{#B_-s= z+a{Lc9bvBm<9R$H+g9J^099@oi&I3*phC_Bk-80xRIq9+(xEc09tc+W0etoNs_0(YE@F47c94|~tH1tg6I=7#kCKOaA?~f=Drxr8XJ^V? z=p`Qh&N9WuPkJfW!+|2dXwS@q)QGOj8rurrpJLM-qi1hl@>rm}%g4StRc z`E3Q)AmK2u#FT|udq%Ayx?vdj-C2U*k8O}15}agvS}^sOxM6Y8id#bR zV+L}IySK+IA=PQ?DlL-;ELS8>52qjlL~4ILtEsh>fd<{K+lPli>58PZ4im~#!>dmS zq8OZELF|*%dt0ReW_9;921kU)aU9s-Lw`o9l#@~ zwj1Ub;+5nbKPqFyOb@R`na?9nxmi%I^*&=K5)Fr^6UJR}SFpxQrQ3g?!OQKuCUgM2 zL}ShC(e)~2B={Jznq@bPbHtX^vfftx8OurR(&b$H9A%V_7GruHTK3h6$L`c=ElatN zhxFwE!&M|2`}eM4#xqrE-xu44IMCb6)#sO&`&;l_$%Dql2J|a8-u?bTabWlKv`)ZZ zCtoqs6X;>5%P+ev-7{_eyu!+=(10wAd%DJMF1vieUNrGHNm&}4{=a{j{ZAwOrq-KU zceb@z#P1ZS!E=l-R<;qN>g`m4vF8(30{;cX?VJ_(RWg9E-x~h2Y z9m1y~QYH!itiTxw@+LNy%Yc<$tVVNTJ7crw=bA9An!na4vw4rsD1AP?%@^lXltu<4 zUd`^IZba3yvDMo^)xx$-4G-w|PQrR6)9QnkY?#bYSuEj?+e6Q;t^Fa(?dv%Nu1mkY zeM~RJ%d9&)rqywRb5i_YM7tYAHS zW8j3dJcwY$i$0-(vn<{^H8YzOu@2Sh7T8f`ABqE9x07OFOK8|aBv-9@ccl{P`wz9Q zXV+q!PWf1{&Sx#wVz@*)#$c_Ax6TlxEEs!Fz_k!;b+c2%ls}r6Mj<(gmShO&wP%oz zco3|!gi$=Rio+7ku)c|e$3#ovJ(DZn)?g-o7R{-Nc+a^uJy9RSwOfug!PhnXwJ`Cf z`QcT|#t>|{;V*T~eNcecFdX^EDD>NUmBxL-2M@EbrWK@ewi}9fTydI;pJgl$yHm@l;Zz$J`DFQe`A)fiMQpRdG zj{Zyen-Pcj$MNA_<3O~zBhqy7SXf#gk#gx%z#5OMdIg6TaG(rpyJH$|3yOo;zq*Ye zOcv@%j6mWmNZs$_7@=A5t6U<5J%X@tY>DtQ%yX*Tz!FX$+T5w1;Mx~9mWs|z_q$%Sty<=(9hb zz_tQ#CC)xMk>LfP%pwTFevNr(MXfxw z`{{l}D|`Iz-$2dy*2!G?lav|Rp?E7hNO^*Xgr+xA=bJYVk#U3GH*XIjV~Ili9WCML zJL>HXB3`2_fbC5-Ci~um8zxyKH#vsxbp%BVS)99ANE$y|V<)J#cKf}-G*QS1I&ur%!Z$WBBG5;ZX+)GgPx;qq+OD?2 zXu3;~O6Mr5&#cpK(8=KrxV9D>I?WtfjZ-i@_Hx>yk#c3b+DCM5zE^ip7w2B;S&hM3 zvXtMdsbQEEXpcmL`Ai6|;7hUCGdGQhbLol&emv5jPq=rN2*2enBRHo~YV?)ESWxNq z`7-M%f1P}vOQY%uY&AR|$CGSQ5ILptYos7VOymsps;|Bwl(;=N7*mLD|HGb5{IxUM z{W`3c{Z1MC0auKSU2u$XAyz(y%(BolCTq!}&{X`ZR454~asL~RI6juom+4u%IWHE+ z)CrB$>w7n!p0P*5j^vxI zFG6<3YJZez7uh&+S|?GCOS1rukO35-V;jHUa8VGzOQlb=6lK@YMD-nc@hXm|>q z&2z&cr+}U2P?`UYE@w|z`t&mA@-OX1HU%hIq}dj6kZ5RGAjV{3f|&LuQqm&pQCQ=$ zC3^4q8S%BpDaO#h#h|hU!$~v(RC|HY^MErNb)>0eVaUhbtt91~?phFQVr69qZNOK3 zu|O>wt=qvs0v@mq-YFtZc^BBi??nGPt8N)=W?Q zjrrCi1_aBW;UV=GdxYMzSKyV>X|5TEqm-cjJ?G(zBK3=!!>6p~{nnl?z*%lx(?P zW%PJ(o#Zo*r!YJ6-D-RnAy**P)W28UO>=!h z)ol@qZMIJ01Yv!;k>4eOC>M&olVdJVcZ9kuh_ zF}@5Y%)`-;l>gRzFVdt)FsMXUn?!ZSY^9Pbl_N56e#(FF;QfWJ8KVTc-&H8tN$=a>cl`{emWBCyQ{ znINenE)8J%XW&`$=AdX(kec$OB-)&$2*6eSu?#8UYi&=e|J?vMb7`wrdZ8>ebo0;z z`G{@BQ(#*`^;6)f!Ux5Onex44^TZH_+_Tx%lYBd7H@p+h)~QxC;YRWAZKkKv&03VN zr%IPw?yUWEsb-C^nc(u+Uos_(3#c&~zBS9)fExRHb$g_Z>RxxE#CyuA*0eZxvJMlR zgAVJ^Q-o?QB`L#RhbYI@A{$qwhtud;pIKl3i9NEVX1Y_#ejIZG6pn!Ygep3f9pWtZ zZQ?(HV`m3?P>^+dzv#19Pn3-*qQCBMH+irZf}NN)U51Y1O%qB)_!VwCVa1eDj@$D@ z&j0}erE#+G{w`7xzduH(?+nQX`-SQCY8DQye29`BK0*lgFRk4Jm6XJ{H{x=HL2U4W ztQe*yIk+rZ#}h*6GQO^gsMikwf$rjB%1g%~C31cO*zVK__F5%md_%vmNb$5cgj6w+ zt2UH#OsQ{irPLBe&GC#i&qZ4f(t0N2M#o^_F11*8uhY%#;caF-!s8GeI50zLcQA&n zOM$*}0;3(Nd`K-SpW`(8qQyG2o{08vSuM)vd-fvf9(NX1rMnL)9LEqcsr$~<8qp)2 zPZDHuBWrhHFwL}J@)4<2;mjyx@wr!*HU|s2100)aQ35m`C%55?j{pmRBatMzd4YWb zV(=EY2}##Y%T0<*A2yQUfg@TV&lT8lWj4^^Es!s<;EA>1w?91mCi2 zWHC#nj!K$$Hm&--h36bZ{A@QFm@#br-H~f4s6O$YmWe@0E_zH|!;o?eRm;ba3W2k% z$TRam+H^>uJBu71XUmRcGS7o3ah`h8a%`|JUgcBvEiF`k<1H_DVLuIPzeh)s`LNNx?<4S3pCd&j%+ED)CpF zY2AYVP?m_qIfi)w7yJkgXxqVEv22Ymi2)kUXu{Ldsq`q)|fTU2Vf zl~niBz9sj32Pu}_0Sju$Bxm#*=$zRrJf;NxQ$@a_i~;55(~?vcHn49fj-R%jG%9hH zknmYC;6N-0+w18{?;`3ZgwKpcJ$fzaIdg`b z?@U(yW9z$RhMKACbauM6N~Pqv+k!FIr`F9^s8#n*b4(|%3el9igvaqq%BE&RE6bQC z?w&qZ9uk>kA&81L?$^agJa(e+8@6i1Vj`jn&5;oCZG?G{e9q^IiKVV>=&9 z`kC6(tWKGsCLXYK-Ik`r56#(&1j&|mMtC&)y+i#=8)QPI(JiTuq2QbddDC`M!XRKS zZq(d;Cs{TO-ozxr;`izyfG=#+MB`u4&t=8CHkv6AP1O?$j_B&cp>{nXe~%NK)s$~l zCv-hvhRSz5aWV~_#b8+nYVrutkTPs}>^(?W-AkOV0L;6+^ch3hWq60+6ph5P;V5^r zqYWC5w0B<&q&ws-Z@M|lvywA*ACs3;+x`~8F%K;N*0r_3&d-_3MBMyaz~e}y?+A5Z zVRaI0OKc_VbFte1C^^y<2NIs{c7BAq-zxEqmbEp^Mn1n&BfmnJ|?gUg*OM(88AWL;lFt5G9+fPSL*gxvaTzp%_rPhv>1L0hST)X5!@4#qv=AR`yw&Zn+)_$DyjA-%b$^Jh}I;u4Kk9;S)Gk0Dr(5v z{EHKgM_urLi9g`_e~do>aPj_c;RlUcr)6v0n6F(*TlkSwU<)92irDO@$KNh_7+*8K zP=pnoV+hrGEG{f%cQm(}8*5v{AIM!hR{$h^*J*se0cXh#Spy8)Byh%q9Rm#fbKZ`6 zznTpi1hj8aaSAKEUat;WKVF_*@0||u`n@6804%d?X6PlStBz(Xe#*%jqy+GX&|7AN zDeOF1orHfUXG_sdS2Ov+;mq?}IfiM}0A4TSYW_=}m1My!!>`M`U8<0#EIZGe?k}_D z1-q*u^(Ds~`8ce=(~3Pd8pdN@!C#qdQqy)uA4$W^%yu9M@ZRt0cPWaLLHl9TajGK7 z)^xBFKlVTCpn|5Ar8T$(y^V~}bLG%Y{#A08Z+OXfQY+R`|NUHoD?y${G;?#ibT`bDe2`XILPX;8fus(dPo`|vPgBdC zoaZPqBw0y|kMTTM+fw|rb<7=&>}qB#DM_^`!_rsoFJ~*m)1N*P^_~!Sd@Ri4(P{-M z#tQp}ZSuu&Iof=4KAUd%S&63Tws>|G(_`eTE`Tl1=(?XEE5Hzm2H(_@;6X4J?e4g} z(kmUgd{a7Lg}z{}dZjm0#q4hLRYbXLwJ7#!iy)KO8$zM8Y%A(OB>TM<-MU{|;!okH z$&WUCm@(c>aI3+pWH*so^6}+KJRD)0^|9twQboCdVUu-Y)Z(mceLbc;S`i@_RiYso z%Q&@^V3WAEL*-C)bc?O%&o#Ra?|Znn1mm*xpKO{^$DoAr*p&(xV+@+N13e=|~O z5$0LyEhD{IB3&7Y9ndlfyeBTVltRVkCf=iT|4I3Qgu4vj2#}fJ>2>ukO2X1i+&*f> z`R1Bh+d#Fhjqzy~9#$3J^s)4_yfVE(>3Zoeudj^ieD^KxD}cN2VE{W5-27uF@zXi( zyYHolCe1(|ALU&I{tbrUGJ-kh_C~lo0&c0P7Y-{05 zd!6JuG?I8?@`>6 z4etwrjzSvk>eRwahI(CmV(M+$$#MME|~-8;TBM?jUgV4EfE{DcW}X z+INSSWp?xjK~9RHk8PHVVFH87cluV)7s;B_-rozSx*^zJSqw`=tqU$wrNrFRgj12r z%;M3S`*qXxecc-6Cj((E;#*qnoLSYteOt<*oC#hNvg>vTPS$UN*zvsH1z)$_KhvwX zGw`PCr-hv3IvK_)?Vsd-_7BA9t0LL3 zpOb$Lnm*wy`W--O<+>rq54d6dRsQ_v|D5{&TjZIVmy{WWawdF8`wmprZo_TFA`=^dBw$BSPyIGj=uqcZY&i)WzD^j#eh&cs2 z&p*-2X2}|H-+yjKsg9|El7E4wsCC znxmAAgo3ftfARVsWK}O$kh#5#gM}jnv}F}@OKWHVpBKevQRw)X(<8GgxtN)|{OjW9 z|J6k(QPs`K$-(^eH&Gp;QhI7Ito4 zeF^|Og)TQc8}tqzF9py4%EI-J%FWHf$H&0|?c0Cm=H{i)<>KOIVdvuDr1)RO^eK3t zSpXFOd4q@Rzv_hE}>lovLR|EhqKlaqywjfeL? z^XcpVcd7gv?*Bt7|0NG()_;Qxw6-(npkU|wAENn3Q8c!P>g)fmEI>PBOV@uYDf&-6 zKoopzTr3@e8Q3p#q^M9KEuURoMM^6e}W^N817CsI>9tvho9&Q!@H-MJ{T7r{> z8(KF7I~xEh94>b5|E-_@c>kaMls9*<1X-~|WyizL4XxMO4rJ~^!3t#`WG-R;PkW&c z+kfPN%9;zx$$t;}JniuEO_7}8HTXSwJ@!q8L*`TZ$bew78TPQKm^7jk4UTj^L6B2M zZX)VPDC#a1LgY`Fy3d#rm+x3pJSl^JB~XOR?f9{JZszCKF_f27v}}Jc-q-zoEsrkY zaqqXWA<~7xxu81#TmARE`uzNFPH{MxckgRZ;fAg3T)2fvY6VG*N{G9^>YkUlb{<}l ze6WgsADN|h(9)z#j_E=an5HaSWnTj74*o;;F1iSofV{NSwzbJ&`CGuiU80slb*VGI zm?5ecnwK&>b`8`@SoxkhVLgJ)#Nl-NXu{;xIZ(UnUD|dT@xg)+&C4tzTLJu+wgwmD zn3u^ukIoMX5j*8l8Vh_x2WLM#8NWOj2iE8Y9x84;y4%0g77*aQmr^KzUpmg4bTuXP ze@cpjwVb&5!HH|LPf`6b$O+!JY{>glk0n+qOMz z+qQQ8#nbLaROC4)D(*wwhl;vc_hgDuDvt_hJW@{)8w@P3vJ)6}A;K{2qcDQJ>C|sV zzru4*c*DP05c38z;u;YP6A!3CYEU!65fPZ_ORPrvPMLo>esWEz^und>Y6bv+`U4%y zSW*6(Z0J=gF@`N4XmXek+)3^PJ-t-54A@ET#9Wz0rwKa(F0N+20>@k=3pD}&aUBlI2!MJjo zj|714Q8ytZ%?fhCq^=O-1$bE@T_F7t4d6T+FG&l)al(!j(9htw^*rG9IthMv$)Sgx zDt-F`ay5S74!mU_==bZ$L9l@kWI=@-Z2wu!20IG1Iw z^5u^-D8=&|aa5B58~u_gh2>m1f%8?sBTT4;i1`c#-MR(4Tk>?ZHuPv2UFbH8}0S@ift}! z>H8i}iGBLO@7)7q+czd1_thKY&=AMk#qI$j%~9jj;Vc4{PJpJ#MM4b}SlVNjO=W|) zvXUQSoNKh0!1G2S20?&#{cJ|%2BUm$pwzwuqm%~>xl|O=PVB=*j4LO>9pW(nHNE;Q zl<0JFoYH&N^VJ^wT*MSV*0rh)&dv$)`oxV-^k7ItG_l1~K~hjZt!17p`x}m*K(e(i z#t8gt<9p-%2Wk$X*iJF#ab2-eQfI@EP>BaR5H2ULdCC zA#Hr`&iU4G^6Cb~hgx3`9=bT?_!;<@Peza4(y!=fdmp;NR;bSF{|fl05@^Mma3vmb zwY->@4APqEAqb5LN{;eBQ+b-!%^mT@*gVoKdvAB2^WNz1!QJRD z^KDO`ziqFm@^s@kZHsulEZG~-n_P_drZMx>`_A{qNk=Hw95#A$F~rWk%xCZ-1XwfG z_IffxeE3FPRoUV}<2Y><@YVMo?W32Y2?V*8djkMl8VvG5QnVD2FbN}GQ)ktDf|{cn zWsw4X!cLG2I*2OwhOqwNFOe3GGh3Xa8eBbbqbNyNkGVzlC!>&T%V7UK|0Msx>8*MT&wFE_7=cj|gapYGsB zR$(>Cu)2HiU;PIz&}MeA-@#WfK>pb6AyWB*yzFUf4__S&+N{1K^#nSilW_K)ee(W$ z=(T9BcpaRhs)Okoeox;K^6MFH#*9B&^qt))>xK3n-I$@Cw_*C=IbhAr6e5>6QN7(`z`5&*hqPH@W|gn&Zx z?8H~+K49~^N|32X>*HeopzZlO$Y+mfE@Iik?-e`sU-J)@>_8h&gm-9wvnINV5x@~S zGQ{BZyHR;T2>v9h&-ast5@}Mg2d8ofz8twrG7IUi32$Nc5?EugG_QqT&=;kkM1C~7 zw5mnmoj2MK0)6j9RT6G&|pLHZ2B)1oKe`ZIVmL@C3<*OOVt^LfRo2hrCn zuU0SRsAnu7sRQGNPCpC5yL7G@09TOX7kqMSuPMyFYh*@(ns5M#gXV!uf8YTzfJ%v+ z7Ow_L23gn!iZ7TQ`f3v_0*Qn4O!TQT6^c`TbAsgF;0*-wPJ2v0wE9gs;1{g$Y4;Tv z<67|kP52E0o?P#{*0hEF(gbua3D1#_7b*b{y!VMNvV^R zyB?Rc)n7s1Is8BHV?blO?2m!nKe%7tg1C1Cee-^r=9nuQ*c*92t=!+O0pEI&jAkJ0SG(ac_9I;B$>by@yx;KMAq(Sex{7YOus+AN?21bnRXkB~rTDs$(4 zx-9zV_uc5*+qaQd1Ow3yhKG4Bywb_R_N)5+a6o$Ae$>l0^Ca2m?7^81imwgG%EZmn z+Yd!w^8@%m6=dB%Ie7F*W?BhM&0G(JSsZ)bD*kBqDGnpLzIL@=7I_e-88M!a4$FKB z##^w(C`kTsh9le<#+xxS#>r2HtX|s^9ykSh(V;deE@q^uFun~Z>@;6%KKHWMu(@tC zvA5%SdJ&EU3;tq#lsq5A`#eZL;h*-&G$LJ~7pWvO1I|#wNU1yh_nM%QkCwqGkmVK# z_$FQpO5Z@iA(y<)f!?P-bl*82qyCY;c|d|(0xpF;Cp$#nCYLWKz5ORUY@gph?yLT3 zuW;&@0Tw+Rn2((wpL=+tar1e-)Q;GDJWkCoxVcozpSUp=T7JjoClX)}56?t?xug>o zk+_YGEIhpp{{iuK_^bHYuk%G%4S+ZQB8-TEB4H>4;Ql!!VVvF-s zgG&=W4PxBMxsX?NCO&Mk=t$SIT9=SxOy)@9aP*L(I+1M(+bE`yTsus=OsDRj{FHq} zkyB;cZrfbjZCg2~%w~LBa@*uSuG5SMwnKR9dY`zRS607yQ-&Q=3#d*d-DIjwRWn^< z4WOoB%eHgyIsXbuC(JI+F7JwLBdZ0Z1;Oq|a!Rg=B37navRbxUKKm!-G3BjlzU;KZ z$l^@v?J~Tv-C{I%T0qCq?S8GHaV1aB#qCSWVK&>v^W^*J?C#m4`x3>`@+8%UVq<*c z5LWvdt!dK;$cwHOn9e=w79cr9=oIM@($+S;VSO_`%iYoWHL zhKA9!Ss^qD-aP5WCCfg}tWA$amqrz*Ov`~l_rj=cfAMQlu2J7iZ^fn^u}Rl-lW^D> z%}H(3`Tj<;ws^()!uG1AraRRegw7q4j-zA6d1lDd0Lh-qS(;a7&UxHIqVVX_VlLjj`qAAh(DQ!h8FY>+gg8EYQfBi^A*{pTZ+=_zjO& z>T~s}*7m_U7W}OzSD%H1Z{9gUCz8?EP91m4{%W27j1`2~L>O)i54P`;M>6i)zrKvvf9%L>!fg6df<`Q_`{5c1=R`tU9Iuz@N5TNfLF^%c z6eqI#ke8{Vs{2w#c`r`xaME_;`Skh6GS2c##f{odQIF17$Gg;1(79Zje~w~I%3o z-9yOW`>G|ZXK|053d~Bln=%xiL}l0RI<=UJsJlj*KpfC3ec-PUo1m?I*lh&(VG_6D zbD`#6Fq=SSyWUd^?)}srK(Pbvx4&#d2zCt|vGM;N zkjpL+bs;(SZDP!!-~F$(;Pz9YLNab-!N)_Qd&W;tfh38sB2yO6Xj`JVq#9Ek&p*6U zxdYipC^7ZE-+k$;yF|s)f{*rP!4v z+==2K<++6dfdBtk|* z>;{qs+>F9*n*chi47?0ov|aRFF*Z+ld-1mgxAnK~du@)woK!v(J_@~MKZOEwi;oiT zC|?=h!u}-|r&{YIr@_C3;yPjarTB>XsqN7TP+G&|qI8Ds_EGke+Nx|Zw)WxcLEMjX z_80f*@31y4+@ic4^R_9jG4jyzP&$WChB1aPhSP@9hBbyXhF6AGhB<~fhFgbPhkGL- zhJyBa_B;2z_doWD?m+Gw+GKnw{YdeV;vxy6^1|~XM~8Z&@*?u048li;{lg6+4I(Ip z>V_SL9EOwj1%_{iRQ6x?dxr>y7Wd;0UG76(lU^HNqh6~##Cc(O@`G9uXg#pq`lCE} z&VzGqk!izD)=tz1OpPiPiDi+@Az4CbLX?d%8S(r`4HoR5xnMRyD4fjrp<9!9+fojb zLJrwC8LkPV@}oX8vPpkXF~a5c;fxy<$)}KOk!#T>hS7|x+9bP_bSbw<>CrVJtA>`1 zv)aVFw7S&0RCH;!iMJzp?(KXOdK5OpEixf)OA(Ob6skzDB;>B7J}e{5N0!3>3;RWI1}?p`lR7(PuEbu0LY2)X~yq zxh^jstxVNA_GM2{TRd%IuXMdNyFS6iz+`6XXk*ZKxZSz>zdt<5q|$d9y&0%)dMr+; zRc-v;?0QbsS=n;?z;mzNaa`GY+4SBS`5;(b`rZ{m5V$`*9&O?6am~6NN>Z?Mzd1eK ze0!|5=y1DP>vZ^OoOKoW8Hi%&{+y_)>;4|ea_IhA0fE@dao_R&nS=1>`P`F`i1qs% zFT9?D@caUEu(oiQ2p% zeyeW(+F@X}+hH|-+~V~?(~C25P&Q1vRfV%r*QH-4q5d^-V`(yRcMD;UtXtmUqo{n{ z!EeF^bsFfgkmF3?H96`-VfXog%$9*j;7fan0DGTLk1~2XG`CaWcv!&+yGVykAz2eB zit-A_M|kK=LDdO7Gy}!c*k}er>^3l+f}#mM=8`4=;kq@QjErm2UCN!yCZfO$UiN^N zAuqv54fc)tgOMpl7F>0Ld$C*V)}~ni7aG-jS9gUg$OY*{Zjrub!%^<&G>Ut#gdZgoN)_nNzqA707wx;?y zImdoHa~4!qii6+GtFXEXlY*Pg!RYWx({z>KifD7Y(Yf)zpN^A#TVF6m(I)H`We#et zyG~!3C4{EnNt)#JHnFVINFZ8@ORgD%Utv2*C^d;51HYgw9B|4wd5R*-p0!gMWA-Tw z1>26KvDm0$M!8~45l$p6t8gQ1U#Uqbk7|YE+BVnk(Ck9zt8>RY$#C$IHvEQ`^pX7U zHv-wgoM;&=%1Y=j3rY5I;FyV736k^e z@XU%9=RehHTy8Vzd8YdRc&{ZZ8)$`c&Epv=C{Vd9LjPU3a)8R~*G_%Y@fxPtXG(Ox zmoaG5Nm~6d01y#Z1I(MUL_dWZ7gzLo>3TdSs+HRTHNy3FcaK111|MVF>>dCRLdfx*e@2%Qr)6rD0|v`QPb{WVHuYR)K@j`)U) zInPm4ntsR0V`<{E}riQPFX>4;?&O?KMKciDzJ zY{(H5#Wk~`tH0u{A8j>~uj~DG!ICcwjg}Ak8t6uph1BfY8raGKH!1CwX)9ylq+QH) zK>RBL1q{~8h}3~Z$qL!puM8S_IgDpD$n4HisDhDfCHT!oNJ1-un2(nioOT5n?JsAc z33~C(fY(o?d#!+nrNv*kXgEVV{_ki3jFc_RHxvZMVEe=f=|r}#IplKt3vcYH$sHVr zIc?9wb1au4co5NcW;Dr-d>@zyx{yo9Mk3xJ`)_Rjc*YyIQJ3K{ZND&~DbZiZ^H|xa zE)76Yshg2n8GFE4YvmwfZaV=2f;A)Y3iBbe)}I#`Zb=D4mW=HRxGCMVx>31$yg*_N zKDYV^fh%EO2)Ez^G-xKoK)Z(!O-Kn{QrL6I*55a1*Sof@jrxvV^>~?ku&&_XW^N}o zA@!^0BP#wJz&Lh#7W-qeV+yF?BH+McMfCsdCO?q6=R#F+Ix67w7DH>TMCjH9LryN2 zR~xWB7eX%bmHxd9M0O%Nms?2)yp zocH{xt}kI0*?(@NJ~CzF*=qm|$zv6%D^>}Xz!CTwd^wBzH()T5bcAOZrU63x2u*vS ze?0Rf=pQf`#4Znf)4v5b^CDBdkH1o}{8aiKXkP31FRw}=c$`qZF?`eZOx~CtIo;T^ z{9v=<3x6~CA~7rOlB^FmF8DnKNPqFR+?6K6CSuxw$N6yN7|Q=Z zw-AX8Tj@V&7bPL+Mhlcu%@+le{twd11tXx7{{#B|$D03w(jR0p%Kt#O5R^<>@jqx2 zg_2Ir7lld#6b0jvT~ZpIAiE|1uUvN^_P^XE0ycR}@ITl^#-dCJ{15ucm=%dZ$e7!L zqmh}V$0&(hps>hHljD_ykN-~%B8{my5P%F$ZzKryKl45^*iC^NNZEZ%Xt?vQ5gpV`Tee$N!n@4#fPIyF|DpTT1)~C&*Wn^RfTI z4B5KpZVNGE#1B8^kIl*)j${Qmhek&PeAGb9?OoW@3tN#rt9hjVCO59R#H5h=m#|6q(22bb{xm&XK%*mw5r`0Xht@o9! zy-TH0SZUMW|3Re7Si;I_&if*Z^Ew$_ZO#LahKbJ8d{d#JcRJ;Q$O zsb+j3fpcI%?D=gD(HnZ-0m+TfR$MbSJ}5pSKGaYIaNA*e$k04WrFShNDGVSUvxN*d zBib0)N0d-YVWRKlZ=?#@Pe4-ihHuB2_l|rXCa}^X)ha~j)xhy<0!3dT* zJgPQ1G}J^5zeZ=%BO5EI=PB2z7?j|b;TOefj8;HYm@JFSm5<~@Eu6A($A}j#gkx6% z#nexFPvs8uurgP|!d8erqjK5hUyk<}3`~NNVMC_{@yJq^4lj-*7`wZZq0T033CKm= z5Oedo$=n@;^E6>bmc_Zm$x>Y_dzW>%C`WFcCss+jBRb1q{p~obqz7J<=?qU(+rpf_ zZsmd8Y-9K0Kt%RDSZ`ovl%SnrISGk{!%|aq43r4Dmp${&_(-b1h8td#-qJCd+{}o+ z9Z5y&bTDk)*k~_$LIvRu&)BwhV zjKZbSB`b}EK^2qMZY<&P~B=#&0EAovxxa2pBxG zk8wwDOB=!%;|cBd2t3BndH}rs!MioydtAU{SkD2&8@wEZKEMMr%kgY4c>@{I-r`ZW zx(4H(G|Ne8sK38wn9H8cd|1_dv#s`FSI^~(i&Hx_b}~@z5CJ)uxCNecqJWz-g4Gb! znH&+r#2#%3*ybCt)<^D^0BryRzU8iqBRGK&9+6uC9N7`a7q``s;4Gg3eeJpr&U@0CaN*n+QV0ZeYh#88!kS{V&8 zQHK=ggI#fVxmH+pR_e*EsTGv4P_s8030&{WAl>3iV7L}Pd@$=RVOf~Gq@~V0+7gU> zPz~b!ySBqEV1W?BYIy_jk-|l|wzyhdIIr7XSzU2aS8p|A(a>eBsjnyWXa%GuoA(mM z+fs>`6lvRbLGTzGzx7>3OG;76Sdnk)_p-6y;dHM6rlsqqyFSo+MSPy7cTJwzN}Jw8 zw9$~ZS0}{clb;1)!$t|kWrtu;J$i40MvuP!G&9Nejx$K%j+K%ekd}Yka}L zU51>j@6{a(^SOE0#+MAu8g^YUx>|I^xg?`@C|L;^&EvEzqS{92$`d9|?jO7bJx>7{`-goM+96(1nHVos-t*`13t5B~ z7Q9HFo>Qn{T=c!X{3Z6u4^*Sy-UIGra(ioEZQo=;EtmJ^Ya8R%`U`eG!S~RUN~Pmy z6K)%#TOknt2`dymBlut{u($8wmfHv6r%uIb8RrdSd6C*nuuWl=s*3^n&Z)_rcXdvj z?w0a$e`f)Y5cxSzNP3KS;UC%cl4P?a0~J&3TsMQvDcf*Dc)0}jV(d3DBb~7H=+QY6uc39GHmOV&1vAH z^C{UOrCsD!&c~O<=kAgF*mdAL@&P|0kNBJiDP}p3!$^%EaJ`6cp-$!~HO^c6?c zIL5{)&7viCO{$QX_K(4`$tJsX$HfzYaa;ml4|da*zsYysKZRHw#e5`rXZi~Lrj!)U zW)#*Wmt+;FB%fal#jxN5KLq_Nm!F=JFhj7Rj`-#LR2wFT1p)@9wDu1C=7L6Pi zbtY@iO88@7Ahrp86wM(IKAL6$FEk=-Y(SL>4RlJJq;r0OBjBU;}3 zAUfl!(D9NuPw-MzMJtI^DycfIXesAcsahggB0FC`e_0l=#Fm1U5}!W$Hb7mJ)C~$c zRkub>OH9&4iqeQQ%=UzSH^Q!sBp_6{%{Ta?$#ae_PcqC%UeOR`{~QRq2D5f(rde4C zoclgxGA9(fV6q?eh|DQUEm{1UuE@|WR0Du(oagR{yCyp@q?j&YITHsObK1O*HR&>d zaZh|J#%f7OGG%SgJ&Z58<&QxuO+bsk@C}a?l_8H;3Bc}yGe*;W7~QEzQgTF<_rOO4 z85S_{JO2ZBI#02ya?hDfl=z}EWe;pl0%4w!#kB1Jn))>1ltVZCgc*TVtnT{Dtzk#L z!bR0&_+l6J6Plu!4^!!YG0E@DDFPup&)cQTs%^kAr5|0_oz0P|HvIsT?4y24TH`=^ z#fW}IxI>W`Lx1q4=9Wk+=;C%k zAot{^n^$cP&H#m_K0#I&fuba2f!nI8NySUuONR}%rW^QrF#H>~NGpt`JlK8L<~74| zjw2x+TC4mD;s}t~>kTWS2d_>%_(j69z>_ zo~=X{Ti5RwIX}~w;LYN7%TSDBQ^{yUG04GrL&%Kmuu)++n#*qZ_gp+Dc0gcmUB6a? z`Zxrb8Rg?0t2US#*9bHj;2(0h6_R0%j0)*c(2WufG?Ku14>_T>3k~;26O+tWp0e1B z+*3k14M6aM$?&)J;2I4sEgq~~jB(}wH#liL{wgDq`&GaB2LkZ)17Cgz9B?83dbShv z8i}Af{^K&Yi)MfWou73G@>R_bX*#zEA;iWHnG0=*c@h*495C1aYftGHQqkJZWIH6r z9;vER^__qy2~&zEn<%fcBgq zI)}``6vQ=rt8(4GDHXcdjxu{BjK2_y?*lst2-Os^eI+Cmj-&nkvbrYFmN;zxj1$Xq zdj}dZRF8bQ_nY+8(wI?GFckJM7WIUlwahT>2xV4idnjp;In>0!9l$WkYe4-oq3I>R z9^`c{d2Fxfjj++oRn(vwbXQ<*&}I!`VNr4U+P7fj9l-{*amxgnIBaSKpw|;Xb}r`q z;!T&x5pGs3%BcE~hV0Q5D;Jjm%~7YRYz`jFX9q4<(^g6*OHoQx#m?cQcB_tsLqbH$ zDrr;_QjEAPlX@mh$&@aX6fEM%D9n%Iz63%a=q1$=2VD?DxzvtIHMLB$s&GS)`-Q~S z)F4_s_D>V{Ycb}t!l(c29_-j92|5StN6_UM+xfwnA{Q14FD2KTo{Y@Bgq79o=!u~Q z*VoOY!={ZN(bx zR<7FU49;r-~|;kGt{x$Nfq{5LWasZPMAb zIcSUupGh7$?PS&(bM%8%fpwVQ=C2Egq(kMVhOy0}F%5<{oMgyINu>*+>p{Ye!ci1q z35?!tTr|~Duq_l4#b7W2+cDWO%Tfg_6=JDi+0MG+xG_<()+-5yIbvxAD8Ka{kL52C z<$AOj(mfL8GUc`%-w%PW)y8<-xl2BylDm?Krbc0uo$I|c!IP8jofGZglp^k&pjdfm zSS^MewOFMh{*{%EBB!@gNL*{e_mr$cSV-?-sW&WNsUNiDz>p}WP{wDVrkB*F@gT$S3WLNCh z>@aE4rnQfOq9P);fkHxvc82)y;Bul1Zfr&})hajvQBm8rae#kBiTV)x2z^wFFb%}V=ap_Fp%HB5!W*XVy& zP%1l8~X}I~)7vRXX(Njx_us!!OVlaQrH_54!wjo;k=ywEt6bE5APCCN6dlozowDcg!J9nGf= z%L4G2e6S1gkhD-e&5T_SH;1y29pbu3tX-C_Pc@44qyAQ@2JRtMj)jvh$KN)Li9M2c zXgxKZqosVeS0li&vJ<&zN?IHK;WM*T|HEg`$WLt2h>h%pY$>cntCNEz)UTB`gdEN` z)M4TugJxo)GVTa4q1zCkoC2(y2;WyfTY~UUDQW$sU%ziE{|j-hthHo}GrBs(UE<@+LJCE4LM@=Y+%gh#Qpk}1lOca8|;veGP5ank}7%yEb z1y3@J#J!%-NOfNvN-D$1rnRu)9oBgNV;4!mbY{G2@-5G$mwONPVmKj{8Z&Ppj7ZL- z3w6rTf|)2DjXC2(fk`5({-~BHjYg1`p2KD=3Sav|+wuyfL%uf>zL%lNMx4djO?896 zx}ruay1{Dzfw?qb7MCp0;D7Af+Up}GC35%Mc{;of#aSArmJ-(tKdDf%D zzUMA7JzG*vNollF%O=Xi)_E#f~nu3+2X zau8VjNYnsZPQ;?p{M=l8iZyNcvs&FO$^>{c*h~T=ayW33y}z|1~tF%8ysl{5J@N;-wpQ|*Mj zLu*3YH^tergs)gr`BzOsHA|04*%K?d3?Kr$PR}~@%d)=smazy*j&;DIF@@CGa})-D zwtY+E3J_*4r>Xm$hUSvPA>}=j)3Nwaa-uOyH7lW!VplvKCvU&mWVaIZ(uijcKEsD_ zz>EBCpQntii>{5XgmMY(DyHk8kL!uiE%Ot(hxV`oOOUQmE^jFxoCEnIhQ`ipYx;Ur zkS2keG5JW}?e*JIUNlxqlavlxpK1%Wi`!Y|)C8$h#9hoX$m3JYg6Z>jITMYg2VqqN zZt}2`qon#SKJor|`7k~<@-H`2FrNGVTUX2wZ^+y#fE0TLR27-JB!a1EzN<4LRAi*O zPE3MQ3BO8%-bvYg1ci!xo~+RZ+%-Fu?TVDprA)u2ikXR#5gW0Du@Z!(RJ52=`bk#) z#CYvR+6+SlJjF=zqC?)Tj|j1Rk7Kr97MI<-oz9CxS0&Nc8iph>Qbgt|zKK&(T#JnG z1{P5$Zx;?MCWjbM(e?Z`PFp`f%uBXaqmu(bBVELlIC)x9NxXF^qyy|;;U?By=`HsY zcn$c*c$y6iWa?+ydp=#QcJx-1RCj!J*@GAeakS47^2CM6KwNBglbtRg6G9VW9mADg zjk_Q&Y8<;&Y$#I)36n6wg?MtAb5-bDlkW{fdaAy zBtAOkDJ?=w<1aB=85eQ5rXu#@)f>v*Z+*r|nx(%=dq}lhZ5`$q!hg5ogX&@NUrzIgb zX3Po!!~vX;CQJqD8K+pr_)f;1Cs{apL$BLX$d%j{7^yrX)6&SbIi9Ze^h7F)H@>%L z8~o(hiTj?*jX&>fX$k8Wsj&n*Q-9~LJVq!7agl-Wo|6rYiHgP?xi=YOqha2HKc$e@BJO zjA>igb1aHKP_cZ?M7JWF7>jbc)7`g(C8_L@@-8H&T};x*QBCZjROBTi3#0(xo)62F zh{$u3*k~zCq{(S3U<>!s^%`E0m6#H3b((kTot&#rWIQLx#GVHE+H7)pdd?a~)LL^p zW8l4Q)!q8fvu1pwl1B3PTk%NOrmg4g=L*d|i8^>YTSR5jthUzeuY|7LwZ{u<=IEDO zOhQP@mprbJ{{*h?5#oxBJ5Lo+OO>Fzs>WPI?>JXOX&D!bXK3v3-X;f7fck~AQ}W2D z{A+4kcaOREks=<}6McNl?$(v%{Yd!XA7RUG!6Ci$hFnyjPB>*Ar*oBoD7hgWCsvb` zDRHZgSH$W-7L>zxFbmWq?OMUcuQ znNiN6g61(wQeVF#x+>;N6CesYQYK;oZ9WNht&ZQc>tn>1kCLp!!}Df5i3j9n4|Mw& zf?~>J(<6d@MyI$db4h4Hv~6Nv!+_RLJ(vw7E1_sB&&4lfzD(TY*cJrkI>yp!&F|HR zN8q@})@j{Q9KTeggWL>ffbYXQr+qcljiqJg4f?@j7?c?7&2uY9E-f$*P60BI%#J%x zw<_{r{tgndnKQSl+xFnTEdCczVT|@xI;zOoKmeX7BO?-wYE-(RzT3Z0XZ8gep;`hg>4n5URiX z>DEYR;$I4zR4pyH^8X|40=$sBENDSX}}~jZ+h+&ZqNF4{u?i{mr(@ zp_Bre7>?mDGNG&_&2nn!HeN?(#{^g1Y;JlYDkh0MyXn};*jahSs^7}tf34wV7eSX%eC*H)sH8!wQcy@VW6aKxkO2wG!qm5hL!5)e@kxl@B;lql z9@53e^Sc3%JZ^YQo2H|Fa&DJPrn58S%=J9+EKu7kSL*?DUAz6Sf(SmHG>*yJ)xp^YyY3!zBOc94~4GW7qz88skpw7tOW*JJv+Vs>;4b56e8e1*u19IVTYUeW<|0<$pz!g2dn#%HxS4UMghe^i zg$JXDP^DvrXf2nyo42QM@nB2ck~T@&gYMn85hp#HWnY;Fn5#m zdA_yPf$Xm3A*3QdbAHGcFmT!`|8)h&5l_i`33m;H4}kLY{uq0*x85(vi~%C9mtLW62$E-MG=!rQa!Ys z5Q#7FcQ3pC#J7P|HhMp0chwwndPyCsHkf&eR>AUZBDF{XTYR~G>72GD%x2*n_Z=8N za}kK4yke>FMFIDzdUqDPbG;sq9VOnJzXj%ID9l1l6lVNGW0H`%hjRaB*m? zqGgwF=rL6j==Mqvc%--xg=9&ul;Zc2h%e`Eup>4cLb95cTNC|yfQL(qu^P^YcX{#MDRQ8qyq zlp1BLv-GBu)3Ck|u;wmeSFdDIqiKUSo=MBwp#8jC=K1?J&Gz!IpBNBaJ=DcLzpY4o zd|nj!em~9sb7lO0PfuFnwJztOX0BBRE&^#1SZmkK5yLXq95>}|6sr_u$l)LSM*J8g zEpB&En>zMCX-c>_P_rzZMaYI9^(x|0E)a{K=Km3A z^n0KT5%+V!E+gUkVGS|=OmpVW91dvLDqsSia^b?8e9EYvlO@Qjqs#Vh7H0Xr;L7PU z?(35JZ`$3FJ7aD1Gg`RFcWQ$(XXL|Er9ewymPvbY_B+$>dF3%Ht#qgB@$2>Jzwuf4 z@?pkJ8!GN7%*L6ITn`&ZA{&M+uMiAgE@7$EW{MeCLSDSK82Cg#Lu4XHH-R8l8L-0h zoyJ%4Vog5CYFlySA2vU_810IJ_%VV5UUZuSWCfivZu6j2h<>M{N zB?M<}FzWr=NCxJlcF%;Vs+E$2^#M!Wc@2GAf7aV5{cNev)sf1@>d==ZhPqvS+K2V; zz-txf^V!aXMA94NSF`iow+^5?Hx1*i$0m6`2igJTW|E_&w%DTGMwaK$U*{`&T$JyW#Z4ZJzv`nO7Z*iPqZxIU7mAk# z<0jhb``ZqU)cnu7`WHzC!F2S)VbNvizBf?`;65|MEg_N3zI=~0W)Lqa4BV#ngVJW- z^w-VWGmB#@84TDL?Zpn{Uqu_sI9kjt22~!Vu{MLoizlt-W@IbR=Srh8zVjQ-`aM?s z4t^;Um@Xa&{}!VLV!Q|i1K4)ZLf=T~W~n}|r+dI`OWkP67}bM$8QIjYW7+b($f+q= z+Fo9&7S%G~ziA{zcw{ z>_cNI*{i?c<(&~;zoqem8uvtQcC~W%nz3_n@?*9A@@8I}2$v%rdoiijqi?aBODF=B zmPV8i$AOfW=R1DzD_zCxCnU&bX^d|2nI6Ny;b;4#0XgdF>%QDyx8=R6 ztwit3t-92&taeHNy425U!(IN>M26|=+~eoF%^GuVhxq#EZ8PhuF+u?2#fhD;j$yXR zIjZs7#_Qx@vC9AV3f0<%h5t^awd#Hm!z4mkFy~`@`sej_f%l6C5iB6LIQ5Tb=#`DXIj=+2M zvO9O}q3^dF-Ur_?=j`9*pOh9~_3tZ4kXdvPHfL&zWAsg$FFt@CIf2b$z*Qm%5sb5= zMe-dNtc;b%$(ZoiFSprTpkZhW@D}*NRN9c4z`aWZ3gYMh+pauD!|s(^7b%k1wAdT5 zPrl@v8xRd2vRR+rRt3h&fVvs6Zl}EsFg@uOGj6#fto7C~dF2^xxCyG<8igJC^&uVI zO_0UgwN?i}xp6CvoAK+-qx{L|W!-t%L1TyoO9mr_Xqcn*k!!+1w92|Rfl(QaxrjnvFI8i-J6;^T&nbr zO`P9wD%=;cxqlTmCag1$pq?l>Y?sQTS$$05Qyy$=jw))~+32S4RV^ADtqK~geMEet z8e?4-nl$a*@)7@bRvuAXG}_qvR%Y)UIxkJ6j2$@|ndpfmsnlB4xGZ&M>KsmBOC%LF zfPrP|=%!3N%fuAvkF*Rsi!@&jC}C%1tIbGTxVVU@G+vh1CE~66+nB{%Dt9<9Ptw&q z$$KVLm4FZDXl^JD7h~yN^f4)J2TuHxph4%l(9xAYtQ9S6wz^Zy$SO7}XdDk*w6SvL z);Ab3D>7cK0f9U+-D$9u3 zx2<$pc8M(weOs*od8)G=4m@u+SEC=@1pnJpJ`%!G?ev$uPz?L9y*5OCrmy=l-ew-x zIS0i~z@pn0clKD@S+oz1(Hl`JsiMg^XF{tR`No6RlTDfslRfg--Doz&mIkty` z$X7l0I!Ch;T(eq{MIj>0+Ef7v%TYgyUWy#tl#mFg47N!IXU?L|;hEg&7q`Ts5BrHg zq;K}LF02TJIBp!ZX%3Sdy;IXz#QkH${o(m}fi9bcU89QltHDClQ*YYE!1j_2$1CcdvuxeewGVF-m&_9^!7Jo8)Cl!#9fUu*254 zn7nWni zG#Bi1<@NnMyLRE(#WTXqbPmnPoE!W)YhB{wgm}B+>xmVM|Gijs?C2)_C!*ZEG99;* zol5UNi8vR1;Z@GF!wK8gl$c%BHqHq8^6bo6Q{J^(LnT&o$DVWDr}BOFOJln)ON2kW zg~lhnnXA0&=;gvx@A>^VkH~agyDI)}+Uh&n3sSh3D$5+dXf(_4%#3OJ%TG4Dr#GkjJXP*fD%?55XR-AwGz%-$!${k^K1vwAG zC*QwZ*TA;ORYto{@OZcKk%^09_qgbYx${n#Q|A~hX6)0J`1e56efQGO?0t?-4~!M% z`lFcFUISip`RhMJAIpWkm`4PGkI{jiTY~SP5@Q3@yXj*mp3gdLz;o<<9hdxWXU{pu zV%Zfhq-sCeV6StVdC9g*l48d#-YMqWM`u~jP;8vP#DD&TP0MPt)^gfZf1mtH;lsg* zw~6!4#WC6+IvJtHwXT;-{Lp@>>f@U<-c7SSaN|~DQs~jjZ-SePKXAvt{aVsuJ#B9# z@4Xws@wKM0Pu{(`!{3v@d_FR^nAiEumrJ{^*Rjl7m~;2l4D-ZD_p|XwpFe64Khd-- zaB|ztuUs|x{s&gs?3)$d{v(Sm?EmR}&r+EyeKQ|4yJNtP2; zv?iRAy%Z)S+adL3Ps#M~^|v#QGS7T_GS&OmC$Ae#Yfig!nTgy<4Cs2zBC%wm>PgkJ z#rk@}6Vjs`tghtle)Xm{Mkh|#d{$Ye&K2@jmqJr#WC%QM5LwL7$ie`4Y!QkW z&@CoL=;j$)8kiYir~{q6gQC~a)X)TYND-Qtu@So849$V+(EJMwD>G9xJAh&q7U=#3 znvU)dkcDPw<{21RnpvQQJy6UX(>&nOOlbBR7#bLuqMK)EUO2*qzej;R^&Y#|VU z#W_$f@X#eRF(X4W4D*bP3^439v9Pd2_pgbiu@QP4nOd5o$2l;93@p*&7np{O(BlZ` zK41)^xzErL)4xW*gQn2UGXkbHbhnsTS{R`F*VMqk7+sy2fw2L)y=Dd`mPqPKiV}fm xO%;I-2@B4wN(E&>{h<8(65#nx;KQeYXK@vmBo>u`b0{zo0W%?&s;aBM8vrN_85{rr literal 0 HcmV?d00001 diff --git a/uncore/doc/TileLink3.0Specification.pdf b/uncore/doc/TileLink3.0Specification.pdf deleted file mode 100644 index 363f9b03d037b340981ea2f27c752bbd2741a20c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123869 zcmb5U1ymg0(&!u9Ew~Tv4uiY9LkJLp+u%cR3GS}Jf;$9vcb7nL2<}cG_#6J;Ip00^ z-uK?Rcdc1_Pghsfu3c5#J+t~pqb4Q8!okXoOtX1-{1cgj2fz++FtI@v5@OTxa5M+7 zNg9KVK@OJ4Y-+}q<}LsZ$axK95fNl_d$Tt^4&eW4@TT{s?QCuj{+BwNhPjJ_tFtM@ z9^zNc*}+uP9P9$nXM-#bVAC>p2LssT?I3m%|C%NLHOm_!vq`&yWi`RZVDmRyS#|&i z@4rJ}lW~A70g=i;q@!0@$=Q?pv-=H}+cYU$u$ z2{LCjb+BV|bq0w#3BR4A;S#6el!1J}kmb#tSxrq@Eg@+zwPH0lbA^Z<%`99Xld`;| zhAKC=k%GC4i@A}6xhpH=1g(p)uoSI}J=jy2gVxo>QdnF@-9}PHR$Ix<(ZmGgqNU=j z;r5Z!MZ);Q7k(o(NfT{#hPSGUh(H3DgrpTxJQo1x|61*T1pjIe8EX(2k`6W*5F{um zb5jR1NZFOl?JdDp0Dg8(4u~&+3)tD**bdn}GuiOC%zVV|Wkc2J6JW))OAOKMgz!~` zU>GK75z`Sc`6zi)`fIsdVR_cN?NZ4lrbvOtx)faf7x!RFRQf?S=kQqc^>*9O?>8L_ z?Mu(b>+z7E_cMm?-=k#X4L^#-zrmufFZ@R})BMqX8CeAy`MzF;ZuII#7)947rTpnnWwWfqK8sR7m6&$0+9PKA|;;ew6`xSWO_iEFSWXVO!g4RToLmf9{IN zB?+y77HW2x3AokQID-Qa#%`_iB96XLXxK%g=7Hk;Y1@CcnIPd2JcAxbFJ|I1kM{*! z{##w)o=4^w{ZON-+CbtK$QV7%awgNzX@b<&KIeL1k&wA~(H>OJq#)Dg8zE zm590ID70xhpl+y@1CD(qVVOI8sMHRiU?acI=;VtKuc3cro0Ro z^_Vg3mqhuIm1nN9Zy-}nY9Hd``~IyBDdmGdJp=9GQRxXFAwSt2OL0XkVzuNwn@z?e zqsFv+4G=HO5|_cyfv`dKCS}GFSH2Z%SLUP4p%im|LM!jj@TN;L1SiX|8nCOAqT&=990KkGig=SkJi$e4!GcPaQv zw47poWSRlu;!g!uX#Ii#+#I7}Gwk&{)kGw8A`yy+`^HD0aIS_(;_4V!?~&pct^oG- z88w<{DE+c$#PK7TG65f(lJRrfbL4!JOQ&VbQA5F{V!8XJGaaIMtIBb$Gqb!zwZIAM zM_UHC`r0&=Mv|%)w1TH{oGvK~w9+m#DPLr;%hfI<5fo=GhE>F2PtLNRB5YwiSg}N8 zo*A27uK&od;@vYKn_t(4G-PP5i4}+q$%MJpw)0}jTRuoIJ)6SZOHsVMki>Dw;fVKx zYE2qxEQ!PB6cT|)cB1j$U8;=01*il@@B3m#RcDrfbhe*7AEDHbLeFGtZy>oiP`Wc+ zOI4GiwNFWM!!*wlU&p{~7z=7oC0$VUWX}dD)W9y66g5{<{sAnxn^5hp$B|LzW=-bi zBDiEc%3S&SN9&GIFN!7n@Y!6yC{9&U6yANLYMP-DPJ3iU)IP}28k%j2kZyACO;Ng> z?`yj9AUV*#FBqeC)x?OSsk`yF!;JPlgj!k6{4)zv^qe9=D|z`&{Z2WGnqLjKNcx+J z{@o843GQ+8MpAs&K$R_z%*pWKPjuO-J6U1g?VbU1qMiJsTGd8oJ{O!D20kA8JC>*e z+{z)d^IZfs9~n8OB6ELg!8I?W>o-2@u=R~XS?_9`G`1}LEL^n9$CV*t`m%yf@?$Ah zyLQ pO-J;f%oH7lIvKd%zVF`ps9`E2dM)y7+}7a7o6ZImrk=cR_RNbe3eX)`+06 zj{NAbtuyW)mN=ZynjF1#5~RHssNBvVyd9#ed{Dh2`An{_jZ_i=`s^sSn}CY4p!NJI z7bYv}z=~CzR)lteKn{3EFR^cr#43NaoOadxp7ukh*%HpXmK;BH$-1Qm-mANmbC|rd z3q0MwM^L4s>uF5h4PQ$z@KfGpIjtN7)Th$mrY9DpEbQ;&736hKzl^w%aC6 z%*oqa=^Z1d?1>Cnl4>XG(stVF1XUtP!681HLz`P)uhY@lHUyc&+|!Xi>iHh+#5R=0 zxO%`XMOAck1b~(mL6srd9ombZ3l#-ce$hWFgXJw(ih_GQ0}v?_l(qf#;A!`b^>>QH z{-$Zo))kNv_qr^hSH$;%E|AGE?XIR25$(#$?{<4F2;aBGPRUKhq4v!cdmnO`4mRiD7NMCE2kxcHcpOg!8&J$jHTKXgfH zb^!!neu)4C2H5dzBP#_JKPe3A6%H1fHEIVGaamGCV8hE-3`UY}efLU-p_!xvAr769 zT75dI@j=n!g2F$Ffr zbm!K3VCA<>u?m0dKiCs<%Ip9cBlL8~*YpafbnR)*OX$h#s~0EW^SDWTj4ncXF=^(V zJZg!V$Jd3yL)1;@f6?c2>Ey@w zIwj>>l=9fePGabHra#4>S%!x1;B+A0*RIPrwSN>qr-tP%mN{~#-5)13r{rjTG^13^ znM{?V#W-wE>yVbS8|Mh2LT;HAftE5+BwGf6JGc_nvArhOWp?top(y#KJ{l*nlT^m3 zpGfivf*GihcO2;~^WUo#X$sd+FVUGFmbSb{TKR}!&M^L%@Ltc_T7Xh_xO1WsvL4** zti+`wg>`A^`GEqdOe)T%bAa6F3~qw_zGwL~DJN`LSXv61{BF?}R;oRs2Fbd%mYjj7 z!1<@}v4Ut7OzAaw;)ZJp?1O~s=Wn?lwQ8nbg`A{#+N}T$h8g`J*02a7Zhie(wCiuy z?z~urlhu@BJ2C~YUa{zQwuL@dPS_)C1@CjVJwk`65_JHP!U9)XBRcdnZL;tgvVU01 zk%2h?8!8hnhTTKVDy)ocB+9APVJoHU=Bb^^#e)D$V)=MmBzP~m$y{Q>X#S!jlGb=t zK^4rA0UfLSO$z;Q%Q7Y!he8eRHdWfa)P*N;UE(-aydCj}F!g=FSHjrb^x&dg<_%vM zb$io~ozF1u3msIK+k2Y)#FCm)j+|e4F_T$FPhK1_M~$16PZ0F2Nm5t(?eR3#Fj|yO zEOBuw!kBQ{H?MLn0yR}88FzdevGEOI@N}QY;qyTEjaw1KVG1t$$$)x{EQu~5=)@N} z57yXjGqDJlWS&K^d^k`dsYe8h`1*2HjINU~?IE|N6fFx1WNb@Kg%GG`U=P720-s%&=m&O7yzZmQv#`>@Za zq1hNmifwqmM9MVm;LYFTP1f-fi+_ltsOyST+VB<&85eP3^`Mynm71B7soP{QwV>PpVg~Y5G7AN?Y9NLB zSCk!io1b77NairwCRTJWslw5CTLAC!+Yr~D&`2?EU3DP6>){PKfX;ixO_1Sq3FCLEJgytZQB2dPk_B2%a*k9TuSVTche74U0(K|{VW zHKNIc55`nV7|~_3+mF9-+r%FKAf&ROl1Vo6leEF;EM??CAdLCC9f#(bPH*VG`4Wfm z`u6iUSCgX`JFSy`yD-7eMvxQc@I8lum*`A7To@mC>_WJGs-efPV!o~NH}>={KWS|} z(WgRfRj3+OOi!1CDG~AXpPMenj)5`Na(K|dSrlj7kv#c>$(?jPyZ89AqH=fT=T0|q zJmChfW*^G_UaE#jPYsC@ri_YzjKucE2y;gMoipwXy2<6DOyoNwX26Hn0d{U0UjlnU zEQscd&2gACjj^FLmBT&@F1V+C_4A)aEeN$du8d?|F}{hks7~H6vuO*E|cm z$9P~v5X$U*#A&eUl_-6oI!rTc$dah6pZHX=tGrKHrh*T&zJTSeQU3Og z3Mt%f7PBKd&X(Rf71pEf`|rx&!*J1JDo;0SMcy&y0c^i32Uf*ti|ck4aboEm`k4WSdv;;db)XyO0Rs4z2 zlBkN2>v<0A@SM#OB%Glk&(uLes-3=~Cu6-3J-~C}MtF+Fbh#3lGIDlOOKh0kt~G=qUw0Q97z1_2Y8Z?L77o_6wJRbqqy zvwN}Nyrrz_lHQ$hEvF1HOXrk)mX3#hnt#Pa>Jh6cE}w_GTez7z)j^@0`ai-kH=NL2HthIvAc$+y$y1jQCDKa&C5w z^B1OdPV-`Dv8@c4#WSv;?SQ#aK5;M_j$IJ!*Jyi*fQrz~Togs@WZ4xN@+;caiR}sv zq*S6ZD%8@RF8Mw|u8Xm;4x)FHdS4|4yQU36Ud zPq<)>Nk**6NIDe4sO)U|9n^(F_c9p0_@!@jDUJeLEfE43s1ul>R4#-%JD1{2vgNrd zWi5~0ETu~WcwcCdYe{etz7%~v^tgBYti3)xQxF2xx|4dU_(7lpuSm zOFVOtV?IIx6XX#X88y!^Z-za#=hyw6ZAgY(p{=(i;tY<#7Ig@o(y8Z0I%?BVMa%d) z)|6_VTONF9ZY1RzsFdAXBQCeFHJJedjE1&c(N4yXd|;qB89ow+3ZHu?H7~?DXc(j( z8HhrsqH!)5M&u?=C-6=y79LsO(OPI<-6PS7@VcntGu~WHbhbmnE)k=!rtzt+s)pJf zP27B;@sC?4pQYh1Nt>DE=?WSCAQyMy9vfu_Pv#)N4^sPW=yIMRpL|tU!^AUGgj7@} z>c%a04!UtvVex{}$z%b9Fvg;QM9r9UwSB2DS!k(0P6pQ4`pjGvm^+rB+1wl3?DB(g zWM|420ijV$cxoe%@n?Ma5q$l2E+vhc2*#CwYYy~F9r^ebbie6m?SS#*@*hiKtZ0zp zA;n2bQV#E_e~tbsgPzd}$Is2}ytsgR)TnWgeMFDj5^*G_C60XRuV#>)v?8@Jm;I_1FECNiia@ME8Hn% z6>0aRyhwc-jvwo?fUf#M-WO3NRYs=qiS-!U9nBdX@aa31h<-`KqgK{Dd*P~k#yX)3 zAr;Ki%x+{++$^xuXZyVa!)x3<+ITmZ@D11Zah0Vd2g@0+ax3Cq3-8jSM<*(VJb4)% z06O{3Tui0w(W4k)OC=}2op~8~>&lMpwd`vZYyv+(^q9%Kco_xL45CMgDP{6&>n7Gp zR>$+$qV#u4877!X`=~2$hj?xKa}eA0{O_fHpCUiI!LH*3;S82tZ#b%Fq3AaGE2ky} zEX(0ypjBNr?Y=V{78+U7gS8tj1fGitN&8BLzJRt(UXqjz#(NbGnp}0hPP{MclLFI> z&(={nuLgSuSZ&{w>2B!M5GC7zT2S~&6Uz0Y`XZ`U=Kr|m)Ur5(Hk}Pt?dkpNG$TSs z){5I%^0v;UaAP7fdV8Z$u5tKV6gmReZy8U~xmOMv1rO{L%+K7Im+j-56+=(hb_@l~ z%14D9@@Hl6LsSXZ|NKx0os8CMb1@PgY}^9C{I30UO5_uUqF?=mSUTV=J{lF!8MoLl>WvyaWAa>W_1%55V5dB5=B@D^@rB1)fsbU=BT8TVx)+X2MEAYl59g z^$+FsdpGMiWqg9s#LCZgM;!^C^6spIJ%!fbbDZ5=}g*tn}6xp`Pl8(5+F+ zNO?NxZU?{SNLju+2zXHroTBYBGES@Z0)VqVKNk`?8xw58fdXBgxPNHZIN-uC;cHiS z8O6z34bYY+b4S<@t*+OB8q#v~lSf0j+I}#e$lThQG5g=+oeeSCK0cG;`ynTqo!pY* zH!u9B^^Kh?sx67275DzvRBsGqKx~ zBj$(mvD=%X6u+}WT4BtQ!7jwhigxE#Cl+LFHT+(ks@JHUG_dhYXbDZK74PuK#jZBS z?i%?F&>{aSwP7hB4z_9^j;_~@(~&$4i9CUK7ev365!xzIjRjw@uyLOGJahSlrOs87qdntc^S?t~$9#w8^@*A0(bQ@^Ns<)BI3$Ik*R z?{w|SF4=tTfM&(2)yF@Ymp=KvQgyG6&U*epUVQj@BG&oG>B18|f|QK$L z^|Oc|tF=-q4W%h+5Bc~Dz=VLWG%kSq=J$=eEtm zT~pBaEipHC^3%azN2{)Y1M`V74CtGO@EpujtgZEQd_}bDF_&@w+b@DO6-!yT{XH_&gsCjA zRWD&US9${lE(S`q#j$h zAR#CVuCR_$>>bl$sm$!*T+IN%6=U#zLl^! zXnfrzBgA53h08F(>OpjbWDlM%i?tJzb4Ynl5$?OqptS%Mq^fIYvF-zG!?o>q8|QN@ zoG9;DZ+|mru~JIJjO}fC-6jU2+-+j0vbIbkD$@eiH0xA|l}Vrv z)A5`c-Rhhyup`s@`Y@WrkJ^3lo1~bE&A@t2@yZ?CB+iHG4~2@*-xD*z5IAd6-(qycdC9c(C4cDa@&m+ z3>CMQDKnVIk_OnOS%&U>x>M_v&t=sv2O$WQYC@KE>22_fy0!9x4zj~lHTyLm3;F1_ zdl)C4t<``W`WsY0p)0|E z-JhXOha0E0(?0FCUur3AHTs-JUun}=Tw9s56t$^$c-mx1Za*V{$@!YQACD&t;jB>_ zKPr=HyZeC}mT_O*H&8J?E=J^>=gJ(!|F{&oZyz{EmV_Zf}g1(n?y1 zSH%=&WT;p-KrlqL5f&HeIO{ey0lIpqF)7rYu)PcX_&C1)b=l@Zd*2B;U4)){0hO10 z_rrF0>-~@ZwulkEt!(wV3H(pl-EGeVtA+!R@BG=_kHkA|zmme#istRE4(crgXbX?JDAO!Es-J6uHOh@G{UaQnW?^n}W#;!VTv)S%J^Zv@I$7 z_9bqQJo@Pp?ST&G@UN`E!r)z&E#yaF3vm2u%WZ#|-F9N*zXW@~o!zR00Xh(wV(9r~ zgK-L6m*7H(-au4p$-Qy5af;$L&-E?Z@N>W(dt~HgrrSWG0C5$7FFOp34_6j*Ge@jEn)x zzmKlqme`_3>SiX~2?+2LIuv9qDJ_260ctlzKlL#}Zc@WAKU;VW%mgrMSlS6qQA}k( zOseP8gqdWaGU~*+RyY09!A4~gx-^Q!4^YL$GA&j(m&p1A%~o0!c_ExiwM|<$)1-F7 zP^Tc+RH7C`{ZmY1Be)jB3^0uWO$)N&!(5Z9Yx0bX>z_Xl?!Rf~ytaAQ{ES0Zg5|FEBVS7AT)Pe%9KT8ZM&>Fdh4&&9r1R8`S|)cHc-QNK`z&CYN?fXafrmjNxj#*S6~Y(iVkI;pI2C)}C&tgZ z3wAuz!;W~ z3+OvwP-MwvkhM?nPiEZSn-nsG(OmUJl%!KGQ*T9Fz$^@nW(e^IJK8*TP5S~eKZWx5 zp8C?!4!P08)-DIJLpgts(=0BrTiIKx=kWBk(C^O0LOx1I(^JnHj}|Deq00EdNHB_5 zb`+8Zd{Qlrbw!A`xocQ6fgd+|`WTO&2VW>sy5jb50v8 zgZe?}f607M?t(tIVk5GKr*7V3&XcqJ5w!uH1oumrV(s+wgmn(#q$3W%S%P9D;Da=; zo=QtZMJ09QZt~{DGWcts{%`CRy556itSQyj2RFFcHXGGfvR1A@Q8$cqd87JA7g*14 zBNK~`xNk4GBX|CowbhM=J()eiQ)nWZCK$p@oX(}k3)Sp39>)%+{7%WRE(%tc`lX6V z8qG2`_{{?9U90LMWS+&>HhYFuFN>>(CRUpaU)K%tq5ObN7Zy|c-t``@Mx01J)d8!m zxoQX(t-%}$!_|>{ZjajtD=b6+72tq>KWInJ1AEdbH9juP8Xta=>0{CUoGK(QmQL8U z(_x&~s(SQo#r7JLVbI=n_oa61ZYAl4n{6=) za#T849F#R)p4tN?eKlT$c7E}5reb6Qt`g|fG0RIZ38bJ{W^QG*u>SkGd9RO^7F9XnTEqz+&_aa(&y=VLVL@6qHYApcMxZ+kMUCa5%y%W)=N@HoCi zB5VpY#|5%On)gYCIQQeki_}p;v|6Rbc0i5(GY&p(ASx5LRepd1w=ept9voYB;Rzs8 zqCPyd-OeuDK2|P5rP?IbFBv0DaGLy#o$U*+=p5e@@$8NjCE;B03M z`cI_m4W;^@+`n;F|A9ljF-;I(62>m({}5PgTF&MsuBKMzU}a4RuVoKGfvoKK{Ad86t6m!$kR?)I zxPZ;=Kh#QR(AS-mmMP1baiwDncKaQfb4H| zWz&7b=k$5l-vB&z2r$RN$_eD>F$8b{AyPhmRz7a7x43!u`B))P9S;Q3<6`CHJUUBwRI=3{5&=ivqLKxQ5wklPRd zp}Lgha&83VhSyfdt9V!^HvM z;f5^82jqpM7GlK%d<%nzog3m35-2A>H!D9U1U=;AVrOON<@*nH9xfon69+`{?`0u7 z=i_4q@z=B8h+CN)||Z__-h;Bnu~yla-U5 z7r?>J$H~gh#lZ#O=jLSPHiO7$idFT@jpX`r(Pc3 zTGGD`N?Y%4J36R43KGGH^7h3O&``Bl31a>s)YR0pK|fsm6^Uv()v zS}oJGEZ3}j2lY<40tSk6d3nvQPv%w{N{S2`Gw0*4A9K&44~3TRuGrpLIT$SeC{y;7 zx_fztVK*9$U(~D-8daL?G9;xitNc^W?I-A%eG1X~`ex#&1;(McJA@#nA4@k6 zt_z5AqOt5n9wSH6>1#_4?-oy7j;h?_fsXh@ zVb|jwZgyeljULLM%9FBSsCqD5YstKI{84uSP;l4dtoZe?dhtH32uynl?K~9K@SSFbEVrhgy zL_wT`Q|r>6Oc|lZpkkX4$;3sJyv4xhYVl-u_)D!I&>~}lM6#B@x2bW?1<)+w?z11k zttgxh1D+H1RkhS=fIFQKz&q$K)(<(t;XCFa>-DefHQ2@|5?tJyo}q{8exh6a%`Xo9 z%&)N~c%_)K+g^s#R8QEn6s-8R#_yjIW+`tM#4%Wc86Q|aNU_Xb8<+1H9~eKI@b8}+ zpCd{dm$Q@6B2j%NiZP^mLPMR!_>99q5Oj{U?yqIQOSs_0<-aM?=>QMfXx^XBZhlfK z%7J1vJuU3sWeP^BpL-?gWrC?SM5kZ)jG1lLNLflhHimc{j9`qn-y{>E930KFBgd9&sT5tZYQ4HB=1+B$OG_@J_{Xmnk$k9DhkxS6XtO5OkL3hVo zue*ico>$MEvP4B!#PP26ndAbM5G|=}TW)L!c;JT04gk>+r1P0x(hR#=t5NB*SR>~ zS1?x-*0pd{eq4t{H>hVv=JVwxYOj7t+bNWAqBE!quRk{2*q7W8F@4-&#!ZHc+}t-{ zTb2Vos$}TW;N?q_DZ^3n>G3DN*dL^J4XSlytB60bsy1&W);QNLeJp#3$chh|ZkQ>o zFsLHw_I@^p$_Co1m~;;LFb1W+ni~9Wd$nlY^!(h3-)s3fqaMm5&09%y96^%6G;v^v z=5MaQk;}bA+F{H+Ris{22Ed*q4#_la2&+%DmzxaZpqfJHdoh%cIN20S`VJlkw!KO1 z?D-}O!$g-M_Wp*7-;G1+C)ve7N4%B$m#VLsrqcA1A7GVd`H~tC{v={_Y;`Q|LrHsb z#@Y4<{s>$7cIien9%8(epN0*TOrV7yRdaw>765Orvek*KY9?O@?R822Alz?L1c^Ty z=eSdW@C-3SZ<$TEds+UnUWK{z4BYSaoo=~@;j;Q`yAv_i7}BnzFXUIJY_^;CPVl** ztVZ46E6fDcJ|o%Ben=mGLHJ7)MLwiQUR9^+C_NzZ( zN{&6pN&SLtj0Mdi-Au?~NbsvejWzHGuWbsdgGRQXvtzx2T-2zA-fd%LaJN7iIz1}| z6%t|JSjkX6^Y19yViNPcN*w}b)7{axK5j|8!ZiYOigAjjM@rg8UZxI_Uv0m&eNHPv z_J|K1b4TBfu8eHmx(YGby;^J)`V_#2lPcN+*W-RDw&cVzE}CQND;oS)zJH4Hg=@={ zV9NHxdi?rVpDn>=yzJzsc9Z?Wl*j(l+%%Ffai(Re*ID;=W>yE(K~raTGA_2*zA+9# zhx_zU{>WXx1O0=G8ve=nXOEm5;V(db0epA4OGt72O?(=hNUE2P{%c$1%fC-06%jav_Hj?TiWxIxlU*3rQ*R_B2g zhry3z{(5cj&?7FM#9f;=e~4zhDU-PL-Z(lcqHzEH|h zRVNf45ZHpx(^SveX)`iwKDMG$?*{%-dGe{@;fTk!y5+Xjbz#GC`ys=3zHW0bx@pyO zrD#ImzH58Z=+)x5vF+^TRQT@Ytn-oF-SJTpL-gt(CM#C>r9V+s_&Lfl^=@gcYoo5o zZNtUS{U!LBt@GxJ{PkwE908ro^Yr4_fyCpwvT5V1+lJR`AV#e3&1H?M$n(kdHr=$2 zty8gXAolTKjU!7-%&Pd)4ZtVbm(hR;gkz6OhO>@?w#Dt#`GIbo%un-aLpzbEbg(?! zruJK|MY-ubv}?e}f=|UF6=!NW^@_KMUP;0;*&uRrqc}Nip722_MH92FxcMj#B6}QkHh`QyAJ!^=qAaGTzg?0O}3q`!{h1Z zurG7iUY<^|uP#k$&waYhT{+AL->G`KeOl*Ycb0wUYhgxeI2BH}N$~-LH-6>^?qqlMXY5tz zuE3$+#(W6G;n2LjS&n#}fh)hse|$ao&Gi@WEO2P^{6cs=wy{t1f`}xj*qDO57tbnS zp$9GWH+wsrLpPQ?c4v_E1#3Inc{kTI>TAf#CX74AdMMssVsA9wJ`=0p^3M#+U(GMx zF^A$#!=Fw=8BQZH1P(*yiGs1s(tX*gjx@Au2&GBpZ_u8`-#)ho6us(W63*~vr{>IkVs-z_TOd&!Wv4}Vd- z#X}PKqVNEh)L-8#l!{3+s-q=`E1dlPYDgW=S0d4{gPb6hAdft+E=&>BEB}{Co~Ccu zF+m26J1RtzibtKRVH%M$Ac{I)PM~2{jz~JcGJOT92AorN>$THl`Pe4?gF! zy&yVy&%POSx?HB&`*bB_V>$-P0W)LX4qZP= z|B}lSLyDsRqExCYJo1N(V@ds^Ktcm4+Tsj*N&b4i$wHE%hmei1r6$`hm~uaFj4umH zW&a%=M=6{dyK~UT6E$Jk2of8%biuXY8$0P3qWPaghsY|CBkforXlU3u5~Q_50oL^>;*%SA(+B=*J%Svf{~ zt1B&KH(dOIbR%F)0TeneYJ;}J#T)eONzc~V&iw=0DV&zu zVFuQ+a&umIcJxguX&aC~=`WdFSLm0-kbGU<_mlq{x?LtKS7o%g*y|YRm7iZi|Gg=l zHMR~B$x_m>dZ3MSHohyDjCW>qj(u%c-mTuFB4z{%{zNSu?W1M$U#6Y~pxmvgz^D)++EBVO%;*Gz_L=$m;Y2m^m7tdv zIoE+@;YlJbqv*cfDq|hzG@-tjZRanYlzu38UkKg8(oYRkG_>%eac$BXen@5oU<~a* zld$KuxF_m?oxdLS8sM4qX$O!*wVIKE5aojlL2{p~@lpntZBM>V`B7+q)Myz-@=?Ey z1(CvV!u`6Jo`GIf`fhAf(`FUtpQ^+N2ZuU&d+n@I6#2XNZ%FUB9ud*|1a{||*e&k% zNWGaN&+iJZ{HrlU2oG^}qXV~t;E7Xxkv?@1pQ=cGL4rhLT-p1m=!%7S`~zs$Z7q!N zg8UqgSIqe^9h&UdpjEzsT$QB{T{XE2UG}+>Jp8~jMz)Pb9fPS9*IZ<;=c%>#$LY?t z(L-7r5`(q(q?#E(0gGm2lNL=KG}!V`yh$(|+$f3wJ$W}Or$=faD6Ml$9yp$MylmO) zY}==lijPs*`!K@OR@Ke1AcD)qd0q(9QBhcnmxQf^@=0iHQSC&8u3JEa@d|z2{)DFc zF-eX5>)K5yFqe^c5Fa^*tcpi-DnYv*62@(UTrn?&@qLfdBWF+{q9SIM z#D}Qqi0Odos2_+u(bGZGVLvc)p>oM`(T$-bSbDvKJP=*5Y%mQFT`(=E#wF6i*)F^edZD3bt6;`B1~5(hLb>$}K#9P8$&yY8u++{1PB}DcYJsDCUn%=z9c1bKTEDAzSTr1Vl$!BWe|H@RE}4 z^QZ&v#aw3BIvhBsS%YdxO$oVj4sen>=gFwj?qh_t?qdX;Ud5bdS1FukH!18mXIZ^! zX-#Pqfj^B37WMbdXqzD$RjQs;O4CS6SBqk=>l2Ru8fzq15^bij4BJYR{shfwHEuf{CB!IWOa2W0ybA z^HW+V*b6H9isirNs;%{@>a`OZC#X$bp7$Cjph1*>_gCi{!+VROlqZ$7GmdI{QmQ_w$8S4Gb!p4FPxigiBn~27U@UX%G=;D#z0mFzyut?%k zn3(j?;*M2t-egS-x2hEp&w-8uFTQCeepBBUH|CjcMUQbF1wRhFH0CXCB;Y<~5DoZB zX`DEOMLfHT*N_)QJoDr&Vth4m3&N)(ia2dG(PmJY#u7qucsN+OaGHO8;w(H*Nrg|lnc~lt&fyypFjOKdp?f8g=y%w#^zVxXMnZ4IN3(wlT z0!5cI+?i28Ikn||c2;hQUe;QCtU1SAjc_aR`tfRl z@!Rm!Y2Ch;cn)p22G>L4M)*T0#!)e$8S1(%87+A?j!oa=5II!b3-MEQyJS`0iUjOB z-N#|X=IFgRBI7)2iVotGO?{MM;vKY0`t_}pO=7}RI{sm zxKcFYQ1w<&h&!7$DD?QW&YYvgdEu3DRn@51rkk37{rfKS1?ZELyW`jSdi|Vp{>o;Q z!fi~d2dKct_kYJpEtp^66}%CCH;ReYvIuw80h{se@cG$?x}yqD-9jd4$zG*4)Y87AsR(7H1!fHt{q0$e3nr$lEOGYeMJOx&`;W;B?AQir>5poUU8!NrVm(M)0hD1Y`R$F=CZTX`Qyu_ zqVN#LngYxpq^5{SGOCKhGH|lyXr>0nJWi_NcrGYa_F*5d^yfZ0t z?tazMWymRlki#B9cj0*y@Xoz zpfF$ofioH^qPnh*pt{6 z1g!svJ^^@*h+k3gD|3sf5F@I0!xM{r{&`0*P3ZaTAei3*tc0iQ3vo0q(-!10u*Fr& z@hKAd8PEMxZn`{NR?cIIfYawHsYG5-_-8Y}>Q+q6IGxa2Oyvu;M%{cuRx)vAP5Ecw zS7AjGqYtlpV@AG@y%}Sv!l#deZoN&RbBPBIg~T~&TPvr6Zj*P*q;8_pXH;Br43~Ku zoBYh{!93rf@fOdN8iTR$D|&qgntSA;2UxBc(>Zx=hpDw`YrmEvWE~<{QftOnFeS)y z%TquT&n6{)diAixBc{LtYDQWo%co)gB9$>t>9ITDM0A;ul8Z--=Els*O;W3D(V~tR zULBoFO^)v7op|{>X#2R)hhTeq8NIN4y2n(z^eLR5QEG zdb-xz7|xv)$vVf|*hI0UK70}(C6ony*vwBo=^Og{mMfe|ZrIHXXFEGYox zqu9@YHp>{w(E|p8^dLKwkXn{pirDkLFDHMx4(Dte1T(4l`SQY6rbX$@iNjVnhVbO~C=JwhhT*7_-EXdbpo!;H z&Q0U?d={}p){jm3F!XVh*@Zby;H1m*wka9GQFO*nuy2Ywt_=yUY zthSM^$=dbbT;!3`CBMn@#(pN0)40*9MTQ7TtH2?A4Qg!h08)`>TN?=sjfaVqplvWGZW9}VzVAIVwh31*5zz_ zmoDsAO`Q60ktA){HzufSXL$O!VClzrLYmT6RddpMwaUQuN5~tz{L{%)V1IWaCoa|K z>c}uQQ^HVhdeSYAe1v^HJt2UC;r+b|<+^-?20hehRx&N8sb-57bBC3fP;i>dZ5C%% z8QRD3Y}42RRkEw-_tmKdF8egrclI|0jIRs5@->(y6yen8fz+?Olb5R=lzCM2);ew4 z3YyV{GRR)h2+`z3s>IUVx&`W^4atjpi{I?l8p&3G86d^7X>Cv#b^Q2qLbQysN5R(# z>~xdr@kB&9p+vcMRRuC8ObVo|SDO7Bb~Cj8Ek+@nQWguV1{I`gohYDb83%^$7s7MS zGwxd=48mBVDFU?5^+^L$(Zv$pe^d%}b6JDCO?GX9k!eOr;IIiZcql0)G^^<0WqEkC z^i6KG9O9SMLu4#Evvly~1P>;PRMi!W?q>cLsoMUGfyHJ&maNaKJ{PMz#1-m*W^+W4 zQ=H{{KXah_eMUJ(ir*adn<(QfC^|+-2@hJupZ{An=yj#xQEs&uPB;7Vmu|F&(`4%T zMVHN1Hl{#Vn4{`S#WXZBG3b-+C+xE0=+qP}n)7G@7ZQEUQNOs^eI0ea|^ZSQ9}SZ=0*;KKxHHdoaDPP3gsa?lzM+U^E0WLI`iXUIr{{7dEuXLqQ&IB?Ei?x$r%W zA?J`NltYF;QG51Ac%;ih9yHuE9<<=KxHmL*R)WaTc77y<|7$1*3vZx~^(piZuUps1Be zayHYZGhO}7F=uq`(X>_Hb`>wBEzDkAW4d4GP)SYP-B%*FPdSZR?cPWp+PsU}=b z^g|mbXWl(k534WJO(*Oq08C&>6C@|p)LD?cOzRats@|I4GJP~YG!~{fkq!HI!T0G* zONfiUdrFnV3Y^F+0YYRL8_A6hk|Wu98r=+k%iaT(HO*5Rki4! zP!WqX0X<+ta-CFIKPYk|Q$&t|PM#leI@>pI?sPY_x`%gk9iZe9L#ou`)y$+urTon> zW~S1GIgJ12eFvyNcGKq*29FcH&)mqGg84cEGrE`@^Ionay z+MvQ^F`#orV5VCns7__@t;$X`cgqmzfy(I2C!_!^zzfoM(k>dlIqN67l;i~Ce!Vkz zEP4W^waM$zFB8VIo{$623<51JRwKRFsxbT4^&oA%_fxpVm+EFax8K=v^jTs>hwIg? z8RDhA_8SKjD-dk@T@OPdGg`XK%ZiMN{rKtQv3^)kFqw|J=)@uid7v$TKs5rpxO=4# zoxB=KRz!xomo8`cZPkXPvfkXP0aDw-c_i>GMHJw?JuO5o4+m|qV$fpH-w+-%qyY?( z6EQAZihD!D2j=9wJZ#nz(8s&6kDK0?+v4VPl3)w>7fa0$bd`u~{0@5+9&UWk2QXoM zazj5rI4*G&v3DaaBQ()Sgrv0Wu#4+=P|JwU>SGhB8<>tk=X9%b+BLE=tb5z%k!TRV=qB2^5?h_y zoXQN8oRz~wD3*wwD^N1EvWj^OGwOwxDr; zuwdp3m?}ubLrz)a^9o8nf!gpo7GuHv5Uk%olc)MA&N|H?=(vB`z10TWcD>wZGfB4pE<2wX&ysD*C%8B0ctv_q{02CSmz^f3 zk@I@z=j!!{)S$ui?Z7zO1j~Ob~fXHoMwZDnCxy7@>NF!*?tsai;9U zeNdMBWAKCPK;C1oou&iCiy>&VLYL(3tOm4UIuO zSeCt8OJ;y8F;rEHAa@o5UK+Lb69_t-Gl_xZTX`@eh*#UX%m{N-6f<*Vl+z{YY>|2- zvj%g+kq$7|%d(w}Xe zW4vj`Z;_Hq%pO4M07`N9@(NOyo~yFAqpNP%;o*C-ir0SklDT$MUnayRo}oNeGESVD z);j12u^pDE8I7OL#Nu8mU_*lGTuO=Fjcy7MgPN}kJt_T`mCt-?v`@K=%pgdpna;=C zbAf9KDr(NppYp2FKabIFmzx36@s{5v-tY&ZwOz%%(nEFb<09mFaqvQ)+i{ts(*g&dQgJvOxk7#gr3AzMfK_MNVk zZxJ{aSozN@eY84rta_9Np51`Lyk*b6K`A(Kl6_RJ#cy+Xyp~UTpWJbc{rb6wO zuabNUzoWD1M@XX4%fPD-^Q*RSQZW4f?CvmN7OPI3n3O&}mAZhnB^_C|L2RzJjgvln zDUCl2Lo{>*e+iu1fF3t^HWy0wS`^5RYRjBogcdzqTRQl_A<$iB#IF0oA&<&j2@$<&Lh%{^;iGccjIsaG!^)bDAvUA zk6mTH&5&LH`(7es_gS{hVVsLRVL#HePUB9PPQT4p6I9^MnF4>pjE&u-F8fXnkVm^d z?&Wp4*<%i5Rn7d@l$hkB#^ciJMgK^9+=!aYYa3~d|7el;a@;mUbK#(5?v3>HH#O?(%)(l!qijc)n7Kc+v{!BRO%25aH8 zLDd5&AEX7g$Q9cCKNH&IU*4{Kr+F2VCv@r^4m;;29QoSrs;wg~3okPWd}_=gNnV=z z%o1p8_;WDc`h(|VwiONmROA^{^DaZA_^_W0nW5GIwDd^ypHq6D>}^4UUp>!%3g$RC zHZ25ZrmceV<6T3y^1Ml@F?DICuiua;MTK_6XGu`VTbs@|H=Rh^^fZzl)}r6L&}MTy zrjeV|reG)&@jmv>RFG|;RN!ObOUh!D2EoICjNwQrVj#oD`e;c{%FB{93q>Zx%>2=t7>kJ;17%KvI0z39gtUS!NWfLI?S|3*Z8JTZ{-*}(C>o+Rv(mJm zjJsh5dJ1DXI!hbXiljDwY?yVG#)4W1Sm{%(;FNGPt%Bau%$%kqC2&D6y;!sq4l8|ozh*h{0sx}; zla{B19}=CLKzOXB{Gnak(X(|vRev%agFKMIr_~)kwB%ybnjtoB<4%8OW#i~xj>cTRKCUZ`y+Yr~%fPX^z7e2YnpRL>56(qxCO}3L=B5D+ z_vWvoA6*qvn;sm1gKqpP9UQgGoQrbJS!9&3V6*9CM>I@L3 zW-*RA`i7>IhZIUQ5$y98BUlb1juUUr=|V`eqZ8tj#P(tHCtIyBxtkIYB zh4yyW%H-rQE689*^P7$MX!G0mSuQ?v<4vYLmtAI-Nv1cpD@_$XFU1rR4}@89P$8J= zx?ihd_jOI&9u+~#cGcaY{A~}!bXAd6;JBLO&p6MKcAa|M1cqw*;g__6MbEoVe@#Q`=$TO{jCizAEXTozm0 z&>C%Nx79oE7t(JOTm9elJE$e-ciPVS++Kg&jX~S-;ZLbh+*G^~xG|Ka+)nGy#A2@* zODy6EMuWNi;aG{&3btpx!8hA45~OaJ2SvBoX@Jw~2#}rvQ{Q!mI<((DZF0^I%{|n= zNF)hNcY@%hDjY`RNX&SZ9wJ{~gHictkE|KnknMX;zU45gs5}6S&~9R%SwDgKOYDS5Gv@izZcm%UME#o|GBongQ7z~c3>+;ZPfj7TaWQ2A(uqgVNwH zIDyWM@E#@tm*;}-u2&Y8@G-2g8tukY?aDzFO}7@8dwi*o>U#Op`sqo}Lex3ODOIcJ zd$N6Lnl7$JlNu6(_UxZAJ)HE9{faub((x3CV55$qZgyWjuLOZB5!-_$WQF1hYrk`E z%t_zr`!yxvZof>O4h_=aE;B(3PID~lb%PDE6X3R^q3HvII%>K9f3op^@#_B=(*F%s zeo@vhdHP?m8+8>&t$G;{+fS(o8X%o7Wg@G>k%8uK>|9Qz0U8n*$G7i8|{Om zq}$H24(NB=ErMCLg3GrH{z$sx%9d0QjVx6`k0ffN36p*F!zk%`eXOU&Wwo4f$nB04 zupm&%L(T`|LuNtF1Dh67?_7}wf1S2@-)ms3PBQe%3+wb&+GaE2wW_&!#Sj@ux&MD0?Z0IC{}8+X zcC!EFcE1SpS2m^pj_Lm)?EhbhHaqLTSv3o0=-mBz@y^w(gE ze}V2VMbGfX=-Ix^IRgVT=l>_P1a*nT7^riCuJ1=KvWToTyg8u)TGXG-z zbgZll9E7yYOkalniH&mR8m%vo62|B~+O(* zk)7@Tx*?u*c={;MKd`qyILICBk&e%?*(MZA=9$`zDwMLof+cy{#Y?4qX`yfCR2W!a zlmSxiM}Gc~U?vSjFeD1pZs1>n0$}|Wkm^WDjl&HOl97MzA79rGJvR^Eci-cBA1iqg ztZRNhtQ-~sLjqyuf?%>_aCq+Z5iH8TfajLs2HPF<615*I)N_ppO3smJ2`AFg}nWFQZ`Enw<%dfg4>HGXv?*Pelikc)|* z(Ka_@_AcQAa9v+{o09OP7J%+vp>o#)R>@s%_s-E_PkiFv=MAtQ@H;qf^#kS z#kOLHIdABhkN&5vSQh}{qOvs;YLPud9`E-|!iq?x1lo?)?GX1P5JWZ)6o~!gG)`Sb zP9+a41u0MePn8ce2O_)W%od#`(62a7hVV|1N0?d`$XizitkKTE>j#OZs8)?m{+)06 zh>Sq-2@dS-81LltHe^BQ<}RWc;yj!;DuNB%~QSp?ibs^cqAoN;H9n1Bp#!?8DE;q{rB! zI>2b3e=RL?q$t_kn)TV76=2LvgjZ{jLs>%f;ojhnkzMv@_xcnvpuFag%pheU--+3n zqaR=xcF6pk>~nyQ5f-x&d_sad;-buTTM7VBP52x5Y=l1=08(%FIk%rsj9=XQ2AA%o z407xVK>&nq*ku=yb-Gb$i(BN$5NHyEL`p~<$9$NRZNP$lv?+3+C<4Q%K3p-)0kJ>u zF)27EI%mv0l=_g?Rb?XPuS*Qc+JLGUFZDdBCXBz`v~lhbvlQ_Q8goLkcP2uuAExq#Y26^H7fJ!n7)(?r;V(;S3`Aj*@1kE%IU9hUy^yKIIr=b9@Qq@y5`+^NO0rf;3VLW|xWlT6Zy$vT` z1FmMO`Lxrq`g#z%$X8KpA)SzE{dG2#_S#3k;?lgzM;y;H@N!8p%Z4kX^+La%S9B3OgTm6&CEpAD8F-;rJ{+y(KgP33>m zcZJfSOyQCm7wK|L{vMhPx`MrNLFSiWei9^>v^M6K5a5ejn1PtT>YBC|bnCzUhStn< z>CgWD`Ry(afNp!Rvcz8$REIfnsF;V6P9Vt~T$63&g_aKzUo}oN3fPtSUCd9(bYU9~ z#RF?w$N<=BbMVErI8?-7=;*p*Ju`vXGW26`OH~m7>C}ipD?FacHgT7*;zv?e{QtcSj?#P z*^KG)8<3s=SB)NDP5i9m;!w%8R!e(wxi`$pqZe<#BaaThzac?LGzy#25~rd0$*8_` zO{u=5z91*Mn+E}`)RC(g2>UK(m*f%OilNSBKwat@=R-m}XJhEHk@cPqtbatD5&d@i zWoLUya$kPLJHYT=4_h8W_?_?je)m%_uXfc+hW?hLD*;|^j68+?k7FgcYVH*Q*LG?Y zuJE@$$JpKdp<4<=z6p}FaY5fcmp+4T6jxfHH6wjQZ({90A9U|&kksLr{4T_q=yoecupb2?0-1*tLNpW>ArZr@V@@BkEuJWhYCQ4WFF zeYJ8P$X@Ako$%J(DS9D2i_jo&(?*D7r1L7J)8OvptwF0K3@hI|(O@4aJ%fyT(3jGW zzSvr!69iDjFg45X21#oKt`or4wb2;0dj5>)l_~ibi_%-@b2Tj%83*~}z^We*{Z>m_ znUm`li%69qo&fYUC9oTZOs+>c9y{Yh$aljU0g0jT_HXC4R}{$tWRn3#p13rB0AR=~ zl4tJt#k|2UtFW$*hLxAHOSXSH2?ksgnHUM#-6y@|Y@SI$rGepn^=w!hL_7h8+Vx-L1CI$|5 zi(Sb0_Xe86pjNE)fJ__lrriK*|8?Y*>TZ9X9=`cI=BJs@36E%-hWxK>dEjhKqd~-O zj_8MC7z0`(GF}ioP50)oN(59-srf_ zh$Y0>g3ASv7-BQSRePd!(g`pS04M+zbv9d+M%YIq$DK?>-#^8VtWc$ys}t<|x1kV? ztUtf^<0onBx{}|(_b5P29qWGZ;DK8fZ@-utdT}*+%HqBb?`s>JeX?YKLYeThD0^ax zKaFrM)Ica5`u{LiSIEhY=-B?8e)^FAamtOSSbXzn?m%iXBm*x9J_9TPQCsHF1T=tS z+z!aj`6l1>ec_L>)yw{UR7pYQRy?v@Gu!*h)9=$4{{CZfk8P>B(QZ>u4^({QEl~}8 zM_Zj&U>;J!t{~6LH}A}@JAMO1lLm@;#(?j}i+%gq&E~`wP*-NJm5Hj>1#uG8s%8|y}!)WjQ&0i+J(7|MGA7LdBRL5I*$VQ z@^;W{LU}tTjXlPUzV0{Il9kj@-KNOB72&-0>)k%uetbfea(g3P5^cP9t*6v7^@vgS z$l(g}7ec-d1s&zAfiMG~OeN2;iMEHkl@iR*Gr+lDS_lV2^{UC>gxq2M`0Cr?!J0)F zKq}xtQzL&HF_BqKFRhRTEVAo$9r*r`rc_K}>pNX+55$bx_9w#sbEPh^Dbxx(`iXMN z8|>oF(`5%aP(tKp@FxCqGlsJJ%BT>&Fd9)35@$jGJK{ICukhO-z>oJ6!s}eK!0-K&!mFP-$l4+k!NcqY{JnE1$vo%t;-|8_BX5Cv z;A8B3p1dWhAK=O~6|8nqd7B&NQ}t=0$6vG+`dCL(X>e9Q^R~ZP zU|mVYLnQ=Am*A(Ngg%JQq7qL5TC89mE008RqUfGteWb_`BXU8}9q$Y9dnD38Y7E#? zP-VindBxJ;+K~56w-2QMPyX(l?a5o&H&?&ZPeR+ojpPq1-MDC{^0TDXA>Td68xPNf zw_F}=J@T4l{bcqbf?iAFu)j)Mx99Ym0SdU(@ASd99_a@k`Ux8w$zQxGSqm zFH$UQE7{Odf_e~^jBnUWcwy`|$h0GZ_6`5EZkLFa7e=wN!4G@7{@mHCRyxp=^IC>A=9 zP4xY6$Wu@Ft1#B8c<1f(7}hLn@VPZ@WzB}#;>L%CR=1%R=G`46}gMq5VlsCe%QzK=bYqZhG^W}(^3#{Js&Pw z%WZ_zEh1W|%d2JkY@L=-3}iPS?mgJ!Wa)SqjYkv>rr`TuJn{V4rH^c-&>5t_6Cd?y zU%tM&Zr@ECY7pW?^#9#Nk z8!uQIopz7uba3gN-YLL~Y2;7$WZ5nKwD20K{v}PNPi>!H@>qNzI?kM4!YXf>Jx8-h zGwYM9S)*B`8SuJ@;KU6jUt-fdr8{$T@`!rZ#!(%aV?Oc1fdl+G9(FGvef!7>gEW~h zJ{R=3y)O}_n>K1Mx`saJf4|-F%_B#oa?qbW#@Z^>BWQGc?6a}(qigKgP4!pz?G@sf zQB#l76TDfkP%6aapV3-u-C!w`QyCA~iQX;)U>;y+Lw4LC=PR0akggtj2jJ`=*DJnu zm{UVc*sy*EkcS=#4+Op+vD?B=kPqFD4mds_uiHTF&;?uO9^d%FvanJ_SYf=&4ZJ7D zN3ir2oR4&SpZFVTP`mgB1wmZLp70IQ%2=%A50eB!sQC9ipuYz4@R8&cgvE(s$A$iD zY>X2kd^Z#atoRlsZW#R*wNp$c#b-~)3C0a|ivV51<}ksYyi>$p9JZ45iNr6K9$#3> zykATsWg@2Kd0OQKUNfoKrSa*dX z=EM(``Ns*zCF$Vec+w?dri+WPPq@Tvt8oe9)^Ru}PF>}u91H?bRt8eit<(b-tR)27Dx4J$+{7FnN zg!|U4GsJ$sW}B}e+J4a23&KCBYuNop_l>qQqig)}%KfSBjrW7^^T*{r!Au0F{sQmTp8}|nD(bE!xJw>@@ijtoIhd z75bQ<_FUqToluHGr$5+k#0OwSso|!l89-n5rxr;8HW0gERyiT93^#Heor}EIDg7?tFWr4X~MS z&6Qp4!C65IS#>{JDrb7wuv{fP64q0@O(M3z zW_6cZw`)h>KNuX5NG{*vd;(~@=setbz3|`aK1O5yV7$%y8d5pt+w8RQ1_pyc>$f*O zh_7FN7hyV@<8yLso~>KwH19lFpAab`|HLl!P)y&fiJ-&KTwX(%km)vg75YIHi>wwuj^Z%>*jUqrN_E^`vte& zBVNsG%%bn$Y3J@&GXE`J@A+z2)XKWTQlqTksT%Fgty2cO^-7^*LFQQ~T)j~Dm_X>} zp{)FEVKHf&u3Q-Ju$d~2S>kl=q9tp69~RkEDaTaH8U|F7xCf=fZgjLL4+APns9wAR z{zdB?{^w;wr(9WSwIbeNOthu&ty)ay0 zQoivGR?ThLdX=HFm=CXYfi+9Kt%>wPBfkCd6SHFK=qc_!2W;4Pc%XRVCpheZ(O#h7 zaF~(JASmp}>fHI@r4`NNV$fPB@ZS@Vk5q6a;a!kP!fOq#r;y_HI;11Hi(cM~qH#ne z2DbwYksw{dHJs*%rxef0iQRu%%-Q)E_t_X^+Zz|?Cf!z`VX9>t<04@)x;RJ%!_lJo zYB<+!${{Uwl&txS);VeqCUB}B6xWX!J!0mBtEo*~i>f`)wJM^;P@&F5zUUY9SLf)Jo^1Mx?6x_>qA26BqeZe)VU#K>n=8)sB%)RKg12dwi-; zQtDYXq97_-QeUni-HJsm0+$MNyRiWXo91}4k%jpVs~@u10Tk;7u-c|=B)|PErUoUU zKP)5q-3;k^?wr;j@{`g51zMz!eJzZH3Ti7;D{6IUL4mWK6}?xUs#_}_shQZAxHV=! zNeU;P?nJzh@FhH{E6hk4d6j-)8UZ^2vgvuRV#Go$&&2Q))S!XE{ zfX?1o`K03=1}EA*XG%p1ghB#)MG2p{2u|zUWIjh?qE zospYw_H7Ss-_a-SeL3loB1v6o49i%M^2=n#Me}fdT57|8VUqBKf10S4imO-yD3h$9 zuU%s(_t^(7i@dr%`72fMxaqCR_G+vm48>OVbnxuGRo}|Dv8b`&Q^PD4{er$z->dKE zmG*PDQwb{U4T?)o?Xd{1&jZBK=8oN(r6&)}M$xT>)K8-c$61G_%~S zXwKr2l#~V9IgzxVBwS&P)(gT%rqB|7QQSsNfu6DmBj;-K+CC?7_*^tpFCOc z;*f^D&Qi-F*Y+4QE63g+^*aK?I-5Y=ZiYy%0lUA9HdULbbmLcBMF)!RBPb*^%C+1P zek~%1p2jdPqM!mfg;LL0BNg+nHY1AN9~VF*N6B8sZJ1?Dn$D}ImOryWaXgME?`(L& zve!)-52`Kvnculu}KnA7i&^buKvLXvn90qz9Q+>~V>)kC;ER=JM{?|2=E zlpm)bRbP;N)r$yb+*MwHXIu)skK0s_^(^fF`2695RsyqB9!ojW&S0|L(D0^Sk2t&D z>7dGooJL{%RtHHmo5fOK6}%W4-45^l`v+OY7DQ4f~>W; z->a5ca+B($W8dP2(4`RFdWcRdOp6PKgI7xPmOS0SgNmvfo735_gIuqTeZQMhU69Hh z5!z&YWPIU{pb}${3Ti^6wxlS0*;QPmixLt!T211@P$V?+hL0+M@QfVITe9Cvi7`Ya zSVA;T#Uufmz!I&x_@g}IMn$PILH0aEkEkP0w;F8lNId36vGTq2f&03X07=={kY;Z* z979eHgT+|cO_67#Q1&ADfqkZ_ZfCU_*1i|&+aQ=1PXCdu&7Q5zpTNhzPCs2ArwA;O z@i8p!6Y;%Wx3{tgk`bg4*AA|bT9`AuprMdaIL=%Et1N75N%(Zr^~j%5`S)#>{(?P9 z_Igo#BS{M0S>s~>8b?3XrrEsKHK6g{eU_jrLts%)9Aic_E)=GYLl8stJ?)^1e6(Ds z#(R}}2qkVp@k9bRSOU?~IyHYl`-eES1)s-}R$+^3Hn7IH5}MWCy4+a)g4$(Hw(egbY-{Ta;SuJzTZb|^46Cx9ahs=sn;e+ zkAi}YUIbSz*D<%HSR}hxO>*5DOFQQ%*n1sFtEYQd2XOM;Y)xKKJWXCOyG2^zT-qJu z4v?JJ25j(+dE)0Ran)=~U8<~Dcy@)F>6^}#PT|8TP;!R1?=Bm zRch4MEy-4b)-^F%u!rFgf7~>YM10JD0IByvqq$#KYAmtl7dbGCBxtCw{I-XfVXe33 zaKSr>(mBH^CHGnrC=!4`u5%5ke&-U=I&`U8_zFp&weJ#b;kB+*A0HP|l!~3_Tmg0A zH&05QU{(aJ?uvtGl=YiXbqc%z{a13}DreP@u8Fld?a|t?z1@C<_k=#dqv|$bXr|g% znz*JgNzCbM^o!aG>rLvFY18)evyWqmt;a7hC9V4C-y~hAk4VRrx+OC+)$=AlwBUXA z^flng4)m=!ZMmLAIIk>cclWb@rob5a(xgy9CeI!dmL|iTxL9$@iL^v3GevRRj*Y2| zHv2G$(lYn&x7Ckr5GF%`&ofOxFI6NjpH?E85}(fz<0GI;K9-6~Kzn@6C*-4JU8%~ejb~fgk|E^m)Ra)b?@LR;8Ws9P{xjBu&v;g zVIJJt;_9Vy5C)Hg!4c3NO~uF0{27vq-wu%LjJAdNlzlLWUuT=LB!Q*7P_K7%@GvPU zZ`^L|56H`sVs|}G+B_-m#P3{F_p9l$tW2?>(Z8E7n%^{OLSEJt(XaZ~N4H0Ju2UfC zT8|9p_f#J0x?GZ7J~;1)wP$%3AV@iFNe*qJ(eE*+Y_;XdC2id|5LUbfqb1)m4U}^GVMm+ zm!Sr8(l)Dj4Qh%tVZF)vtH*0C8XK-SIf_bjWv&DZw*Sa(042k~2q%7MX+My-N}=%B zB_i;%^FtX0P%sq;{qjPNEJxwq^GgX`@^U3MpXw;(s=TS7jkGb0s)(bTiZo27%w8Ce zt*@8xKs%PZgj@)UZ!InNT1&%JikF+tzZO3IyCpX%iBe$-VOrQ9IK)?cN3p4+)FrHV zddhbS^lCXE1Gq{GV*}o^;PpM8a0>5L17^tB5)>Y|$KG8A2=_X{)EkDoZt0%8K}jbY zW|k1u=Gdx0Ukx`x9>~>h$+bEH1FkWn9K$M2gFsjdgbo7`8p3rWWkZI{E#^ElmmqEf z;ZIS~ zyP2=h8MkG8O`xA4I7K>(@Qx8*P~Ss&C-e_;tU`Ax?L)3>ffjl$`^2;oH}=q((%0k< z;>3rX>SJ*U%`LoFVq*#pmn_&KI=Z>H=2O0U8j6-@c%Eedq^%S$l6Q|9b~bExVxFl1 z1^OV9J7bOAejGm+p`Jn|`UJ*2B1$^nhkZSGCD7vz=gg-v>QX@WaBe^^1iIp}BXOvc zEXzGcBGrG$iG|%bzObyvE&{)xyeGD3fR7r|Y7kMIqBy8HFB!Er*4B7JZ%!Hy9qL$? z0fe+js*>EI9xc=Tw~{{;&u?9qT?Y#gvv72=t0L`UeBt1Agx6@UAXUQ)2qG6XbS{f1 z9dWjpjYKwz;%sJOT!bgS{*s1Co!Fgjh;B$d1@T@hOdy<&VVvLxZSwJ5$7C&FRstUS zPSv;)b>FTxGjdUhEc!I(_ z;4gc^uFBptKX^a&uxWfLvy$(>dO;g?Xg^`HQr*(^(l^q-BFT=cjUTQ!U8ABk{;^i? zJRzi5hE*GD{3Db;n0VcH!#Hw_h}qq${7a=ZbgT#=Zk$EARohN_H38oNKlUMZ*8LdT zE56ZmT~d7pV<56i1&v^NqF>(+y*QUlU+&+iFePo&C%a-GP06#fPnub0{qc^URs z7tKfZ7b_rN-X>BS$o@4s-OGUX;LM9_e(G@ddR$S-%};+%RG&B}S&tqH(;;ehE>t}VqM(@UEqV^(? z=}X)-oIAHuGH=yC!L^p5J~muQq3iJSjb9bq3H9H(|Lzj%Hgl>l-1VbV0X2hwJA7sx zi1y40GxXz_Dsz8wsW;jNV^7q3D!OjMhIr!f+g(ft>Mb(w&zN<1UPFlX+<1dha*FBh z^G}>7s6pljXrG{pDIi86-nh>}gYoL_g)YKvPVomIixnUXX2eq<7rtM)mPhoMNb~;P z;fKbv_&~QvKrl-C@I1tZX@!r#xIu^npd8e@4x2D>D_hV!zy8#5@JQiW=IpenfnTwL zT%~}TzN5EO76I+BFo&>M34)D`VETD{9vz5D5m+Z=QHKR-{K;AAH-7?M5}`k;7h%1%-X zDzp3Bq;XGF;9hY2qJ)~%(j}Cuf*C6OKy;B7%mXQX|qy~ zHGwFn#xgceyo~uwt7@+Ie?{hKSdvF_6C9BU*Sh`U(Mc56S5B0{nHo2akAmE0Yl*JH zgkgIk-j{^_h_{pCCsy#k)aZQQqf6OO+KT;+L01IwxW0nM6{C<$)fkBU`~yP0RE6HK zC^LR)O%<4&&9C5y70Vcd$Hj zzDTP?b#x)~%9-`>O$o=+dYL?GwWQxche1*o{yLH*I-=@@9I9gddM6ZOD$uMU8Qr*4 zZ@DQ)B#bGB9F;i?nn_bLwRuu&L!TA3#Rg4{Xm&51b-h-j&2Lxznt&EC+T~8)#DkDl z3WN@@S6lh=V+}5R^=3s5PvWd03-=kkHdr9pcyglv6F$cP?vgO4a+zoNbT|Ihb}X45 z$;K=0mD$ijuTk_mx7BMl2=PfmOgCl5rJg55b|C$TXX+NNktkEsp^}DcLEFtH`BCnsFe*tir7ap{Dj9J=NHO+vd~Gqc;c@3U3Cd~^Yo z(~2!|4MN1uxSqSc_7P#4T!jc zvmfn$Oau>!MsqyLBU z?UCD=fp^I5tXVKwn??_W8&=uOZ3#WIh;frE4EU^B3?jF>NX`_G%2XGl^m0vM`$Iq&)~$M?q%0W4f5hWT@{KGeULj&tI;|n`K4AF83UB zI*Q^LO+3uZ4CEZi<~`u`K9F#X zM4nwL~Z8R+AaAdldXa-a1F`8FMZRXkX&iVa*3jKGr!VW zqJ~HQskOn`n@C~XHJ5f4?v@`kG-K;dn;PcSfLe3`u~&mOLx^J*?CrS0$w26!Z4h}u zl*L1CTF#=#-p`<%ZW-5*$#pO>BIjOw>>}BCclOmi*qxXhqjOVSqAhp|Kl~m@WGqzx za3MfqQcs?Bt1YK02kTgRKMQFn2SckxW1&?!frd%ff^p~(L4!~NH}zEWub!m>=&X&i z2)wmaeO)2(1khkpD+i5M;=9{LEP#Vvs;21l-xdb_+P4L<@i=CH&KB!Ke;yoOsew)G z6OK~ZF0$K#)1vdDRk4j1BfW)~NJX5{j0QfnL)&%x^r>32qJbqZokPAy77Q=RpTm9WvU**eQo=9c2^Qc(9|)jz_PQQMN1wBFr-Xj`@pb1tBg89&*EbX&^>08270 zd##zb&jwvF@Iych@NhwqqvhlB!E(0A=(xPsLfpJd^nETT)LC5qQW7d1tdMFDh!Wl^ z+Ju!tuEvk6xk48bY1u(Rif1vq_nE{ zo$9&!Wle)NO4*Oo&Ky>UphFKd>`nd-!l&tQmWuTzSZ1egjYs9V5xU*EkYC#{_PryqY$K^!tv&kS zr57!rM;O=LtoOV|p!a+9c596bJXFBwYtn{wkX|B^KGv!d1N|8lapHOmK4sOH?un}1 zSxnHjT=JM!%GclL8lj1^x`C`ux&6IV+I3rH;x|a_YsmcN6~z_qWTkKN{v5sug5>I= zc+`TZ>}kKm!j^>DwuH+RYi3o9NqnddJ}ms6CBZ!wr7WY)h3=-?6>CS1kTAhz^Vv%F zVr`9;{H%m7{j_z)y=#*p%&sCtV;=ms?NJBEa^88t(T1U_6;dPPw6v|J77N5$cjC60 z@v$7bQ%ERNFy`rlU5)w?s)9u7dI>eHp%RLhy{RkhlcL5prO^B-ys#hX8!Ab<-LS0R zT?mNS+)<2|#N<}t^reWb5Xqt~H)-LMbE7JD=@vV-((9b2iNB&a%bk*RQK>6hp}|aLRpCfD zJ=Jt9M1U?U+-IfR+!u#u9$GPubEUJ6i(4<2p5$;>U~$oc2Ad!~^O4<}%I@KwzmJc` z&JK+400etE;J)p$(DKTyQ+I^&OjvSZoT*M{;~SeeinHeFd3w~af?Xn`gHz>1R(vH{emg;3UQ8CVH-7pih}({Xq?}+ zCDwmL($yvTcu-gnPNTUx=xs9X2t&_ZfV_CX$MWWn{En_*w;NLE6vl>r5okg-(#fWJj2M$W7?=1OxPBIH?a)N z4S-!I(TXB~ktdZ6WN_%xY9sMWQ9>b}G9^gbSsuvP35Fsb-yi*en?||7uheq0z z5A+(#3u^=p1r7~-I9D{N?0X85H93vB4ym(Bbh^uSy;L5FSTMyvMjw_)9|kMCb#E37 zCR^=q=K!`$9E1aO3#J5Bb$@FnaOYjYc7y3EckYh`mO=#39DC&$nwza+?71El;<;|c|NvXCb_l=Cz z&TXi{Zk(=RC8x0hwoT|PpNWp!-JD%cC*G&etv4tv^>N9{9^bBp2B+zd^q8`F8n|D2 z>QJVWv^OY%HhzBKugl*^o{$@P(0F0*WJA{U-m?ebfQM@T1jIDVR!gbB(vHx&*)bFs zKnITEZIaj$*+$ZbY;~bFQ5ljX)VN4cd|k`EHJh2LyJ=Sfy2nQxid~zL+oOJ9w*QB* zdybAQ2p2^^Gnv@Q#Gcr;lZiF4ZL`y{F|lpiwr$(C-O0SnIp?nP&bxQr^;ZAWYwxPs z)w{ZTtG@5|Z1%K9K>N=5D(FPlVmnl4yphWP!&7b5rZHlj>*K*pdxJ~#P>z6SPQN!M zu^(NMe=n6I`)f@g0UnA5a_-2b3WCM8UDfj8xsJQQWFmH(gOGKBkZRg*_L8)-!rv+L z{J&9B2{(o=6AtTMW>H*XD}ox%32~h=gXSvdE`LPDgRLlNh9pTuwI&521i4J7;Ee*& z18f|&_9-9HAzOnW%%sZS@%(xGXCYi5Um;AwV{9D5VK*=|j@2=_PWjL!owL1E3edC3 zJn%b-8hn^c!3xch9Jp_wLd5?32nmG}PdPb37i~y6!Pb_fI5A1x9jhEJT=_sMO(1I7 zS=gQCy-Y3bG&e8=qWQ4Oy$HMzUY7M5>3<`WLD-!P3y!C5G$3M1US%SpRw3f35K;#< zamK-BdLF(2H^VNMk%$Hepk7+?F56i5mU5SIBfW$bq9X7;#)eHt9S(o21IOr?o`?0B z4Bj+nFe0r&GS#N*&&?Hni`++vbp0{dIUZ8CjeU+4lI6-e*hBGm+Ji*%U zkm24OvtFpNWm{3Wjmg}Kk4dmayQwetAQhyA4t>$Jk%S$%t7(tT{>je+4kmavk(uV{ z3GdIl#~TL#rd`#H&rUV)=)0{bSaTYFZ006Mh-NJqG-=SPgvpyq6@2_C(rHvji<5Ua zbCOk+bZ8v||A14@-LMH8D3?J@$x0maNI_+$W>Pl^;wD=0ZM6Jw_-jWQE7uaZMQi0L z)Xj($rP)mQUDY(QMIzGarAKHN%s>_4tU+JOSH#?~p#ga@SXv>X9Ve5CEGCz0p z_U@g?<7RQr6oe_+&xEFqm9N%LS5 zh8&T_{SJxMD41(s?{6Vzrv70OvsG)1U(0X*=(wycd#Ru*Orc?RZCOZk#m}&l(Afde z6tTJKK3)vz?FxF~HFTycsns zHPYz%n5`0mQQ65P2U&1S!<-2xqUhwl3yfua{swmkGpE)pxbM;$=4j|8O=+-+e_OJ8 z@}9>49+7D8I76mZ#w5lvP&3eE4_Qh~(dAWx0T0FqJILFq`|%-tQY-mkH`cNngukW4 zGg)go8oE7KE`LB;QPXen{?xd88Jdwo1p?LocIQ;gKCVXSidbgpic9lXXErEvH^Hq{ z@I9e+;g_e7l_RDQT+Sm2Ee174w|&vjfVHCCU>C$VlWC;8M8Pzpt$}x<9;+XtiMMy6 zHipmqaSxFvZ_1=eBkCSILFT1D9?{rt7x&#JD_b;8dHR|V(v8b!2-XRB9^j)nYcYps z)7H1shy`eM3WoP*E1NO6p;T$hRkP;GTfB&-Q2}ouuEm}~h=d($humHCX z9ZSfqUy9k>1|#0Y`@AV2uS#hcQny-plPwzPXd9mt3y{<^mdK)BL3Z9`v%~hhz*3ri0%y`)X!N{oj zpd|?Mb?KdAH5?@~RD&@j(pnxd_KHo)=PGoSU?Y)mJx&jWG!Njx>A31ZO`)uK=zo}! zbBExG=yKeu=(t_Y!p<1Z7mjJj-z7 zA@l$0g$s=}Voa0%B`>X_sSx}1NK^*GM0ytIPx&-S1|Q`JBv-rxULM(J4(oGK;<+=Y zaQRdtCaj2T>r+C6_@7yfR{4V#O#7@^UNv4xnI~lu4~axh-~}$bB?xB+6Z!>7wYvH+ z=rCuV(N@6o$D};FgZpMeqq1>Nt{1D76(+diU1jG4kBOK{6jG#bD!=t=piU^~FI!^! z3rywK--#dEa#}C20;N&(>#bkb4| z`l(RzaN6&t8vCIPms8DX`^d2v{7U*1L=9Z4R2HDj<2Q&@Ac=a`!op1Y-HeEix1*R7 zV?}|j=x1+ES)<2G&vf3%Sy^mmsClWpsNfDohJH6$OlSsoq#q!aiaQ{fSv3b>zbMmz zEAMs4zKtKd>Ani(Sk%$;A4v5Q1G`g?lEu(*`Ec>_PCE9d`lNoXsn0MP)vY>vE%z1R zMwBz8g7uS0q2yFKpwD-6Z7A*z^aOU9Cp~L!a~20?Sw+fndL+~QiB$h-O8XV%jf)GS zDVV@lP;yy3wEqoX{19)gjNbip>w#fivK@4Aoa_w}PA9dA%-F%-M8FoHjz!lgWER z%i-y=PCGlGRE?cYn#X78cEe(C^xImo2xz7{ra4Pbvh3mRK@RfEcps{19;895rkWhK zIlMbRMF*i`&jw`{9{#-hc?aV-PE)~P%aFy)T({DvwQ!$MJ=f8Hd1IE%A(eHe9~2~< zkb`t@sxqh9E2)AhBsq*>fxe0qD>MPC)FOqr&rZFpETD=W{Y~{V9T{n+GJ}~zMe0KY{3tmikqI5~15{-n&=AR_D z4y;TbImoGBR|Zw8SCG)LPaB$q*#6QPG9uvOUI zR?6>0kmrnAVHkg0MF^8>?&#moF|KcbIK8b-|9)o_R-b?a}8 z`I$~Kmvu&c$7PcUFUCa%3NHrlJ)C-L-QRXk0(=+M_JuPv?!L*~`J(PlJB*AN7hb$L zuGUww2S5UOLR)68Tn)KQS;7LPXa%J@e`2J*VL{1YNFmHz*dT3d#Vf67-rVN}FRk%m zI8h>W;t9G5k&v3LMIqk>b#O>Erb@2_N7P#8fPui$^dY0!I=;=d+8@%n^qA1RVU~TD z8FIm>9kM85$F3k3RuSw{^UTyPu?EVx$z5xQT~m|B9KXFrPv_gfy126}d`;L?s$<-w zj8b4wh!hm1+MRVTc<1?7Sd*IG2=gNyUX*vwTSUYv{=_W+)y^)qevS^Cz*cLZ^46I2YNAgOx+V5I8Z0c+?Mt|-ofoTh9s?o31^5y!LEe*Y+ z6~I*{pv9x5teF8eoy(G9AIr$eO9zVI!aGf9{Oe)+b^K`y42V=isN|hCKp{_M2rI}y z^xQiH-ICW~>fT>*$cwMe!+cH#&Zm^KY*U@^a`u|IqlPLw-^fg*2NX6&J$61^ z#zZ|5PPOK;73LXnMw<@x5w~yKPk)q4Q_VfFha2tXL$}62RD6SxH%q*zb{g?w6q#FJ zBx`9Y>Zov+beFST+urBP{0!%%6y^nv0lDdB!)(CQ`wbjg+&BZvCe!i942%ne&;pCx z?y-^liio9Re>ZFC1FjZ~jgQ2n`{Q@f5R3hDJOum9dzsuqmQsq0EP7K`UR0}Ta>6(_ z&SW2clO619rN`&Fy#b1p1p6+m74&ztU3&$SSz2_KcM+S;vAf*96MjTnV60TE;k>tB z1Rrx6@H4laV>cm6eI>V5#3?22gWk-+B;<#|(e!mo+Nk=AiX zddZ*1N;LB-Z$mGEr}dsVlKgOMEF~+o@B?!&X?uZw`yh zWyZ~Jjzjzpm*Y6%1iN}=&uj7-fZ&`AuN3lGZZ;O1T)LjGB~LDQ7>2r2ZE?Wpi|ydB zUx>#V+B9P$J$0CuZ7$fnnq&wDfd+qh!!}!9BL^$hj59L9wR7KsKvxHIchZZlvFsrIjweO@p5|4HiIU?tf!8- zlMvH%-El^L2bSyUmQlNWrz8Jsj+Iy=G2Hd z9giuBLwDlsn4I<<+qeO?e(W@{p?PLhG&S)f5k3qXz~=Ob!G3;pwPY7=((LWhcPok( z5owfh^B0#R$r!+CEXKOg+~P|vk*C2h5DclV-$kW|f%}C>8!=!_3O`CjH+_fkG$XQI zFO%=snZOQt0L?j}jY?}BP!#x`s4uN$u6E0z7p2aws$h5)wZgzbHjn#Vh-gL=4aQXv z_lP`%_)_F%0KMicF$1NZ;pW;YR;qng=}GoRFMN=7E)YM;%?puKJ14Wrnql9${m!?= zhB*bD)9oWt4*k6OX&^>Swew}_fS%9pZ|cL-W=A1O0gwUMpRE$(ntxqp95Ak{BG3I8 z^$x5uUdcLNeVJLi>WnDkhxEyimz%+9@d+}=$*OT7 z{ZsEo9J@`>7VfmymG;h%t5!+D$C_G(_ea2zUw~s(sx&@I5bV2&C<_gC0SBh;|B}UuIwNEQ~!4=wPOHL2d~?r|fWj3Ah6WsLBhU5J96> z+VV8t_O(-&+c4OW)c(~Z1r!466n)K5WtUgY(Tox2DhWHe*TK6~#+FTyH9?C&&g@T%NOQTK69$J;Q{N%BD9;uS|L zg$qArD{2up|NWudg%5;OIGiq}Wm9nGr_;8srkmRpCbK}GzcreQVT=+joSNMzU{gWO z^uUsEtI+aGvDDpXi>DAM6r=#fYueW0>p z-2}y9{rAW9SXEjnlqP|T7(BIIhGN>d~mJ+s@7;r8mh}vdS zC9TNBIwX*}+mbVcJp)crnwquc*{i8($4l2ShGgA)-?leRbyt31uG{SHrAdi8rOgBd zC*ET+l^eYURRGlQQ`Um()oWWXp)~+c8y%S*`ImORi(bL&4!vSKBq1XbP7w+f9%G$y zTvK#pBU;B+jz-(OFzSW4c3DnC>+3}gX&16M*xEn2W!&Oc%qCfi6PE<8vvPtpwZ-*a zH;y)(a@`BKcQ+3XZA9;ICj)L8-qUUX3pkswfj4u6shiIdn(b{ZR_+v=K5e~rk!$$47>1t76V%b_Fu44Ozm8vH3VO~E3EJelLd=z)&c zSNl2pfHP^!83s|KxT~^iUX@)MyKt{~PePJ<|9gK{Q(wa8!F&O+%Q2qg>dk&LEi8%fFvWC@AW!QZu7;C-kF!2E9fnTp6x#1F<6&@Dk1?V zz7mH1R>Wie%+d@l|A5P3jw@g0p7zmvt$8(n^SqU_LCHPLzS`36RPBOe57W5aQtOm% z%k;5oUAMp^(QDL;>OG(S&=JBVGIr0bXN@A`Wnz@@pnhklzSy8Tb*2p+Qe!0qQiFlq z6n`33gkUl0jQU>7n_OZv&$W@Tdkf#Z!zG{88(0|81y9p_E1srQWh5J%0W(B4Hd?8- zkiKAzkH`Dey<9?miBn%3@-ScytnxgWq{PWH%x9-^f664#eDtU{(P?4>uv$HI@)F>K z?5;3|t3KB+OYA3o!*^kB9~W>mYiy&vki23X6^YEF2Ag)xD#l?k7mWmcDDV z&7b`xslBV*tKl}{+I!(6B8jtzUJ0xvWed$O-v7A_Bn34wZX_F%cdLBswL+=s>u6+B zcO)GKej#u@?#g^1{OIsCahT3<7Rq)3zU53vH7~yRmGl*X(nta4|LLKP)ejc9Np2VDhzPkZ&1g2YNEVs*W z=g`U{!+zm-4BN*&=s0aGw~xNXt_+;!9@TZHzu zC(h0BwFQwO0v!9wdOdfj`=Wxqah*qYT6UbC+4YasuZ#LTp3S|ejRpb^F=2RN^X14)3<1Jqx=+v28$9UiHkCaM`TG z7E_It*ijvEfM!pcsol1TEJoC!@`|Ehz3WW+OLRCZ_zz`hDg@OIxiV6Nu(rf9weL-w zL@wz^S-MiEM{n)D-%1iK&Cy9!%jy!xl}+X(`uL)Q*|oZLmfiZ`Sz@=Ogm<)iG9}l> z)6!Dch@R9JnGNYtKRz=^O9!d&5^cF%be1KtZNBvKdX^>8gRK5`@zfF9b*HgDT~wBz z7kAvOv^A#CgT|tCcPFdX;use*`MX+c%ASHIa0WFG8+Dv6>>d&`G~EuDCJ-dyrBtrg zTK1?3Ji|3K`sTA<0@0oX4m;{vTc<2bo1&Xd`&_xvSrD$+q2hm_QHXG;EHTvz1Cm&QLm?pL*URc z)yn=h%3HLs|nls=Q#ciy^ ztWZ6Y^cdy$-9Cr{z~R`@!BZ+Y#?od~WvzunY}`kd{$kg%AcBgPrbB@VrHDERhwfZq zACUNbe_S*z=fK2{fyI*vQR|l}pR=^175uL1*$vpNH#1cU}+uv3rxXh<$Y4W)=8DiD8>aHV?s{(;(2ZW6Es-IQz*_{l|j3Pt=4 z4|O~Q#GyC$%$><2=UOX3v%9!)y5$#iq?y^pb-;>;Et7~0E1dkS6_JO?}*J2XA3pD;Rf-NSJn@=h;ta?&}8Z z6;_PVh9LARJGpS$b|6KF5>*`x>}g%ZP~?MqfD6dW@8xNA96kLNRDLv(aWPoxuS`8< zm%B31MWiIt^&Jks^NU_S-%xJESda{LFBTkj=wwWJZkmg5EYtjZy>B$S!eYN?=cX6l zH85$X9iW5@7Eui9DgGJU95@M%;%&cu+kNp>%xATT1E|xaz`izVlc$@tHhtmbI*?ta z0#}A6yX)loD@yk_8j4fEIzXtxlw`Ksq%0?kG|VZ&?W%=-RCm@i8gYS^KSBp{n}qry~!gy%>l!w>iR z)l)&hflDcCNop!jv7wZS_lPWTQ^f<9_ecBI(KX%?ex#&ER)0ak8L1A}q-{i~=T|zs zWQ#3z)p1KV4 zHZN2s^DqKW8g9DRKgJN=SgHoS9Z%7Edk(U=T$7zfQ{6mH(QG*1Y~KdKHr}ISVHL>{ zTe&%Dl9#$}x>43)kqUJ%L6IeqvIrsp#&_MEm=kMy1{Kkq@Kj|CG zdYyesR1qXn6^X|#6u+F8&y{?Jw$+!o@LcLK_>F-lf8B#tvDi1;>!IQy-8OT!!y(V| zI-OCTB#q*GNhmlX@btL7o{OS;2FHBIkb~@;Dk8i5=s|qRdD1z2!h2bZI_bKF`#6@< zdxuHlCo4K5}BL6Ubv4}5Nhna$3 zzrfECelK=xb8kCh`m4RZF}smz2(rI=;@Nuo+jd*KXSyrr(WRslr?hb3u)Ov}hs=I$ zte*4sRC}%%n)VdBAM2{FphHOir+rp!z0^MRrL5;dABH4dx=sN zCb=yhDWaX!{J6JNwD{{ib&spF6On5bXW3LN*wkw4TcwDr4Cmcp_{8huqV~AY@zaJ~ zlTM=TApGmm1-D1D{aj|nE$*Z0ROGfvtEqFprY@R&w@6l2GKRceh`UNaSr!Iz9w2bf zTqz96ei|IB)lf%-R~b<|%AUoM3Ane<+vS_O;5%@=6S11UzP0PZ@WXbwKDCx z6Z=Hgfy%#2x5?KM=2qnzm&Ek}^;))L!$-1Y&vdZc(<&}ymIi9x|MQwl_`9N@PBVwS zNmYs3bHSpnXl3nO^HBRx^R$b6+gx7L4@22TwF$Cmib%I#X?$OJm3fvZa^y@Rb z7bl7mgDEXtIQ;&R=W#9&!?_kF=H0%u27N$+F&* zE#KD{RT1ULRM?#^*A{s>LB+GPzuUwzFRW6U$$@>Hj9ydjTkfFVi+80dS_WA;-^}-Q z+RvML>E80&%9T7-iJUa@Tw&B`9Z++^0=ly$AH*8)Fz zv)ABqIJ$)5T?AvXIq}J$GV`{~_5p4Iy`(36yOs}9AbVLpxVT?|VVTHg#FuVc9bS>o zc^_q@uP-$(of3v%fx;IG(o9%b(m~RXc^8^{nFN z!Tz)b@lZtz>--`EbSLOC&G^ z5#LTeK;TslQS;hep=gkdJPJQ89hdvSh24<~3~7j9?1M%4Z^y(#M#HD$F-TcD%1Ab(AbKq2{c3M$D9lYS>vr#R}kYg0*v}5 zYq=UDt(F&K;Mcfn^)Mi-_Xa;+$JWt z!+K&7?is~n8vQvYfylyk{ZaXq$?uxXE|IdCMn#I)a+?y2q2uhgjR8EoB+7IrUUq|8 zb&D=v7MOLF>JoAVPF#8IGLpGQ~FwGG~ zecnh6Sau=;8^of~6Iur-jB0`ztt@xmei-Cr#TT*M>eY6J|FjvEhie)A$@&_!7`0b4 z;f+vYE}oN4je4Vb%@;W3+Rxitj8krdZNE|vdjQ>U%^1?ZJ(4)S@<7vVo~zkEv#sw!5InpNg&}trCtR{>>9~j=)%ln{DbL0aq`Tv#xkr!K`+i0 z+~xfHcn4{I-Y?O(b`i1z-8ih4C&a%2=f^`6= zuZdKuu4G7FBacIJFsy@Oy71!eSw~F52os(w$cj>9xwG5;>l?-DZxl~6-=;?_FTemx zg}gU=_@i7yo1c~8Jxpif>*#^q65-}n0N3!gOZCur7RW0RfeJiX4bIcxLmJ7)5O`x z%0B9(i4DtqU^wM!9xVt_Vfft=<5lq*1n!RA-?aCeUo{_#`t`S_GpCj2ttSKy-n?)E zsNsTK`Y^3-90*5@o_FkCV%5ZKF&`Grn8mA>{$Q252q|8ur}>hL!LmHe7HUZD?=@=R zW%y_1y=7k+^Fql$6e}^dDOrMk@;q$yme5Tv?M$#RDwR)=_IQ4D;)0BnZt$(Il>^V=!co_&R z>~!Je!U(sHgUaHj(;cZ-hAo_T=@9{aS@2KMq;@kZW5Wu=GRGn{QF_u$F!x;B^ikdy z98GcrH;nflSL|}X&1fShJ!a_6)u)X%6L0ImdFy$DZA|P{Qbn$vJXV>_A-0l>eZ88K z4&g17S?Y4_d-(@}kc)EPDiE4AkBN#ax$G`mP%8Q($ydWz|F-)sRtKXO?5J1$(W&K&X^TNt#P)lusI=6DDnR}_c|AX zOFLrGL21^+fpVhty{XdK8t^m6nlV_cyu5K>7(0cLxRWfSHk?Ex7Hg@Ni^b2uQ9KMs z-H~IVuwpXVO(;x|b|WBZ#A$F^)IoyR*o2mrE!Xru863Vg5o^q1@lWRL_C|9e{;YI1 zQ!3-G==iY7TuOOi(QYcjvtteNgi-N$yvcz=P$GK;vYCy5IhU-aB1aAwyjc|(b08D7 zZ9dJM&nn{=hm;#`!b!|Lnv!E^VXUi|_1k(eR! z(c&67Y(sG6J{pS+-~R|pVH38+WBH%H9v6Q#kRE^|GzeSE@0|@V%VCxvW~>z4s?Fa2 zJ`lr#N}&_BB9k~1CKx6tJZ2oxOe)t-Qhf3qSr01XK1~e&&j> zJ4i4Q(Gv@Tn5BDjx-k26(dzY$kS4LxN`ZqU|2}!M}*Z0x}+e@5d?X5PG2LanH zlM368$CyplGq$IyIThRGTAPhRKpLb+tQkAo$3lfUR%zSF$OEbce>$=&31RhB{i&y9RAXEhFC>p^4eglYyGy?Eeh5A1d z0mYn}dNAQ#6rl!&7CEcQnZqakdI@gm<)!U_wno-KXFi={@cR%-$e@5qW3h;2PNPAS zM=dbQ!f7FMOud#1pHEAA^mxWAm>m<1f@#e6WgpWkSXfSXXbbFA# z;%#UMQd?c~+$RwzR?*@g6vb_X^#?5@;j^B!n4vDL9}B#0AP zTXXY>v!k@*R;aiU^O?gg^%zqn~Z+&KFY-{Y|fHUszo44ekmP(icF5OZHh z0)6T7ebN!h?l!JL5JBGppWZQPW~2W=(Ap1S;6ma}abyObNen4Q4vpHKU}FBr^bTu| zFs~0cUq;GujC8DV%yaa=^Jo^!Zj7WfNDm|?UjKT@2ib!ZMb~S@z>~)T^`hGb0ATzd zXbINO^!xvn$o~J9xt{qm$Nv9-JYv^u{2Abau3SU4oH5WOV0gu$N~TqI4Im>fzM2_u zL%UE<7Apu6MigOv) zWv_#xH5f}x_I{mA7f8lfhBrl1k#TJ`13HMNE+M7%?iVDFlsd4U8#a=;a)Yj2+BvObIFeSGCWDiU2!1D`V?_Gy}uu*9C@Nl~D7)K#PCI z1~UUA9TOWTt2QAs+b13I32iVle4-1V9>yo*z`@A&pR&h4=7WWkjggRzoso{2gNXu$!c(XmSJY1V`pUHAY|oWq+?}a{;bIOxq#!}HdxvJ zQ+D{LWPG9#te=DfE9YnNPf&u8iSrYC_$P$?>*c3eL&wR)$oa1hC&TA*CPG#=*3WJK zM=Sr!AYox*r(?I2`1Kp<|!AQu=$wtTWk0<$8>z_FB*#P@zakkH1|7+{>Nsi!OTI&$@Y(>_-``fKmCsXC1x-(eeTxJo$>!B zGZ_AFRt6&j105>|E9WP^!Ny7VS(EwGb7A77)s(~*uQ`ENfro4?EaLG&=z=2hNP>p-XxP8C?7v3P z68=PzNBT9LY}VML=ZwlahFND3E}>y4IxhWKY=4`YER7-)zGgH-)2U7t#yI<)QQEo9 zl=|e^dGvP2J9Yoyl%WYIJ)K`EQx^DA-3^1JFW@O`{9fcZX!GsQE6$fo?caP|w?m!G zDqrB%5%hTZy4nkLc*^SM)Ziu@78aG`*MKXJ-}*fe2shMtz2(Q|F&;aF_g21;6<1c$ zAbM})BVWjU=RE;Tu4SFT6+P(NDPP34lnq8sd^y&E-^OHa>a_SUMC4J1Md~CF_-Hk_ z`Kbz+>ZtraDYoqCUx*2TL~hbcJWwgtkIc-PFbAJ2(AfsQHzr&S2bm@#A30 z@(QjHh|7|);Q!0?;2tS8Z}SJH5Kkn0A*5K!xHj>i*mX7jYvy;VAAWG(j-w$I%`G}X ztRpa_gefy(<3C%m{jPRUcesb+Kn$_lg09g0V{$^gUf*e19x=gcpx0R*Id<4l$H`q( z!XYzWVMzId6x%|#W&Lpd$f0b=nlrLP^pNvR9ihDwwp~*VQEYvv3J~{4oTCfa^&;-U zrQ_Wd+_`(^szvDvp3v2Z;Q}EGaI2-Z6uoI5{(eJfNnV9#kY5$OK)2w}VBCN+)SXC!2L$f`arfQkUWZ$=pK9p%TE?KZ=_Zvq&<|^L3LL!&x=Z`+oewdytPWjnyEm*j@l2>GB)Z2~V0y~|d%@`*8^f{WM5ud#|oqYsd{ zGqa`d`HgMqD^b5?n}gn1=Xh5)*RqnrJ=OnW%A*Z~8TF(K)vt?xRP(brCc8GQDa`i< zfd5C+=RSxXnSLnWDVdt@oBBLHsTxg!P((W(3@1q+N^V=)F(yx$4(zuRW6Yg#6Z|W` zHa`L>V+Q|gECmR-1VcjQRk_C)e2_-DzdN+WKGw7xr=SC=SMo>7Mc^T%H`<=xNdP}B zFAgYU6~;vImNB`{s$Nk=nlBZmZ(e=7u!Y#nGwj& z)iY#20;o;{XW8n&>~uZUhl_$ofUF{{ypfz|$_CS#KHKPdz@S_lfzuwBA&5Q*-Nw|3 z?uLj%x;omaL8Aq$5M`VMvuA(OXw3NNz_c^#jdF#yAuERe!Qi9cDs{5D)>6*Io$;t) z{9}aPY!c)9TQlzPWvj-m+zXU#oBay|&bigf%Lu<3^{A*jua%+mgjH>sajl^#ah9={ zVLjmJW`Eo3SH-@+?>fP-4)+=zCtw8DAw;#-a9!XT-T|ebsbiEKe2Eg!12C|)ctbHl z8i%Sx4fSiOML>z-WR7qVwv=DjB<_Mk4J5io3S+H>z>Z>_gKFy4syqw%CB-PV`d!uG zL)IsMJfLQ_ZGQ9n5{$6lQZ?TRD@C+ZvK(lW@<$KYEJUb(dDK9^CXu=Z9yW$;Y+qbezx%1zc%;Q)2wTWD|a!sc0zkfM0RHm?Ftj=k((6oEFul^E>W4gMAK1 z350kwrL2Qe>P>^%GhcOxvpM5OP~wSLyhD)NLd_<1!__hcnQ~bZekKcDwYEo_IW?;k zsBB2TnFtm3n(>o8<1;O2zN8u!@Z2s z&@iX^G0{Z#h>^z%ss$E#J2%9?Ew}6k596EBhw{@acz8#x(#~JzJn~w|Mf0=QHRnke z@VEvYL36*n5Ao>+8a^8;74hqw2V*qRY~7{cz+}2CR?Pe`_IT4zf<`LJ4@EU>=0rZ)4eie|Zr|w#0@li)R#I zL~AUipZ!R%OF)L5gQ_HHmVHIKMCS8L!$j&#Sam}`Q;#uHl0i^GlsBG_5b$*rNgOt( z15IaBjD(h^jH6p7(`q$|{o=~a&0wtA*sVO?B;HJjKE6zur{*jGW_6alzZSt`CTB8I zINGT^M(3y^v{^huO%2O)wrsY8{ALVch9YtJ^>_1gM-&TXsp(Sq_D)ke)eDRd7}^YE z57tXhQ!Nh{CUt+Sc2(!t%g^l|CXz~3X*+K`jH#~M?a8oj-4-n-G&NSVUOaMNwx3

I;< zr0e?7u(ajkxOB;Dd-49Z_P60G1NhVh`S*FdFA4z{^c*Lgh0o`3zic0>Id+^c)guhp z``wJ6w8xA2;}Oa|uy6OlAk8K1`OkBtQ39DbbAiIiC4STWbCMKL_)Ll&QArE!AY>%uBxE2|74QLFhZ6XUl#z$=CnHsjra@GHX9#$RaY%wh z6bWy}{Zl?Lt`ns5tn;o71)&~@fMa&W+-tM?MtI@LeBl}VoE#7*^K8ElwcM_e2rHgJaJ9`9CIla)yrIs@z`C*fJ3!SmkeerOxh0a z9Sq#*(j#aG)9ulEhCB~``iy<@+uF8yhT`)<%)##TwcSx^hOp4X9}m`0`+}2=fG)(# zB0$X2^Yil@=bBZ*gp$VBa)(O|;^G>LCiv_G-4QH3Nb&k7et*=Ap9-8XPAJ6=xeFBk zKp#b(-%fEJo-&W0*bOHBkoT@Brf&5sXuui&jc z#Ao6USnu8}30BB!3=o>!o~;Xn_fQ>XgzS0umq9%m=RS^Sa*rTYJ>V^RH$EnPHX&3l zA#`mv6jdw3R|*CFpRM3B4El1?y*kM>$>6X*VXiqnd?}Z4-t=^Pr~;tbh_t4`+mMKU z8Bjg;eI%1{1W*uZ*ugLg(SXn(w*{n!;GR)rp*FS&af5}gDRF}guNiR%b~wKAa-fLm zBUtrn-5cN^8h(u&WK{cOXpb8?1kHg=u8%A`NXda#)YGRHPRfC2y1l@G>d65sRRcF> z&!)1CTi)NI7PwH2Zh4KJ4vA<$J2?cd7W~kRgj0=-p^r*8XkyhvS{+e(vnfRI414aw z9*A6xr{kk-N8FB3F--*L^S0Bq1Ml)}Ggx(pm8Iv(4!t{$v%qfr*bl^GYcp9KlZo6{ zh3$MWO|+u!&ph|~Bqei_Fh7@pNw8Oawxf?;C!vU@jshN_RK;C2esHCHnB>+13-ek7 z1abo6vzFu!f%2-Gfh=CCSmch{;wMC-Eg14{Lh+e&lJnYqdTL!tNlO>ES7U&-Rr|qw zlwR8@fE@5e&fWues6Yoq8m@zaViq)pqnL4aYnU>2Hl?YxaY#E>@X)9+&Uu+G>DZ;T zY|)5hK{TGVTe%fV*%w1B6bd|6`C8A=SfML@n>ckNm#(xs5g{d>Jo!c#Ty>!80HV?z zQ|@yKQm(8hscPIw4P)E4!X%rwosW6F!uH zl}lSoGGS=w0BavFP6=S!suzU8=BK?5>pmKRPb%>j3v)DbnUoCss^8az7~#YN8ag~l z`2s&#hW-OBPEzQKE*Y;0p2ys?=q-?m?bhN*d|rvIHrFxV(k+GCw`5e)1f;Hox2Mav zwm_edKlPVnh5#G*GN5BzM_sC}sk3r8IRQUnEg7;EC-tXjoudqR=EIUpPoF0|&%h{6 zbyL)G_KxsWjPw=IoQz;WmUK-y$kIhTglLR5NaLN#o^*3?+n8_=glS~iLY{CQ5WZbg z;jd*sW!g=cJo$Vrb`-7P=DH?%IQ}q{G%!K*mVQPRB6aOv824P*CPh`mOxptRzA~)W z{@!lJizy-!`;aQ<_o9So8|DwV zHa$Ey=wU!yo;m(sEFqA3g;f{^Xn^RT)i*vUu7&fsZXTmP8hVzipG6n4Wkb z=+%_iFj-XEShBvw1=cW5Oeb@}as~I3+-jYonqh4Y)MM`5_We=~X`AKUkYU?TQR|%? zG@5wOFc*bhYTSvlPHnSuz&Qh$2Df%Eu&p8*$>KYtpZ^ie+2jf3v9@-#QPt5@ z*K~Ea8?gF}cWAPvrLih|O02P#5)ZiO!B8D|x%+>Zd#mU;f?&bYvPBlNEM}I)%*@Qp z%*%b!Y=D#d7{#h zQhS@%OD@aXX{?^Z4o+w~86_ynW`O$X28~V3))hD|sm8DS90W-E;H$qIRVi;eG{*bOYZibA2FEX@)J} z%K^=m5uBrBGZ`|M+Tu4nZBFeiLVN?7Bgv3c5~qtWWEv0)%3kz6q68WRT7m<=lOsiw zjFMcT1lJ87FnW|-8s_DX>?rmf$hx!fZrth2)5u=3;n^zb!ngkhla5l*ztq(4eXqU8 zS?cZ7bF9Iy_ZqUAQ;u`jjk{m2X}242eg7M)OR@ZOve-HJ;LguBQ1`w$nj8Jyh%DW8-3ed!Y@Qsn%QiNAQFI#EW*JBnMzN^g!Dvh^ z=GoaSbj_mM?{TS`);@*??RC8L$PVwz{Y=@wm++dn3j5zXO_U2dzrBHlJ=Ja{Ub3a2DYv6ds0D)RoW-=T64>~ zXj-8{vK6Lmk&{-AAr;aVYYJk^1tKxF0Yg`lDnwluJ<}wVouLnMh^ou;jAt{suH^ZF%iXW!)=@z40@eryvwFoRY1 zY3{*TVua@l(aSf#*0&3KX+ThzC}Nf)l5~G9c6G__Z&o5;{RSnnhI%IX{UI2g*rgij zI8|0!07{g*HCG0!kNUD`aq=qE+$kWt4q7P!D}--c!8DplxI;{{vY85zG7DWfF`LT! zpu(wSgLIpCn}VMxYY@#kLUp#9aS6>jN)<(>FwhL^Mukzn(1&_)!phs>!%bHm<~wJ{ zrq#jd`#FX*^z~qdOzSZv}aI9`etYA6{8*dU*T#tOk~_E()?bfYZqlG}{5T@hvu zNk8xyph`L`pk|+Y#)K90$+sI*IDL;c%dvp~-@DH_!OGuio=X8o zgKhS0@SbkaTiKP~T>+-PLGN5GSbS}%)F3!2Pf*A!ExhVf1(~&`R>M+tixo}_?;6;7 zu-Tl+Xo|BcnI^ITqj3N!l|nvs=J?%AbBA+Fbnx`V((IW9|0t8 zuCL}px`3B}G6eU@RQKFp@n#1O02|zm!wg*Urd5`5qEGObou*zsCPrn zyuQY%GH(sL`+i1whH`_r81!}+N4zNqdC9}zJ|Fz;8*4b`0x0skQNLqR%r`|mF(lpD zJi*rspe=YfbyG(^aUF^mE(Z$*vv)}EG-8ByRPLZ9hR_n}uaMv(sUQ`KuAV|OuZm?& zAqN^a(+2azZA3-zk!@Rw0s@1VfS>E2PW%5GOKR;K7RzcZC>#X7iOn$faM39 z;t&|lHulwXcW&z|Y^&BR@kxhLEj9On38(aQnrS@nPGiWVDJr1|`Cw&^Jl{FPquKyD zz%hGnh^Y;0yCQByGc-978>L>Ii7{6$dmS$_(!xQ`3mF8Js^clUGh%VZDt!DQfNICv zRgu4JY)#@!;GD(t`lBabb?U$cZs7z3fa_!i%SyH{L>fh*!G?sjo9JHztMFlU5 z?KBF&2No(Vh?p0~bM&YRl6ACGq~7(RUcNQ*F4E6 zI|KL21G6)Re^6~-!aN9mI`h7NtxwxkT`qo`0Ha8u?^Urpsm->gV8N< zvV<*oT7_!ajSAMKt4j%bnPZ;y1Az=R#0Y$XtXQWWvCx2` zhr4cni#xY3p{h26^P{12J^;vpHt<31ce6Z3Zwtr6?yPPnFC%bf@BvP{|BZ|^PGawf zLf*ToQEjz6T%n3qIk;ku5+j(1oJKP;zKl=?UYtmuIu(3>ZE4{LtLT&(>;g)QmaU?C z<=+L_O0tC^Jrh3v3;Uhj`R=u-6z!Q$SXyERx)V_dd#JN^D9tjUmYFNi_au`?gJE;a$nXn2$_x;Jq z0Xx+6EAAs)816GnSI(Piz3a;}iI4XX-n@QB*tJZ&@Hc&eWBq4%NoHOKSfR(3&^xHu1wh`>`zMa;y~hto&Y#ijwHUJUZav>cg)ZcG|$ z3Y!Z=6)o95qdGE&x|}VI#q%9sH2_T9gB9mO`bVM4+-_Yuvu+SB5wI|uYDTLSHpqca zT})8$(}fm7(?^p|m2-*+OwIcyOx!TUL7Mq#8xaYywE}e}HN7VZYwP%DwXM+voOE@dxp1l~^N&d%lrzlY*yt($GNacE66S7O5Dq)JYtVNr`z$2E?{ zeHOB;Mwy{fwfXP!b>#3e31(3$CQ%GolWdCsEdA*TUD`&{70&s}O=yNWMK{NlM$Un{ zI9Tv;_Df#k4ehzad612QpQpwMBAGbIeSs;jK;Wk06=4zF?Z>I)f(>IUT9!qpTJw4} z5MixhGu9vuE~|WiNwh7~HH54rPNQI|RZl`|*W1`jB2%W5xK5+qVP&CW0Qz^^Uf#Yt zhm-@kZg%%22y}e69=Ht?;o=2fdu`$^YT{3dmNoTw>1P3RP@MwcvP>$LV6ekWTqiW} zzS541(Zwil7L!z^9SttJ1Qr(jKKZ-TL1qGbhnHbmw!bSkx#4DZJ-g19MCnJ+)wLXG z4{uwnd>8%;SDwE%#-~xob^{DmPl_m%Ycf)nb-_?#KPUu~ie$i&lBBDY^Dh@D;d1G37>M6nOe$;sW~e8Yi^T5$8LFDEYnnMIyM!<@u@$vos1wjRO&vMqhLI#% z0A8;lsOVhE0rBENG1Tl+64}3rAgWjmY$7BG{>*fWi2Dmw#YLdGW)w}6G?TW_YM~$p zuIXxzo1gZD*z>Bs2^>O_OJ2bxJ>iG@4Mj7AoG(au2P!#Gwhc0KE!d79nBVtDfB`xH z0R9*^-nlWwuIoka7TndJoT0#a?efg0K3A0cvWghcqNdg4_^Mt-uK_oygN0Kx52y^HnSD@8I zvXjJov`g>JI3MDf!lWcHJ*#mx6}MRLl}NO~TXNA~AU7 z$Td{#oT@%)D-XfiuS6Q4*=PZS2sjcijx{7z=1e-JPRps~pk%LRi&g%lZSwD$&oX$_?cp`vA+bu}q4(+fWpim3GwDW`HeZZ61Ttzv!B zcZr9=$US|13*!|wnqT2f>2?o3<{FTwkar{LWF$giW?&NbIy>$A435G4t)(1az!Wkx z?{$c1W4_^6WI0ao0p9X=_5>4?NZTMuCc~?0bDPxAK$}P=vJ4StjkbC|!@~5hKMA2# z7>%7dm=MvlNP|=E>>4x*LTWV5x~ikT6#zbthL9 z8mo4YLmF59+RGdE9rc?w75Re*`q0l?JANPcRXazK5%m6}*Gp!nK|bA&SvwAw_ag2G zf80=`GW z-d)Ss`JWkP{-!%9?K`@$RSR_mV|`>2!IvDhioNJB*zMrmv(t{Kjty{4aJWQ}dF&2a z;tW{r(BH1w1LiY@j&!h$g5kcY_mQDtw?pyF0*g8p86$TP`8et>j*rQaV%8lduD>z%JhcTl*R%?`@lTYO$I z4~{wO0_I`Hx6D2G>^tj3&aIqlT}2W|Mk3Ex;?Hr8r!xSES;$mS=0Z`Wlyv@&di4DtMj0_vVun3S%hLSstSoLEh1Vsmp-SOsb zRX!dQQ+|Bw6rBw=tpBBRcEjR)avt|E4z!P=lP9W0(?{7x(=WKJiT~!RfG8c_8p=_9 zs6xe3#NP7F5SlmW={NSNHQ}0AU`KZ3ludreSy#4~6}0x3H%k-J1ptuCeoE|JP12UE zCF<$>^T!`7a)6eMo-7Ntmiri)XZr$7kwx#aZ9=+9&Lyx>u%$_pj@X@gEB5gn_*_pt-IQdLrx*f@9pr-2xc=(&7?s83+^$M%&Yi*D^%WY7##(nh zPsFfVchWMEf*^(cVm7oZqZ@AMV%T%rE-Z{ekQe7s?eOINE3l+q9jX~W7p_w1>NdTC z*b>xg4>L;fLxr$;r(RFwt@Ui8KHm#CLX{nb6`ISj?8)4kf5*&dI`kjSHjW4!vuCW) z9q@g=X@2s#6vMa2aNK-VWP6LeV@C8VNa-c6oKThn+NTlF@2K^ri|6nca(@i^_hq2b z!>83II~@}z|MV=bDi*RFC8*{)@K@B_e?!bhZ$#Y@&^m~-SdrkBJc}#lN)^+jr7Vl; z`v4oCWf=Ki0Ez7X9p(Dpv0KcH|0BAko49Ee%zzYfd5hMup-1~JdL9TJbsmWMR?v#L zLk?t9_^>~_0~GN-Tl3fas0ES-0|>PlfZbZg!MkfVJ(NEXtUv{39edsVeBC{2+IjPcZ>HpjxeyxrisPuG&sWLNI z==DNLI_DX^+$!LKz2T4pdX=aUMg*4=j>j`89TrbI6(*OHqDbr^SByr;iNGuQiBL-y zh93ViY*Tsq34F@}d=O?)_4tHDC2c>C6Z+3m_&-ZYT^M>% z4;OJ|7eg1*f5(b55HkMLy3&8}od4Lvf3;$twf_Lge>fr|A=5u>^4~`$!vDr7exjfM z=ucTw6H7xOdyjt*=s#4GiIAO>>GK4_pS^rut25z0F7iJ*Hnsc2O08i23yl2#Khge! z|NM`u{XdfJAG`ZsW&1C#^Z%u6|BtR1|8EWcpDODg{`y}9`ltW@0bTw7#t#2yf&SUy z|4U^t{x=K${44&`692i;|0JCAKht&n^Z-^EdTC3OPk&=?|J}&R$imLb@;_j&DZb~Gl-+h)AI|8P@Fu3Ow!&1TvT}ZHxW^F5 zYzh(uCvU>{qq$hx*p}Uf17;1atv~&vtK-C#9)X4Y{+P#gYKA(8ksJ?$1YL)_rAQ32YUZWg7-K5j~md3)7y33N1WJq^$Pzxg(E~3 zz{g&}br#Bt>&IadZ7|AHT9MwRe>cP2YXJH4NblRh<%d-9L0s=!U2m{AQ4l0d%iDFE z|I37bXKnh&ErI{b_GNEq)w_;5KKE1#xNN(hP?Trcm@1$-kJ|E`+QLY|`QU@t@Gq3k5dP@EIfbrphG=vTgMzmY<8^U zJ^;tIAYbv-C9^8jZ&DR}s@-odC8#T;=p$LDiCz8oo;<9j$KS|+){ceaSk!U(=nAi8 z3b9OSz-R}`l>szv3V^=kX zpP0=QUfG;F*Z3bNv}MkR%3;a9%3(p;5e}Y+c)Hog*J6RcD6gYGW$s$y79v3@r6TGr zF}GYFLB%KhtdTR)K3ow!4}zf8oT^zJ;A=AaOvs@WiPuQ3)$56R&BeP4SCpGyAQ?=i znx7H}{~uz$g9ZA+@3rAlydV3AekIL?dHBLNtrYsZ6brYo$tcwN&AfqPjY;38M0vcW zXIxl?q$&7i0kaRY!CPtB%Fh)n8ScfL;$F~^T~nOwg&iBV(;*%go2YzPZAYYvGE3mngq42Kz@WAPiYcQP~pzx%hT$p{pU9R=$%fxGx}N{vQO^GtTu8PoyZR=Br9A` ztq{GBgh3F+OrVI5&7SK^RWTO-{pnE>$ug@&ty2ea58y9BpaFZrMvDzrI_c?P4tqahV1PQ zY13Ncg!MaF-g22l=7VtLvBR>cmI#cHeX97J1?*?fp^Ny7I#$_o)|YeW3a+zj#&%n4 zIJFwKTaDNm6jV6?^UY}g_bkJ!<{1!x)cmDf+nOwdij1#6Oe6yvQ-*-A-%dA8z{`ok z(K(vrJmHJ74K7e(Z1`}lnYHu2q~4R|96Bpc#h$xRTb;-xQ+jq(KBO9u$7TzkW!@N9 zYiYg0lN2n|-DXT@xjcI>%j@c%jqsV~9`qaXEIfC!=|}NL65H_wQSu2*jyW~DUG~fI z#hNkc5R3j4Uf6W=cfWm2k@Vj+Q1*z1FFG+~=h7s#KBNt*;4w@RQZJT)=1=1yN`ZB` zI;kfTl5@9h@#mR;H-s#OciXLOw$gA7Z{)Iueq%=rPBm(7l}Nwk#j@xOt>=J8wSBK3 zQuWUEvR&-Lir8xMYGClQO!gF^m5w01_xGya8(lqsI#k5$ub0G57ZTP!YY?+XTzx1=TS+qFoaP3Q>?kiZ_n%j+4>k86 zR;(X%&Ks7`I-YIoGW5Mms}xHa&6uil)S(@4)+NJp2RmTeW$Vsz>N>_waOTy6#4xkbyKP^tA0{LZw}O!EE7E}LrhdxCRHhMUB>IY%W316YrLdM z(Pex(#WqcU8s|pyA$7R`MRATj(V1Adj>84g>_H^-F-ol`09j)b9P!~a`~h>II5w?9 zqB8GtEgS&=jVxxrW42R^fQ~;!rSgkwUlEZhcbPFR-j6^qR{*%>LUZAvlJ|-QBE$?e z{>lhbW47aHD?M5E3vG+rAo}WY6Nn-7cXW>}ZUY?x92w}@K_~n?CFYQT6yAvS3g4Cm zfG}Y)pX^4fyDDGrE{(hOFzAFsJN>YL=%-?E97JHb;4)Z|>8*o|&48o1uH&S8hrrq@ zcJ>zFh_-^=a3lo=mU9>h&qU~F#W57eG$d@%p*raE?5HEC1J4%gQ;=}nP}$Mh#_3w`Bc~c#T(F7EVuefD zhda~o`sa@!*Xoz*UoaqWFb*4%J9hfDmU{-zYm@6pB*g>Uowz)q|^S{7SgFtEpq=ZT>?a587~g*#j$H&n+5jW`X#)3AE-sBhGxY$(^c3^*Z4Fayh zJc|>v-BW(gl4ZWLaMM9o z(mGH_u+~7=@q}BIz@p!f$u!+E;fU>l!NNobzIhKbb%1p!fp?@Skp$8Cm&JF|b?7f+ z++@ezeY*Ie&K=}i^N&Jq1WQWaOxC&QPLf~pW1D87jj~O{dP}I&o|eU?KPK?>oHXb% zwkrLiQB^gdUz3e-O>4kFgJPY$=+T;?gPdI(cF>v3a=kGT3(SO zX+?MEuuF4P$jK2%U))iF66x4`LPoeH@6I~M`R$_$qDf|=?({QhFwzJTY43>G-Lw5hrr5C2Z##fyI57*!$hTvG z<{=S9yl5=gsNxTqJDMXtUkqIcV1q- z@B~ySrLjA1yu_cgb?w}X?bl5$t(&<4N7*7_2e_48500M=j>C8->mF)?pQk42_)L7j z6+657GD(zUiJk?3g8t3ubzO^y)(gOiJ^|sLzph3KQ(pjF&hgTBUV`#*Q!*?qgt8*EY^NpuVtF zMSzCEqKE$#zO!FZF>cU&Qp7K1m zIm2F?N(iTVw`+aGK<53Yq3C4;JZ=s{f8aH?QvlZc4w8{2W!Wt;p zWa~^tb2?q?cxdJBHpP}E^822?zEi<835rJr7xe@{Wz9~oi8L^^0zUB$#EH-=j$+%LR3nm9= z($%Ogmz~4dx2}E8_(`qSVeX`8GuGpeBMxe%#b;ZR72Iz>IY006K#H1C&gukv@2TII z-&aj8D4E^%H|MZ8BgeR9V{sj+kdo zZ1cM8B5CHt`)RVDB*$}she|BI!&{TilcK>N zsI?bX1}W3sQrMmtzoa5t{zeyClSQl&Sf0~3PH@saFR33(^!!C2zL0i!9!}ET-61ad zOBk91$6~hjV^3^3`X}c-d@Z-w!Gh!$ zw0co@T?mCf#f^?H89*(VX`T97;wnxvH;R!wE8C0rFWj#1X?V#>7WX`KAuitn>)hS5 z61V=zgy{Hz{lwz6e5Hzi((gPd+lJB_$33S+F6CsaSCgke~o zW18RUu;T8EbxM2#h~jGlUtm`)^pV-g1$-7&))5Dj1J^-)x^#)F5Sx&?vxiu`+c;iU z9{7`IY|*B&<={JpWh3XZX&#dp?^7- z=8ydtw7xf;N9qryu7vU{-fZ+gyg%Le5BN454ZxkKcg2d7_`x_3jxeO$HHCgLc4$j{ z2sPG2UQQV9H|8rk?`BB6ZJ2Q)^pggP)! z0kIRh>ODJmNGt$P6r|dysr3Qe-C9cg2?4U1Y_2fK@_nX_GT8F{3~w|30M2bsyNAWm z;y7_Zl0U*%IAa|yYDYP}=KSZ;I&y*L9aGWRQ**j6zXtYv(ib{^AZzJWe_KzNtktx)laUAfny;tnzS4)(?+9cj(CDu3U>w+<uG+l%+!RO}g4>?|wZIw3;{=4s1vZ&~kvV>SgCIno}Gl}On zZ?JLs$l(mc$n5B%wl{=C_P`8IOm4V_v2t*ztFV~UEIDN1ZbuE*&BlA83by{Om zJSKIi3Dl59;)flHfnLY>0b80A?40Fy>~luw`(h;b zCuOe_(zlig4r~W(v3IOI@q1~9%*u(Itm_;AIf-9i^PH)?*~1QM9X;sk*~MI*s60EY z+5GqnMs#O=Qf8x<|HKLjg1G%I>v=97y9aK7fYn2#!{Kv4L)%r&dt*$|WXyVrvHi^7 zfAwblP6alk&9m@=DP~YD2LFettSIE^r)w zT=mQh^_{_F1&cm+`@fXDnHDS&m)Y7WSE!7)1K8M8`J+<>)>1(+cO=`9c!k~>hJ>Yk%Hlr)h6&X+Dvz~tLAr)Di$sEV^r{gS0##yw<8?Q%c_>T zC&`9Z*$*0Fr3R7`RlrwybAQPGg~c4J=sc$!cD*C*ubiH)_OP79TfPDDhO<)G*9RuJ zUHOU_U#@}ET!{g z)QgG_r3hMW5!@k7B0$ZTkVUa%F+ylzDIFbiaFWV* zI##%iY6l#N<{HCr_06C##gRZUO;UNTlDCYWljLcRa_&U=xx(qM<;K^|(!ZE%5z7R* z)`L1ph}uz-q=PUD8b>?g7qWBfD#SmZ>hK&@%c2xBr{(^(XPLYvxKDgMT4_*@HFfE{a z_FR^2`CPAGc2=COI|+=O1hH=Oou(3MI^M7NO77shDl=T?QmZ8-iF`4QIC z7dlttWjl?pUt-$~Rqio=@SX6`p-=Rv>8xxC$+ac3FiCvbOqp<4<6+Kh*B#;DB17%; zb*h6L8I&Ho(2>Z=%(e9PzgPl$uoT_zpVF9=CaEcMBS|~QndwNL7pZTn=O+)Gn(>eU z&&qQ=dR%=a<=6;9!cs~Rqa#!IU$s!lhhQbWbp)1xxQ2dL_O6T!v9gstf6PjF^l6I@ zzkki6W_rM2noOHO;#gNkoR`(@K+8maPFgBweSzXHAiO2bHrdwqqd4$ z0#mkS2T_D|RRh^##{+y+ypss%J^(Z437IVY))Wy;`S~)X6TCkK>?yu1zPGnOMtRe> z-$}<^l;wVg6*K`Un;-uyv*7feFKEm|tVj;l!EA~QyDOqQY@ePvY@#DC5;~s=4WcdR ziZl7HAXW9Umv1pa`(*ErN|vH}v%Sl?S@L{-YjkJJrOzHR`_sL`&m5PKRu}|AI@+4K zyrgkt96Y=<^#3aS!14ASF2g-~qQ=GkVD?Qp`oep1`~%K=1sKeIX2=u`Ep5T+i39yY z=-b=U`kqyXc%l2ouHSpyOCK;Cy!Cu3M#8%;uU#D@w&>EkZp7)lY1_bt5~B9?Hxt2` zMX$g1BcVBN^2c<115S(z`|dpAy-p`JS>O#mMNFk1G%bbtPan(XEvZ+B#r`xgqFMZr zk5;!7N_Xz}^EXet4>o?}gLh74KW}A@pX*SHN1NC_LUusHmkd?FCICo3C4P zQ6(*iGz(I0liLK*&N3l)%>q}W;fsWYj}(rjZ)s{D$BVCo1jo%x2pELC+6Ed5sB1CR zMN4hrE89=*mBoK->BTp;B3-Y-or^1Z4A=wX(Ro`W_%64 zdJ(~MHtxq)I?q;5yH~>065Sg#p;lE*`|(Xvw$+Kk(`-p@iV!i$F%sc9^x;(4VB*on zq?e@eh}_bzE9~fV{0*Gc5*WkO*T$C!)#veY0r}PHO57bE`Q4o1Z!!83qos*O<>h*s zx^U-pket;g?ATmRoUvLS6IEWdzTXt>EjpdgaKia~=W~FQqV$?Ii|arqU4BfckaI%? zJIZE2x2_o0BDJN;lX51_-A=4cThq?niG=@fc-$y_ICj1^+S0>>J0885An(dcjhFo*!%8n*dim(R|E zsK6sNpisaZIm}ztFo}XOMTmE4F>fwau6~SyYn2a%kfiK(G$X2Z*ad%89geXpL#g_) z;rv71wSQ^Qlgte5M@IrGVfxfD+nzvEiV)jCTL4^@>GnEw`;tBSX7qLAvu{fffvF2f z@aqv~rs5)p6`jS7CGP9-{-}6XE2EwXE&chT;23Y_jc`$3BE))4N+|*L4@bDp&yYTGiO73D~%L=h5h9S$O%3?a+e#4;V? z!}D|S9^~L+OHuQ7yw^y-^y%o~8`j5laYx&g=Q_^K7k>UVr$4{F`11s9M>!)i6sukM z9-K3_k{jx$(et)VvP@l-+>Z3^I%HwCkZpD7+7${@qPft0E9TAp(C1ulb)72(maopm z(i`5Pu5NjGV_w#{-Vw3Rek%9xea)%^=5uvY{@mzuy%n4ZGhOS`l8p{@23J#a2=nq^ z?WQB6UQPbb(DzkAY-3kA=Kf;ZS2$Vb0(%s%A0^QHNfQO!FEOvKRN;zv&6g|lkE(HVnIXTe zIzi+%eZOlsAWIJT+_Mcae1&E#>$bKf@;x(Lab}Zq%P+2 zbAiwFAbQy&L~duC9??W)86@g4JkcakWJnrh$3D=shNU-&lJb}$*BjO3JPFqXNX%eY zAUWY~qLgpck|gCxJ#RbVWe})|)?n4fn$Ls`SD`XoMx$EzgY$D0wC8h;jK!VtXYzTd z*)=(Ibxrb$$mdg9s!%i3SVI$05Z$Is3yso8q*K^3J1KIT!)j{{J>$A*=?-1noCFIw z3rF8-hqOOee0j?4F(Prmm)CD7V2&%ROLqufo^P8=&+W0Lo1a&=@t6XhI9uS$=P@q> zPu7PSd3}dM(|L0FvhtQcTP;S5+PGhfJ6$Zp^0L$D&L>3)7NV8yfj%l>xe^~n!k#-P zKn~Z2U0U?5wo^F>ae{Aec&XZX0w1>I2e7ovx+{i|f9&LLTf)HXN+qzr=Y|5N{b*kX zJkg!JRm*#rwR7x%l$cvJ@%AhzSUET2GcELpT~{j~`_*`V>?BEw9jRqwZnKDTtX@|j zw^0@COy=@#MR@krmGnY!sP6z4RPMqdY}l$fwVMC{e%L}N+0P=^?}{d8d%yG2#H8!n zpTk$33r1JYNFD&MSk1FG-*ibGtc$R!_WGoY_P$@qGjlisFN5LahXWG~31rhc;nKKh z^vdVclDW?%pUF-{!+xTbWe@ z0&u4hB~2&vCxW`?d|Wu~;EWtKO4+S2;{w*JMO$%_JGSWxBI5Ayab)H5YDkWvqt4lDsUl&k8rB+=7P-0Sb<$@N-Jt|~A=--=&YotctZM>&j!os(=K1I%I$*Ez(8g=O8h}ThnTU!>_~$Z`gv~)D|zC zF#lLzZ~(UZ$C(cP7U~XP(cfDaH}Z35b_d)5#ix9_^ULi_frt4Y2LI{K&zoA@oKc*x zSbsIf6@YRFqkoCDeIgL=Pr_)L*s7+BFFBp1vG^ib70rp2_LBHWl{7#CJ?&bS*Nca_ zCjyq;9!P0TGSffB#(f6jkaXeSv0g;2NAw*gnnBcblnLZ#yBy_I8f$UL>83IvZr+Xj z7OUc9zG;hVxwO#QdCBDr*3RNeeLIP@H?!H3?ffOR3nrp+93^dWVri8i9eYE6;+=G5 zVTTFOIx3*Pixx=JJ=Xt>OiSa6iG5gS4yjG{vbv2pnWCD9oTiG$2U`8AO`Aiwg*lr9!{t z_MiV^6S~dv#XAlbZdGqX{3~9Fw9SZ4@7Ef@^2>dc_wBIhneFH@*yS z~vA;O?0F|4d5|yq19D(=VBzRvB~P2Yo9auYst^=!*PLT`faHu1GNPL z%mx#?P><^q-PGm4rexAvY?0qaa3&7Je^e{}9Q-(5uOMA53vt+PX0ah;DdbWHbNaGn z>e{z5{1j*;3o!~Q#J?6ZCHH&I7$@fL?5hU%JtIyR{CQ{={wk*NUZa1}i4`uZy6;zG zw$^UG07m8T;s@|y8$YB8R2J-X_zzDC`1t8)00LR{zG<%VO<>z2a^viqrn0 zv)P-5Eaw8lqiTOse~vSW6qZV}Ss*`q9bzVXvc|@nq4{4IpR7+>R~^0X*Bvv-T*9u+ zw91(=|LUA+$c{08ufVw!{092sDsCdrXW#V)~jYZ`lbvW#Z`z89NBqb`2%8H+uHAv7M?AuEJ%x=f<;TQ){MMshXG8* ze^ta&K&Wz$j;G94)En6V-Nm+k#hFdjGxq%0-sP<+iLx3-Sxl-Sd{xyx zkehIT^!$TFe&PI0n8Oka@khU0!!K8cw@fz}XqUcEX4o+g?kUqx7BvofbUBbW!#d1$ zmNW*OZLoOw2|fFdUnM`{MG%&zy~0!;GYP>B&3G!o=-Iwm=;H7sdb*tYie=BZ#X&#{ z2e7Y04|%1W+{`XB^Lj^9x4y^c?OvWFp%fh}?xX1E)U)#M?n3ssM-T`G)6xMb?p>!q z=Y~O0$!79%Yh~Q3zgOb5Zo1>4&`7EZJ&GCgv>Tkxjcj6@%UfDKXS~;AQd}){`++VV8+<3?BwsyBJz=XE&p*X4U7H4E z=5!|JrMEypN4N1V9y@l(a7Wb53MT%dQQ6<^tpT`nysmd!LkN&N8cU~^jjmYMn=2zg zY?hLl8K8rUUmr5_Nhu%QV~0<#1r+XH{qof-r^@2w*{QDDAOa7|^9n;cyCep2Z(0^9 zrf%k3m`s2Rm^jsqz4ZJbM(y%OR2(pKt7@JpBxE5TNm#@5x^&v+|$*UW4MVG@_jxb@Q8W z%1%&w!}qS$95_^wOMd(Mm|U#LYRtI&V~bC{udL8R-gFota>b!)+BUC*O7cPXvcub4 z5v(E6zOC^x+?-82r-ga&w0JJW&ca`bw|72n7`16Ve89QrZLBOh(Y>d2o!s>yV=GES zF5}92H|PD{<~dZ8^;*FK!0`E~7aPLNf58RE;9bPK^82A@5%1z)azUak3Jdo3R~&m> z9vW>&LM10ac?)7@R1x9gpdQ#2-4WnnT_YXU`@Gt%4M^$2hHyF#?oW-<+7w>rtAo2? z`1`Gu7D5*cfi!!)np%a}Ct;(Cv4Ez#Q(NOgi{ta@co&9T3P+tRXRjk6{Z!qdEh2&T zg<4vTKUeAaf_=2g7NDv&-GnuVok)iUQd5=!3G0r=tv!as@|V?qy8-KF22rc3Mq#EO zOS2zQRSzz3lpfZ10V1Z>H%2dIcVAQT+g;`4pG^Y0zbKVm{1&zx=ZxOC&3+kfIVT42 z7XFkYic`!lQlZXEa;dK}p;e_(=-v8eOxfLZKVIjvlw}F5;Vj7pY+0P>0JmJ2KngcV&1eR#+4RgY)sS5l4Tr z;IA2R_D5^4P0Y!N0_Biz#O(U%ef~6{H%6=j^EA+!Q<@^g^i=GU0fs{z!((za$8B?| zvYR_bWBUK%>n(%gXuh!T0KqK;xN(QzzPKe2oZ#;6wk)m*8VHNK!$NS^#e%yN+}#}( zC(r)Bz4bm-&zJ71sgIRrHj=lgxeas%5i@nZC0D_Pi5A!6Vb}F)~!;uXRR0!sfC*m-g83>_a_@d zAy9H>nd=f`6g(F+=jUrBoxNet+^X1V11w?HY+x+}1W`-p^TJhrmQ(!6DmB=7xq4Fv zB}Htm7nSeKLSY!(rg-0~$FVbwj_MHO0bFfr4Oq?h;jz`MDipfPS&rfANv*~}?61SfIUwrOX?mi=$|Gqsj8yE_YDq%`g# zCKYzK&5;x=Lvk24zh2LX*`j8Jeoj~+s9JTor4)89Wysw3-{@+n`iz_jy5)~FMJvR+ zmf(&Fen^NN;q@(DWq3@d-}7u$!${D}p=@#qXJBux=|nAJV^?95P1M)yi!|%Q3qz(o74CZaeP`ZQ9{@ ziH@>e5>3(hFN?wI6pVAw{nAKvR3hNuK)?!ys)%AlB;RJ<5N&uTdeT&?D;ta zMkvJ{ruXKGFFr%Vw~ZLRoQtf$-{dw?L>qibFkX^xg!Car-~K!``j~Td#DY#uUe%UZ zt38nBF zW~eiWaRPUF<9B+p`DT4^s}K9Kg+O0$1`d_Ax^UcK))D;n8Q#P;#FinWW^eN1i+bOR zAl$$s795cd4x|1p8^iLcn);9(*d4z}()YoZh}|_TOZrKke=tLDFn$C(e~!d8(E6b_ z9ka7mcM_PiK@jybPc3k^!Pw)O?L%F9b!)xEW!ufK-RUkBy@GpW2li}0Q=MpCIuwGz0-Wt^ zx@jk02Ic@-kN7Kp{bCbx7&M?zwj*VjG~{Z7(%b3{6NYD(PNikus5b-Y3oAFohMm4d zDrFg}0P}N?Wk8SH!5#ToNoKz|Etd*iqcHbAh6WE@Qu+n8NK1-KfTR<*%dAhwkadLH zsG1{Sbh5fJH*@TcJ*KBBX-3&t#lV45TZO z<5+YiNQRMAT@?A(ZLf4jdCD#Ya~a`elqgIuPyec7=w@c9jB+R_ozUl0tfDX=+#SQl z2Yc%y1AAkuXzK5QX{}IOqvHL-IFn53%Rmm0)PWtS=#H=>Zpo-PbJ;$`b=S zcASW{4xv@PgAtah%|S90 z{r)?-E?X8LC{C;G1a=`>&2UE5eV!f~3@wFNd|l zG0>o(&gM#hl(;Tf#_kl2&sFQ4-(HROa!K!Ax-y^;&nS?14m9J>5HQxN20I&$K0fMw zZsFlxW(QEKbKH-dWkW^o?FYo@1}8As#kMpmcwOh4~=)B1AC zyAs%kMXG|CdIar~ppxN^6sc`tTz`t|JG_??|I{oU1cqmO0z;~a zTGxHrS1*!0m|=@K1nPpUfJ)CD@At*b_K=f0}|8PNlc82LbahNe^fKaO+NIjH`a00 zox_UzCI#S<7hyoP4dWnKh7oDdyXDy(+WtEB~O9KdHrL@wu~#7S-CCv2TF8 z*Ubp1U3Nw;&{T|Cch0nAd*KF zoZogT+hb#tjQ$O>`^$=#hj36c%QUIfw}3upK_;tE-S;jZ#F4xmTK9E4mle0p{zC{{ z7o5v%vX5{91r!R#vn<8}liaBUFG1vc|0H``Kdv@cj1`ILknjW7?cCWvCKww! zx$fxsG=EoT;+TwA0#6?g*WWr%`}9vg5Bj7xgXu2^KdzOp9$CMlmN;-#UxQtI>>T|^ zWh}N0j7hVnb#%t*ZL&1A*&kA6Am3d4NBLt~&s8bi+#Y}M^uq0QCA`6l$tRam^RpGA zt>Z=FnYGKGT>vvuYn);+FOIru=>2=B&gl*{-J$z0=T$kqIG5%U-UXzOjb0x*{#h5PJO}r?`PH8NB0a%& zTpLiMp01N{U;v)hi9_XcYx{1aHFtD_|7Ky>scd2^^}d> zBA4iIww~7d7RhB&oJtPEZRmTb@G^SUsk; zio=qH4;{_^Y*FJ#yoQGe4obUOSvM|;@+REsWc(h8S99`wP@wrlL&rTfas;a*b{a{Z zJ+sQMzOTq~cA+U-og6uuFq{g7wnPOOW>pNvNZ4e#i2$9%>`E#AANy$d&>`M(t+1*4NC`F_J%S1&y*?ur9j3>BJk^vnfHn=I%Zqdy|k!E zcNjgdG8Bp_kI?v9V~>8B)yBT|*DM~(fdR>I>?NLZy82{O*1kPjrF1(#2^3F@Cp)~( zM~@Vdvc5WrD83jO=3`et9sz)_v`4lgqB(qak}>(K&NX39)l}UpVfHZ7Wb<@f`S=yC+(!1Tq7R9LRMV{?Hu7A_OQzZnppWXdjzP?Gl z3m@G`P5LQN%Ic`qE?-?&%=f{v&0xKpSv|r{%{@!qJNlSrcI&{xk`@KTD*{rgfL}Hq z$~JrNWaLPX`cV2zrr^TxMlqoC{O5BP_+(NYba#SB1g!{sHS)cnSwQX?S!*~fny&rR zCO@qR87+Y5*XOAP>_%iL5_Ltf4zx~}w5&I>Nc{nr7+Wiwt;i*{omNd8u)UT}z-Mvk zZI65IIz1V1#iQXe+n=KQCg5L?Yw9}afyPJ>PVZo~k&!roN)JpcoMUk&OJc|FGP0p8 z%dEe5jifA)QtWD(Iv z*KP;lz}2^)Q;`8Dnuv$aUBN^9#EA<<1bQe8=sL*?I@|dxKCz~3*?Byu{Yv}iLT)lQ zX+yKj1^<+a&{TfM&#t!t2^L;?a9b>W6S5(fVdOD(xaN9>~Mqhxt}f6b~22I zp#FPk$K({4RTh_@w21tG(9Cty%nQ5=evl zJd2ps?ncCC<29vJ;U7i$OhI8XbC>FM3nO_O_WB#H)P7Ndo$kG%hN^$r4)J9A7a%*n z*aM4McKvxz(Fhs45V4ny+wl4u&F*+n5`yl%t|sF9)(OF3+4Pb9ri=Wd>PBw#bGr&t zT#50;fXF{nu>)fJ9TWrEn9kc*KMAH4+3@hIf)N|>(-ug`q>3$gM{_*0nXcC5PKav+ z#xAKePcupUUHdghf{LP~vaVK`M$`nd(3|g+1Dbu!F1>DKF<)9qoONd~~8Rv@=m@W|2Hv&UWB>=^SoQr>=={yFy=S;~<>#KeQijKF9lZ2rc_~i-p zA@enQMTq=!WsY$oPe&uX`0ol9HkjG59!MteT|fl&zTshfU!8oS^?J=_t}_hn0rd-H zRBh_bbtbtN)Ztg+47i5dn=rv+pkY<2dj#RFA|J;2OoB3Vue$58dU_@Izt%dP@ocJK z_}Ys3K8Iuw)<=|eaQLoR<7ycg7mT@ZN>UH`^BRJuSDTYoI<37vwGx9J0tK<8GDYNf zd&9=qg`#m_CK9*#?icvd+S+&k69#`9FLE1l=Of=4+o{h6ONvbeKYmy7`pG-xXLsXQ z;lhHZADo+?a{oS1a<)UR(t)wxc6fIvGY4#B8BM?!*t(0Pxg_ID0CACoJ9~f6!i%|w zQ`?2|t0lDwot=6*_v(RzuZ&dS$w*Q09DLrxdLQMn0GZud`5cCCCs8(gGUu_LcDcye z3(=y^C+b!EZ70zkp7TBQVE_5y&z&v2;{%hjzzLGQhDieBc#^%qgU~s8@z%=xf4e#V+^|SE> zNb&sqMYxt%ddrq?>oOPOJQ@?$CMkRuuj?|OLo8pGgr?uzrx<8b)!JE8Kzht_6M{e4 z&;#8I0*ug~n;yQPhDJ88*(SyC{pY-(=ciIAg7S~eR|~`(;ALok*6L0Sdrb8J=TAs_ zxwW_H!q3nMN!VAszx^2NCy1(eNR#B~>M9~#6oe`>z$PMUBUsl%({sb=`@YnOYDIqc zU-H{aP)iGs#{Zq5{r{=j^Z!isA^b^vz+tiJq=jT6RCbe%*uN#5C?iI{? z{wO~#T>QIQ`G*Ef33|AD{`Y!T{l{Rp$Z?nEHEcNdPt_i;Tbal==HTCZDyLSb01u1| zO7U+~NkIvKJ$%W@K|!^t=-XkdCX1{f0zl$^2Jjp}qe!Z+lRdcXAZ({TluKYPv0oPa zX+VYjF|FKFUPD4`V0)O0LEmqDEm1|`HBnYE$6x(7bVl&r0WCukbmw{=whR5O5-H@% zqtqC1j5coM2f?IkdgV;84D7*LvVz{L43|PrqaW;O$k}wDMj^U6d{EHzbQ+Y3w!*xP zCq^dsAf9N)Qkayz=+Mb72B5<}z9TgFm~Q16dyqB2IKF#kJ8?GaHk^Y7LeIr3@Z&jz z1@u8^a6wfK1`w+Y(_NXFQlFNcso}JGXav_R<*SrRIJxY& z7Ik-gM>bQkch&ar2NhjAz=G23Uh%9pTE?Y-ycDQWwVj1%zEBh}`*Cf>B8k;py zS@(Q=1*RZtaeEY)a+nl&I6&yK;Xp4$^e@p6$RGYdk14cZD1Iv+c`g_>KerF{yn%IB6tU1qu*c>C56CsO{UD z{ViqR=|-i@vDMM*w<7Mw<@s4m2MMO`etoA3RbjoFeqqn}P9LFr3-Y4P1fIX7f8O=C ze#2W59$hY$mOTeBiZ`@FhfbIP2r z{}I?Vce)??kb?+RMj~?tr;efSKyP(UdvYsj_XN|J^D3LUH0gIoi@Kx6wM)a6_l1Cta%6UB^V28Eo3P3k&$UM*RlxM$@ zSdC@*U51$3yKyjkMud#<_GiUZz%}>Jnwu3GN)HEGBX^#6I4Q;c-nM$d`*YzJ71?RD zmSp1lQli(q-TeMe!}uT;_gM|`ef2=p#mf_|+QA>#xi_^%*KZ6@^zmmq&dkvdq5+;L z>|(K}Q8t0qK>P(4NPbangkA=!P*<;7dGj^-=GM@G zX)DIwUYhgT1`pnLjVbe3M(;-V7H$2f==i$eA!F&^fUnWk7tQS6#?k>!bEYQ*bv&3yS0tlJLK zrCsIe?HDTT33PMp7a?Q7E{L4lBcTl0Yx5cKxgnM#9mdUyh8K0nIgpH<9D=l5(`2XL zuHBF;JWlOsEJUpBryJ)|+cpe--_PjY_$BXF72tbPR`|ioSGEJXiA`MT>B{cQ4pBio29*aeGstaPc+2F*d2AO=dU=+%$33b$^FMhn;fgv0+-;jitYm$>@96)U`-wZI zAq!t`Q>fLZf}tXy#l>aR@<-}sRmYz4#_yY)Px;i{4pVLARNZzTjzs-rAzB|N-BcHM z0K(=*{;frAp*diB+a$E3C2DSducOOPV7#@n0y=&NCr*{%vp(;+f*>x$@H@3%?nlSU zc^%}1P)^Rl8fO7N%9PZde6?jU$$*yx(J{GZM7HmHd&KjVmJ3qsjFUz)U3Z%}^?L7m zm%n&ZO2({ZVrWZd-DD|}_M=aC=^6!ojP$v*u((OgUmHBVPLzlK+#!~=TfHSNwvxv> zabwK-^}l|==`GT1?f2iOGeFJ3@tr4W+CkD=xaR%lML)HbkI;YNcZiNS7&k-3=+$2W z^;(^Va_QdOne+edUly1AF&<^Z6$ubFS40Ji{)kt$!@exAnSG`>N{R`d^ybfQj{p1P z!zg#^>Xlq>7>n&+!?69bvt8L3jZa3mi82z(w>URvqrZv7__xztO!zwvyNE42116T? zwTFH#xf>!yOg+Nj$&KF>8M)+1_e3a0f7t$t>l9B#nzkn43MHIrmQEXz)TMcK;@ImM z(vjOp|HY=|Nw72{Z&{+;R4G9E$RCf#y(5(D%c;N+#`4W&L1P^LDe=;P5eetMHCLASo!=af;S1DTuZd+7KjJt@zA1Vg*?_sEK2* zb96TlXMm}gVuiEkCX9s5*6bnu`+9QVz(i4dsq$5x?iIs5aEQ$@2kVC|adc13qlq2Z->(}lGJi%+1+&M|A-(7O={Wi7^@@`1i8gVZ{EM(V6vIpCV z@4+y$_k5{!l}F%_Z=#bkGK~GdQLjY*eVh3#e$Hfh#&HqKzNcV6eyzO7n4I*Y@)4ju zLdaO*dB3mOt_sQ#Rm4T>iv2hQ^Ivd%ga_2- zktkk2*%h`Shyts2V2A9a~8gy;ES zI&(CKjbHCHEe^!npDj3TT_!#|847(F_9oNuB&Zp~%-)I)+378QMfX$0gb?yYqx^TE zGUDM~jH(~x)GJ1!dc^4f&xRdq=Cg?6LJh@FY%3yJFS8n7G~?)NFgRZ49>b#XywnEo-YDVPFmHRJqgtY5g!g7n7=q`;eQ#s@QN9Dlw_`$F z&0|p|Sl$bN-~h#)8H?F)NxF`dKb}i#VY`uG%)x{~Drni^#q9Q-yM5p4W)<&UaQeEKj>x z0-E1KyOM~dOSgnxMKzqwKiP3es;MxBVa+FG7@s`)GVegv<4)jARbT^|n-kTaWtX8w zI{1hKflR*O*uzLVY$8H3Ito)Atjat*Iw_v1k_J{x-cXYVs{M?hT@IZ_o7NvbC3Lt$ zyUrHXGn+A2v3gvEm)%@euDEvbkyCz>-0GHE`*)qbQ~aJ1PMs`ve_>m1L%LSeVdzAEOW3t+(^@pvllUh#0(vK0N*6XTEL{qJ}U#9hlc@-hNE6n8#xH~) z5)}^0jYPuVqW@01ir?^H7RYQvvnJbT_-Zb0{COHEBy3pwgA{eT z%_1SY22uo5?~!#+n(t)D;A`=@XEIw8ig~<6hlc4A`m6StowICHe;<}k@6BC@GlJOF zC}2=Z6Hs$CIfY>`K3ConGvX*m)7y{M4}4m=spJ;enGKzJ^>+~+RoPsi+Z{_j_YW`- zKjqu@4|v{V-1TL__XP3mWf2wN>ah-;_^tKI(Xf3W+gqm+TCHv~<@dQ;B9&kLxp6k4KozwHc zwa=YQwv==$Ds7pkB5eO7z}~1U??BKfXXT8tHg}GZVNH){V&L)1^m@kUl$d|{{-(#c z;IZB^LDKPmOmvsa4xr>v6rD}c>b41eDk3&R3T~0l>FX(nC`_Zd7&a-XuGSApCig*1 zc@3`@Ml$GSS(LwGZW*je+(Vrt9z5P=n1*PVV312sF#wNlTHHfW3&jgJz*Bm7O}RPL2r*31)WJdAinDsflvwoe;n zPUOYXG$_9AeyU^~!pI*J^7VMS_EyVilV-Zsk2)w{)u~qweFlk)ksu#T<^w+bVU*q+ z<1XhzhV*66iQHrL`-_;JyE)Ie)_`k~6eru-;^9@+yH?AE4H-J59F}@K#yz7U8%mO#?NWKg&Fj z*=p)NxC%oC#0P|Gno@$YQx(m~KJ8!I#c@;&sA+Y}i97bM7+|=ZkEFvR8j&75zGrmd zhNd`{%NGdd=SDb?iW={3;MtjE#zpXs@U~q88$4L$Gi}Jwx)RpnPxAPo{`vvuvy*|_ z0`ddv!cAQ>X3p}+UArSUBuB^MrpuT}%Cg_MUDDD*e8a_#6roGE|4#kKyt3{P6^hex%oL?2#J@#q&e7U@5#V66migj1(qk@e` z$Kcak&YceuF9pd&U{T33&)2^J2)xZu%a`=d_l*Mg4kMQf>Z>+L%{E11yEm)TvwDT3{qr4 z?01Pd_eLu+mPW2uLeHI$V;#k{xp$@(cY5%-tA+6M^m+$K4x{Iwosh+XKX&qGX=^!c zBd0d7O;f)5Vfd)Mh#PJeG+g^UL$em=T;|tCdK6BVEuc3($2(1bZ?Wv^s}KAW>=dqo zJ)TOFP5Hz{s1i<5Y}40_+iVwcJ7tU$>C$AWk^fs|xy-t0ddRJVOaA?MxjfH^P-+9? z5r~k4>v`%>vsWG-fXS`IjvC^y={p^WjgFlROuq|T zKy+utjDx;gjE>HNku|ADZXxtF-r^)>of_d*8mq;qz%pnw_e~3^AV1INm&Tpms>MiT zIMl<7<0*&l3j|AUH_!9_Hwb5or-3}fvzw5dd)A@G9${&eg~p`u`>9njakXf#6M1P% zVgcRY^i`2aF_295i{7!;&d`5m%$q*$7iGOQkqI&+@TrQ2$0_HEe9Rxd_~4AcN-X!= zHCwD8PU2laBXV1c$HmJ0xSC4hx=n~*f1eFbaqom@-Y!-@NyCMtccu+rUG^O7xE?$q z@$nUiph`^@;l?gQ#dVo?y^8+@!Eo%Ij(Cy5%7c(n-`lX&co{Eab%w0?YB?|+ zh=$Hn%3AGC?Ud4wz1H%QTBWN27QyC84#`)6cx%{P*OnqC~`GHL9gB7&7zs*!+ zTls2Nm0yK@ga*`n-M*6G@lTa`2;w1)8gje;Pd0lxUYT1yO#AcX*@%M`!Jp#na-(E| z%9pxJ(hV+ZUIA$>(p5rFXz2vP+@c{sTQr3RDe()04{!QQFQ1KeB_(38@ZQA($|~fa zf#jQ0e+60INn!}3GBz%R)s>eYUh&6Zd^cp!q)d2_!plv%`BnJRHuH~(kB;z!!s5k3 zuAdW{_{qR(@w!GvC*xEK!s?X|6D&WN_4QP36X3#t-?yS>TF&G@i!i}LfhLbpO47&} z%)L&E+4a}k;b_U#2Q_SORn=ziZJ!x~X0zQN-)_n%8fxjzDmhi*-<5gm{M-+gk)9qw z=33oPm}k;2sW{w}g7}`~W&AgYdD;jG!$2ZDW8dpTM|l-FdE%ddi{#TrfBR)7N_%ky z7SM-9#LJ<|!uOKN5Of{mGFjeRLC#40^3by;4l9NNd6oN=7&!yW+q{ljLyivx_1C^T z>)B?<;NE6VX=g9v@AYm3-^Ljw%Es|bs|>y8#m9#-iKA(}_A}nS@?9Xhp;S<-0r@kS zZFGi^TZ&vel|K%{?P*}Vu_A2atfw{@(89oc0t!#n+H}_G#Z2X>$1rvG0R&x7J!639_-5)>H{r2?6#jSrjPL%gt#wCdgopmS54c76f*U zz#9tgsHnMd5l`EyERccD>C`IQgwx(O*M#9Chs()W^2K%2gyHv*s>OWfy}3M_7F5C% z7vi&|eEX?H2?j8TBOQ6g`6a*Jv(8@!tmC(UH|dZdL#-Xx!7}K>KD@H#r@~_)Ixwh(D1g6P8QT`k=4pj8<*r>$?NmW{NN1r2W2Id?)-|ny-uN)4pgQ}L= z*z|9{X`0=8o7Y6=>h_H?UiBN4e&bs7C3_YwAI{A5--Q&?z*{Jc0r zs*12n0bb8X0M_z;LcXx3kQuf&MD5OHAdJ%8TLu&xHvzN+(E{#GZ{g|N<61m4Bw~wW zQ|B6RPlg{CE>ql)&XYGRme@S zjxazD?)cQ&2QOZAe!;vPIt0}$BRU;D&yiBTV>w{uyh{_kNMUcv`oV3;2R?1CqVA1# zAaw$;SY)xa{%1r&EA4a^IkDRwYpX9&&nmVkH4!xRLtwzTBC=_k~&P_lK z*z%=QaHVcSvqWsdXKqE1WymwttWHIg(F7=&{aWT$OVJ&RlV=oO&r2AMT?KOHEZ=YI zEoYY`ricC zdKp_8a&K~97c+8{M$B*74Pm9D5dN zFpvHeu?!9!w`W$;!fI*_P<=i6e1=T#2oASO6kfn+IbsMVTQWOKp#F-I-=0Gl%ut$g z=OJWZL`VQTIxogk!~M!YJOB-E2&kE7XN(|b!|@OmRPY5~;0pwEqnMS;c9gLjcWkr8 zS|&1oquO+hKjyCqJ_cdm_I09KD-RW14#ERVqi(#%XYVr-`bdsL2F8kA-D7XIlE>dv z8+GC$RiJT;9>>EKIR80ls7KiONB`c5oE7}+ymlaWxzai*9>!#E!T_FWQFG#jn{!;+ zYD_W=egRVm#YE96iKX5qN{}}D@JzK=bBwR$GdClfZmYB?{LWKb6wnPc=b(`NTr{0% zV+28eDNMA{f}k%4c)0NBtYCGtyyOXB?Y!{^TAC9$tez{6wO#k|N6i%BO)`&@p<3Z< zGeE#HM>AJtmX_67inP%;Mrz7Fr^1X6%QI#H&o7#X+lm+4>8zZ17{3hGVX^ zmTlU0b*8qJ7brUzsGxUzR=5%&yjC2so>XpI+F>Avr7x|R4W!(AQ-lb=Z)$6eprasA z+Z&;PAv>0|bSH~s0S^N*@;xp>`GiTNrwk*O>{jk!eJ`f(MA4Xz);V3*Zj^OSUc~3d zGQjiC+Pn^*5~6tPGO(#nI4jz3`?o&?)L%K`TcdP6j}O%6a~ovMSwuKwUuigTWGH4c zh^PF%Y^nG9n3>laG+90cMSIr##1(y?*`G_?VqGWD>tc`H-Xf&L%LvHJ`q`4#l0Y2q>}yAyCy}YiQjt~WQJ6+cJ91zShLdh6{s z{E#<-8LtOBlZ~bhHjbv;_RNXoCBcosz+LUifGbiaBwCY4aoCX=lX_Z}eV;15^NtlK z1xlZ!zZjrSom}61Zh93LhXu2O1s9gm6RN94dCKl|STi*-S|x?{-OTFV3MI-GBO|xw z%~HMn=ZNe@Gui3C!Kr7H>o|fBhvht8Cs=jAB&g_JO6iIC4@hKu9^#p(Z3%EnMoR%$ zhZFdVw(G&~vKTL?7JU%Oz&!}@F|7^$en&$Lo7LmJF@1Y<$)~H%>0|(l(;)Hi@tv`6#H|AO7;crqeJf0Lo^Gut zA6DA>)8EUJdy^!7uU4Lsvm$Oy{tBgUidRN9A`ta_? zxN6RZd@f|`SERQs!STNHx4j{s0ai7C_&zon$g^vxAaY@UHd0CMq8rIfX^F2S6pexC zZ$^#_#z0)my@jLDl8tD2dcSr8XPBN&3ZAMLu$>DDiFwi;&V z8jxHW7*(~l6&nisFmLcru40~$%gQ2jDzu9&G+U(59l(%F7N75!uA2Ni*J{W!Tm;x` zIP>xu|F!B^W<=lcX)fP4ch&`P^0Tk|^T(;y9E+{kxRKv#0sF5#9vaOhHCVp2u#Ft#xM}-IoSW9pfPB z383P+;VQXP)}ZY1s;y*w&AF%X^+SQ#DRiHR&-_4rl9O~mISt!HpfURhV?lVKx$Yiw z`FZzA8^%4><;iXtDMsWv&G0CCz$vq)L?A_bS;hpnpmZX!vG^~D!(CjD*pU(c;nzL? zB#W2q(Lsmm{QJ(@e0rUUSInNKo5~;QsQ`=<$~i&(-ap)1xFQPOL1waidYP#zs}p2k zPV(}owiX{@hev++D)A%&M+L=3C_<@PKmM4hbU^R_ih)yag zDy#CyRZ-~+pB&c??pF@x(~plq4nd0FFQL?@`_4SR5eCr3dvcF}@I|Y(n4~;(V+h2$jVDS1p zCN!N-K$jS-XQjSOJ-Cue5=gc8SDlisNZO{~Aq)A%+9*Sj834>}_RRzv7M>>edK zK&J{?`MW9qQB5?KC{VML>r97`vtH>PV_WQLYjfL<&Y4!lql}~gDCz@}ZuZyK^yZpRA9bbn&HK^nm4F|0yQ;dZ%pIla~sGDW|B#i&qONxj;MFFmsRK zhgIa@3}MP1=JkfHB^eg|Ax4J4Pvy<8-<^Ng>7v(p6}CI9ZBIy zeb^$HgC|ezP5-LlcabI0>&?4z{&YoM>51@{;>dIXiWMhDr0FLETFiXYe5e(;ExojG z>C@O6>@+a!jNp3)%lt5U(n6hIXCL?*3*rb<|6y~N6ypg!82w!16})ymEEas4Y~vc) zq1V-0@v(YBxr;R@2XjpUVw|ie5My;2l1O_swyeQz87?8!!io_(S}_n|yU_C4AsI;f_2*LZ(HZ5( zHviaw*F|PdJIfk`sMNeiW=oU(m4hz!$%TKKTcdAbNQ+_}t&veb@$RHQXq|pyp`97k zjQyfHH|nXic=2hKX1@)im{$#2?&=J1%ktGZzWDTl=z*fs9!Mt3KN>A%ZIujA_oA+M^AKtBktx_Z_H?`n`Pw_CCD;s2!sqE8 z5|lH(#*&(R)3p7k*~6X85*--@GfeGC$COrpa)bq}=ZDAzCS6bM&%s91F~L|ZXAAJO zAisw4v($PRNtN$MnA*37vv1Lm{I)CJr30JEyN27kSlyV%YfQL~!)zS*#b82Zh4VS1 z2bCn1--7bMWrfo~6n;cjbfd{6#(V06nesg$(oh~|hJcgIL{Z9mNRHx{?}Np_XD(UM z6937w-Yn=8UW^Epa`LB!g z?&JH;@81n@&mGXxy6v^BseGzpOePt(u&rw(VH3-0N=7s7yQEcd9lf|wrJ$0v&Jn8t zR&tjM5!0?aQIfhIdcDu~zllD$GEs6Gnr#GRwJ@ip$qIGOOtK=PZDq+bvK|vMA2|LF zFi#5(xR3jsR_u*J*#3RqsQ9@Z$(U3+e6P zcSq5gWAt`TpODIF5B)=^oU4Mr|*i8G3ndW8)RZH zs<&@8bQMuVQY5-}EI8ZINOaL=s}m!bzOLw`bNbPM-&3nZ(T*?W*Ms~qEH#T9S#@+$ zw$n%L1IrkhI)vD=;p}62+Q>-B7tSW!L>Pe^=0<|HWlT&Y@_nGVe@d$SGOM z%ZuQ1+#JU8Rzwm_-42A6qG1zU+OiYJ19%ekQtY8PoYTET^pjm0C@b=FIN-3}uCIz_ zOXFjVWYf+`B$uWWdev5wbF}3IBTH=LYTnuSuD^(4L0n zEC8FMnoJnGc(p&~cj&0#^FAX({`}DwT{*FDI4JM05BT?PQkOmFOa$TZ&dPR?crTiN zYH0<7TCjjRejh<^-tCi7FdqAyjJuB3bJ#?aR{>u}Ipe)Zx6_1zs|rf9wY7Uw?~R4| z@Zbz{H#}kF?i(qYCRkfrq@CM{qCGrL$zVtqn)0m%x@dcP!@C z+P9(6|M+;eoK{uzVj#4VdgDcjI&C?M2s)i=beRW%_s%!ArlS2@Bd&$z@BIYwr9W3y zn{$wjE54@BX^-qj_+^Dz=(RQ}wF|2(8H`jWKyUuSd2Wf>>In^)b{7|E7RGiBR_`IX zjD>;Rf&Nc?KNepHZQxym42<6*GXh`Z>?C5D=gU<4kCjpAk;@`XYoc!nJrp0h4h@hu zyRLXTfgwnZKlF%IQIZ~2$c{e<``v#{1A4b8jY`q&%)#*mtzfGiS@W4$tKf8U2bZK=vwg=EN5M^RvUQ|OSoy8QH;g`D^ac*=04tp6`M_4USV>Vbo`i|&oruJck z(i){OeC$>eMba?CbRzq;Kz9U1y%Ghc(kGP{i#|X)` zUPJB1h!FJhG^R>QbGX3Zl(%p@M^6mBgDmFbnh+x*0I8C)My2WmZ_NAQX1J-w=<(i& zLZesk3Nu{fVI}qE4F6~A$|M^@w}*6r+#IQW4g9|CqMzwdjYZb>irtNcfA`qnylz`) zpX)re$aNL3g0`O@zi@<}jfG$T&|tW3YdjX;J-3++d?#zh1*OYHX%vG;OfK~zlmlnl z3A^=xcs5znRrZ%LSovyhrQri(!uW(hF&ZLw)>m?A)xs`w5U|)*7B5?j zy0I@)P!efz&e#dZ$84U(p39gGqYptq77H#JoM>WSQJTHJ8Be%U?lPD z{FA9Fi3QAwneIzUHzuq=Ta&((3L_?}X1Nm#QNo^S>+y+fEd3(6FW$0ZSXM)Sym$Pm zbvoUN&ra+4acxaf;aCCMAc1g5G#3`9#Y_^a+^40p^jo#57kb3=UP`CqF*iZ`W)9Vh zOr|k7N{7nOAp1)r30xf##>Fpa)n|(ur@uP12k1h3Xk*+5T6$=YPX&?f)(Jc|Lb)<& zoIZsv8JY6ZjQuA{ey6+tV+o7-x3ABg0I8K^7s1wK^*C+;rJ;U68U>o*aknv?r6MGu z$uSTaP$N8D8cX94so`m%D(l6`6DHsFTKw{7$Lh=C_Lq1FaH5FU_e3k!noKtN=9v7W>{(;V7L(Sv>K~l^_)=J~Kf@O=<3y3>pu%9>8RYjzorc5> zt6NW}V+Sk7e-qP6FtHa`2`!vg=;e0B!NtUipSP^#Hz!}fe_6xnn8P@VoE{=xa1Yp% zyHLyt%W4X}j`Hoo0Gy?n4*(4Qfxv6;>xyf;O{7UUbZkxKeN#nEM#nDWMQqCyP3g9X z@L#d!J>lpj$n89=p0LXhjkJ;QvmG#SK!%R)zb(%nE`X80Ek>Jg1i>=D8aex6d$>66 zEgs7I`;|U#BEU+gl@}+D|8zajHgHUE+=cdDA~y(wM~n@C$j-oW9Eq1gk~f8BF^A+W z_jdt3KtfN34E#b~vxD4vAO#Ue-Qy76naXQJ+5kIo+%dn`r_XwkJD_}utj&I$I)w2S z29JZtmx@1(AEb&{W-1EvtLeAICLW~LxwGdOIk|=X2!4%MYEBO>Ikj^ioSUTOLz6s7xWiF-^hT`>1{ zeq?*iYja42;yTa}f^L`*W|I}ZLI_W`*D18o4*!Lia)n>YVYvPLme(Owd1AqJs=4#L z>P>S(@E?k6SNVnhW4M$ME<4aJpPMkH&M7#JlU zlSq||=p8^t>vB;w-00g7dsi@QDTTapD{Al&%D-y0j~pJ}5O;VXApHfr``)#+pikKK zlj`?5XZ-40#|))H{WdDg z9ym716W(w}l>Q$iA~C^DXnb)E@V_$8n0pMTihq7Q-~Hj`rg%{~A(NqUs$~#{3RSzl zu({cvT9eLEfMdD8VnVJX5G}kr1Ann$*-jVc#{3F6MmXO=;67SAP zUVQF$voE0ha)}gH-pNBS+h>z~nU#B!1O!%CX-fxPQoQsgelPbK!)6^o{f)twuK#_N z7s!c6LWZWqTN%W~SLuzU^(gJG1ZO!VM~G+?)RV=@K%8oxCq$@| zW1*2J0+htJV=sLEWb8kl>CGa^dduIha$|Y7+%ns@tAYw@@lp2TjCA_KR+(HGkoR$2 zQ1?03;&xbZLe=d+_)}5}-cof4AADr{9cXbUZ(zgZKk~#;iY?)*X z9%aaiUL{4+X&ImRlg_~Jf8MAj7hbQQe_)m)_H|u+Ge}JIAwBAmCDgVb-I$ZY-Htf= zT4Mc71zl8M**tg4IB{`iOosX^vzD&qqzI)sc~GqwQ@Yg8@y;(;0;SO zrNlbC>VN*qwG~(GAoFk)n26;1Oh^MHBv9Q%DYqw4oVwN?l4#|=DMxOZl_}iU!#mHP zB{*YifoKFL8sZ-RbD(0j>eyPYA7-{OE?3W|a zz8*8KMl_LYY02Uw3?1+&2JpM1aZ%|p&Rm_V5@oAV`2`w6Epi%NNWh29b(Z~1*840+VeDbPc{UV+!*fG;LS`kq^UnhEd?d6JKa z;-oTZ0LEb|1K1lL_HiRjqpiy>hL0(uWla`wt@9Ga$pg59ZhW8pDxb|>N^kMR9d>+x z85s(A4#HxE6eSoX_-N?+q!Ar+54)Xhn__meYt0)ao-%?b8#3iqc0JEAy(@-E>-pg1 zOFQC;tzDZvHko2KUo)u=*{k=alHD|?=s)jdkKG^vTG(3Z6}|K$)K&oQv#FpK^dSze z!~21C8G(_T&MyuZP%>TaKsJcBL;2Bz@EmAW@+mxd^T3%uFtJjOvGE@4KU*RAjpjuN zV*v=S-7j)<2Mw{ue)0jGNDR3qpQX7>v5yU6TXX3pv>%d&LIAhhA?`R7>~C57AyMH{ z;imTc3*2c<*0)-Xt)48v$MwYhq_Mm?)EDH?P zK#fllzwYnkM6pZ_4+>CDkI~Gy$<&T!2>2qFasnI@y6lz)YiB`R9I&x1QUy73u4?I8 zU)d~K)Mc>%4v0rz^-RDf3=rap~a}Pgc`3IEt~x+hC1TrDN}0cT;6# zXzEH=*J`36k+V=LnqR#?1p~_yttAiVr(B!YytCn!E?g;F@0*l%f}Qn5S$Dz-4k)9u z)bR5y>rz6vC@%Q~-k<%0hIK#V{GX$x{!>=)|DrG+FE`Ksx29;b&_#t^yWPJdhAHHq zmM?2zsxou(eyu3?kh0kHMKkvw`Mu-unwY{6B_C z|BtsVzn3eZBelR5{O2cG&->bv|IT)05t{eE12P|af}IV#&zB_;|Ja^)aAmvq%ju;h z;6L~on6xu3*Ozev2C9mlpX89{W;jALfzi2>k;iKd;wVd)^*ioe$1h zd){|R6?MbNTAv@*zzm1aZf|nC{obY@YA?rXe>E1ev2)_58M2ptGsYU;oB!HWxAQ%v z52hD;jC+BMK&I31q{3A^iC5k;WtRtFS|ajMGSXX&x`)H<`0h@@p$!Xpfy^Y)KBpDJ zKi=hvuEy=LXOQpQSpCbF*cu&9gqF5FmWKxA%o*E|;!E?wkEOEx z?|5{rXVMK}##QeN18A>J97vxnz%mHD7@ki;Zy}+gc!$^=4q%Xp^FC>?^zRG*rRxRr zv5oiukLLDmB(Ts|CO^>hX(^F{9nR5~<|&^Y5`~0!SNsi~=2cNRj+ocvzecJIen*4- zUSmdm%aj7d;p3Ju-oS!7xDoR?HlZ^B9M2g@39w8VkZ4=~mjrm-uHsN4*{@)d)@ITn z!qA9s@;}0{eCf!f(TKs>Tv2U~f!ouw6Rf1M-T?o3tyrM2@IRyFf33~_+)Nvx9Fg<3 z2U1OUtjp2H!3^s&-G=Q$wxFh^1D8U!R-Ee9P8QoD=0dHMIJtud8M+d5nc?6CMHz$% z-|Oy9>+8sVk3*5?ZJrX6H%mv>wg9zy*gezNe4}?9!_Pz%TbblO_#(Aa+U#EK$_H2h zW4*g)c({3vI-naSJ0cu;w=@1B2HX)0>E$Ra9>ejXG0hM@!W?rO}^}Y(E zAHtAQNO+1kjk+&}hhctjmP2THSIlDSO3z8kE)W(Xb zW3M5Uw-yU*^rErJ8j6rk@2i`HwwOD!?#}e!ur)^mtpvr+iaFEj!CJ8l9#;Oro9UWP z;g$DJ(f#e(VuH+i^r&$}L}kFHx9H$v@@KSBsV$f~rLQ$95PYOc`+9uWNl2S~;(f1eag|XUk}0jK8un z+*~+q)+awDD<0K^Nj1m){!H2@#QiJVFo+`DcC#wS3;^oS6PW+vfzQY@k zjT(=&d|JBPa{;2fOWItZi^kHRJ{s~9syMa(98%=vT!9n)+3M1ihwR90vn{HaHITmt#REb7PrKF;OV@PkWbxAnVy3$&89`?Hc*|NMuUpHDHzNoG7 z62JZg>Q#n%^#Rl%I^HB2RgfM&7wdu!M~K?Hwck<+B?O$_UPZB;hesPnuWnJ(4kiz( zCUtYe6`7yb-sef6P%!DT2o13B7saOa<*HIc+q6qJw08(AizS;oVfYS@7wo3@^>i;M zbO=lsJX2pP0VJER!hFgVM0$4#qJga?%MU2@4t9LZrMuX|oywFMQ1|N80>%AN0q51q zm!0lI58fk0Q?r;HTOdC}Huo>IoDYKcY@OifH~V{}>$c$rTiTci&fW|+ z;xIiKieQ%E?b<-PApsFtBMDm=tIHcCDA3Coc`<1%Iop*9I2YemYtxXk@62*0z{Bm^ z%rarQM1#IOa}Z&VS^yjhFHF11d1%DZk~^-}OTGtbw9Uv7NNS0inoVdPwL(3rIkexk z>sPaXf9N5y0A9a(kifh3!fynHV&j`L7tbc(t}vc^t9OJEuuhG#xx^P+PVWSf3Ljt^ zL4IU1w=ND-dM7r)z}7CSNXy5$VqLdM1#{kp?1HpmXIW8CN_lOprourHO{>MWpxxP9 zG%~7girESmoLH_7Xz%a-vc3biFn)w(fl;>Urr~hY?OguqHF#_|tC93O!+t>$KgnN2 zW6Xq+<$##1Q2q$XZOAV9MC~D_$`w{>;y3Rb(ty5=+#}AiOEodAbGqA+2&!NkD=*5l z-YXqlIoa%&oZm%lQ10U8bdauyP;%S8{0}74_up_P1ST5rja4oMLKK$z}sBmMkA(%ejEheu`nHATp!jW)1 z2iro51N2O(Bp50*F5@YuG8%j#v_ZTb1>#e>p!Vtlii|aS*-K8T*q4}NG zhyjcnLNKNaSCe>+Z@WY~;w2Uw)dgjf=jjw4k3?_xuFLlh1e8P%!5c41%C@^7Eod8F zeM^7Emzqw|V=hXZ^8Q-7#<%tZM=>D;+s%V9ReBGqnee-DQdg-~&E75DUlajNIz7=oG%P26=9mnK`g%iYT(A z-|DF7`7&$%ZTX?0Mr~a=C0WpNs%!N3(S&VR+k}y$#FzQGejv&yE}Tq`X@dyUB;_e5 zdZIhMb8m9g$OES9h!SL8<2{hMjVZbKgT-hS6qF9Yiy859xLD9;rzDzeBql1WK4f;r zIu0gYp8Y`#Cthy9MtGUahoY&%ttMEP1Z3G)5ENApPsJeQACa0WxW zE<}uuXDn{XoL)+~_c+RZMbH{wn}zZM{je@~CDve-7D905 zD=jap@sr>~9nEDW%}25H&Uy_7@iNn$yDw@r+CGKbGG3A}+kcs;EG}$i0({dY zVYY3pcWtS2PfrRp=vc{Ph1FdS?EB4p?n{FP$DMm>W#oP)Zt1H7!+&OlsgEMHzCXLQ z7~Xw~39Z4xYxDH8GXE&$!Jv%^3JP>LPr#gPHeM(A@lzP?e$Za)plPvDHn}Rba9xHVsv@$ zU1+%u90-WN?FwYE9Bl_a;nx)0JK}d<$#FWg#+-F;$@SYtr0+2It(p*|bq!hX2EGlv z?aJR6acA`X&T-OjzZid@Esm-dF9P&nc7%DK6r!_TM;_|%ZK1Ov1eB3mxtILxb8K#? zfJRBzA^B$XY>Ezm_D+tQp)%C^`=$ZY2g;B=iQC6a7g@1cD6~R;f5W`4*UUu@WF@@E z;Kf9bcM487IFUA~2d~91{7?#`SAwI3NkI|qKj(Q5$fB+zQ_uLd7AZ}6fZ<4VR|z=N-#&FiTv@0=ckA*>|h%Sw(3%yag*@OfzOJ=r&~`{qEI z{l){_(l)_Qxtqz(br;cGqi{@TDZIFHewG|0wNPC77Q#A>>@F@{Psvs?_wIvnSeN1r zsRNwl!9KT|H>VyKsESF=_ff!0STxjOAN&XQa;0U;pC}uI7!Rr(Qh|v8(b_Isj}V(# zs)daRZI3n^cCtt!fN9YmJnDpYkNRmJYQ^i$uXEqmnzr0kGT(~)IS^+m(_9H^PtBn% zG+Cn`nkv)$sk4Tk@ZIcdjiiiy2OiRYUIPf6glJ>1PXg`IZj;p^s7(KgV1Hq{Tax?f za|@q{f$!sb_=JOVFsZYhnDae2+P_rV136ze91`t0sZ^V-ASX}g1udJ#2Nu^2pzp

wz6v7kMotpYxDL_oMQGbXPpIT2{nDKR!j5+>ixU35+ z9m1?&uC^i@V^3`RyV^_j*(iPvx89#kxGzTh>T0b|N7GcT1@pgiexVq&n(~Pruoc+9 zFpRR9MI69rAsl9!@QDgC-cP#zpxUXwju_~@OHeE&I77UY2$khr#7<7Up1Coq?scTn z$n;uyu5l|N25X&d&Zgkjn? z%uCTbt!$^|#AoM|0X)g}W*IC6-G(>h>D!e1@`^FEvzN!2tF1z(?_>E_1*7igL634zMdYA;%KA4 zr7(Ma=h8-iqk3o8zn`dam2zpn#>w%0C{%-*l4~~8o&j#OE?sT4QaVJLk4{@|g4E?rb^-jwG3JEcSue(#QwJFQHv7v__#D9bR^;L1K9Ccf;V zx7T^i1Fz5UoOO<<3ExGs^SW@<>yLL^M2a#V%l%qB@@dyuB;1j7qv;CIB-QUty1<=j z=EM0oSgN&nA-h0C(nRv;LM28~Dq=bRr|&dY&o-MsH+)qQNiTg049^VE5Kt^hSS!Lb z%*4(xGKktCSKce5J zYSB63JR{}DSE1HF`V9HE#y?`)SJf_+t|PQMl|;;-s8k)Fb!jwgYG%*dbam^VIE=~H z-on`Uu6%1Q_FU1>ieKJf_yv$fL;HBhSf+6fq zVZS+?t6`9w)s=TQD5`0zf39vmm!AR$`q^B$7XKRGQzlX!D#bEiDk+FS1D90%%GRgl zja3YDs6&u z1M#|%(niZBHNwL%Vbh@m|40>S+13SuIxKt=1A85(;cd09Y&eVwM@scw_dX_h^xvBm z-h6A}L=Nxx(k!ry+5p&x=W}M&Q|WDh44nOLryqWlsl&x5m>2P{oO0ZML^zG7J}0oU zFQ=Kl{1E}cI^SB}x2K8}+VA8fRdUPxV;Y}IWgD{Op$L-hjPU;@`^OqNUE33$b88oW zw+|0bh_Bd&#s?(uOD)9WQefc-r-xn3HwtgiO@!~O4NKy7)>IUu_-<9%{X~2#IH#$- zSZMaHxS@U171zkgM#$KANGvUwJH<`)k*@4(HV2Jrj+|>;H41mjqVrO*N67vLP=kY| zh5ge|G)lb3U(OELlkv?gPG`a4!MOD8*v%`@Og?K zxN~15*v)T|@Oy9LJgv8L6zd^P0=(vcTh*RLU$sht14(7ESrXYfRj*n3sCCxQ`cY;K zOA!^k)Md12TN3&CS~&DM5>-Bh5p&~OM)d9v52+p@J&(C9YumJYq}j^%vI_q&<20g55_Y?0c`if+a}#a(rcN z5OlOb)#Igd?XU%9x*+0?$b@w-XVgc`3Q39ofQ5l<4}=oIn(#$L8TENYBcD_8{>Osv z2`|W18AMjFRon5HBJ-jg!rndhZGc|5S=TTYgaCa1gb49#TI4cL5UfS~uzEm;xuF`X z#hO6BpBn*%Eo1q_s$EbhTGFM4-2ZE7D9M!gUWnCpvqc z6NY2h(GF~U)r3*a{@d)#Fofwo>o7*eDEzY|(A52ZWttXAr6J?{k41QKMgY`&NSf)G z+n~?hd&dG0H3}JgQJ>UdQ(9OL@=S|xGN!uF!!>j;797By&3p+-Y-MMS?Q|saM3%z$ z)YcY#7QXk~=J0daZ_gbo(P(h5V-5HhlXAp!Bs+rUI9$?bB827J)6By> z&UOnK=^sew&1w2CICKg4(VvPX7=r7)u%=F(_w1Sl$pXGMy??2%M+Ou+1Z2TIc>~$A zZU-QN0E5q|(PK4Z&X$}cOv{9HkN>vAE+ZBEvZz^4i(9KDS8C<9H~1GQtx_sbx-QflYIE^x zc|NrLDb6`GW`B(qwLBrgR|D+B&C4}Gy>>jLkfg|OMspjG;+Dk7i_Ya8Cb-#@GxJ+V zSocO;qW&Po>uagH?{icF^~PVk!6w+c?KAzX%(+V*o!!^@xAqI3oz~!?@-#Q!nR^yp zuiW%aY^i^^h<$~A(ILHMTi_Q)#ug>Tz0Z*zHS+r!?R&zlH1qTK5aT|j^hQ`<(}ZyS z_6tX6{kA`Krg0zoiqCQYa1PVqa(x3PqmH(s=9XlPMj*KKUH4yD6blGK%jtmQOX>?v zJWOlaPUeBM7`Nz@yOFX8H~t9t#1BqyAJJ&n7b0s#e~jz^8#O7R$8 z;;zipDfNgg+{ocHR{>8td-?G569GE^|tWCiooAl?!Y7LKK;`&X9Xd1fTh@GG@xyOJwroVrM zCh#e}cKCkrs6UP{E}Qj6M<2a}N1$~5zGle)Rr1;~tw7O5JB?!12=uO=f}VL@h~5xa znsptTO=6fG$_3FS5IQBGsR4`TQdU98{N~DMs1SF zxLH!cjm6^4`owc>M3rJ#;t)}4=-%EF2(NI}sV96z2vzJhNB$3}6TX^!bUIumj(oRQ zB8VLjN{7R97=#(=fx<-t2ReHa+U#ABfOgkF7KnH*aPV#pv-a}f>$5*`)_eSZV+7pko#9PKt7L;ni_mdtB zp91d)8Sb-8#d!Y*%M@zb1y8zLXxgAIU$OhBJ(_CEl2F^CsM*mhg3byfj>hUSKZm#v zyf4Kh3i6|$ed-FQf{%NIi4}t$MuYi{k7b%YlRau@B@Y4Kd3v7t6p!pqiU(sX&t!x- zSmxJlOh`8>S67VL{OO00niK=SgK(|;e7+(#_x4=%quuz-g&U|J3!ew^z1D>|EQHgr zZQU6A(`K!Olk~q-Vap8^TI7#xDmN;J_;Vpk8vSMZk|8%KhKbLGT$~%75lTsJjVoJ) z?PPkzl*~RqA$C$~W1f$ao)L;gm*7#Rgd4T3(x^NnJHeC;$=RqJ=3xPDnQc%GF#}(( z#MP+0i-Psgo(Wb>Ijv;3c34Gqe;6WO72TsvU&|Kh*})S=J5NeP`}4NM8; zIL`QgxQpfJ&_fk~hiPqrE35DChI=Xv_qVRA{@*V6?(*QP_Rj{P$G7a$$Bc;WNK4k6 z5)V?zJMg@2tKi>s=A&}P_Fpsoz4MEmyW>VqVR&v;cm^9nb+1DZ7J}&5=#X=wq#Cym zIMQ+7(Iq*Vo++~839r@pb#cd*%Q=xH-G|tb%|{$9^66tzLpf-{3PCC~&yO($EB8x$ zYG{E+6I1fwTTefj3YPa~`;_4gE+q$O=cAmTXCyXTFXt-$l* z&J_m<6x%;Dg!gc)ze{rZ;ffO#o^1kHVhm2aQodSLPkkTb($&`im=>-($(;h0*a)Yz zO@`1ljgzDMJL?zgNEjghbyEnmY)dn*d=E7qRZMTBsk|gH8NGG&FdhAr;clf72)Q#> zbE}`~k&47c776NcPG^cEPcyRH`(EiJ@7+eYMu=Z9^oUU7Woh?fe0?eXg5FOrMU}k6 z*K~E@f+#*g$DU?;qc^DEK&MIYQB&1!H2OB2O;7q+!bMx|hbH6~y1FfD#}>u;_}qLv z>IV?7U!;Tv29~qW?(uYm3Dnr5+Gc6g4cAI44nBL9(eh2kUo~r2@x;7XstHI>WTFTd zZd5$yzhO;~;2MX&kVs7IuN_yJuR||$otdyvqQ$>LnVt_xNmgb|)AU&EN#n7Ow!fe} zS^l}?mh7GR$Rkl*Xo*(pPVK*bdmdfF!f*bmv&(+l3^uIvP|YGf@5AzaXJZ*bbm=Cq z)U1+XkzwiN!P}3AaRDZzGe4hSl#N?##6`2;2p!q@OlMpoIXx`Qn)EFfFAv_d2)C<` zG#}>R$NrJKIkO48P<4{*&3xMRv|*87*nI+lRWRA844b1?Is7>(G$zm^#!hq8HLB@c zh!aR*(MBNs6$*)Jt{t4)f$b-x8^!G&cZIGKMSDFiii2683NaiN6*xyu4UZd~n;3(z zfLsv%NxSOgqU^m4x$5Q>yvakXF~TXt>2)tQO`lC${<^{_zK zJ9IO#+HCXmKd2`ibk+aaS%jVK|IS;4hmVT`Nm!WN<)^c`u^p1f?`ngKG4l?)PZ6z~ z&2O6TACScfE?%@fdT94hve*buLU&T&7a#T-T^AnjI>ky8ZnJMe=d`7BBJ%0CM|( ze5R2-E*8-~I{|<_|2~FA2=+ejr@^4zXC>DQ(xL-^9gC2tCRwyAlom#8?>(ifl88*s zN^qvA;VFe^5?nrXZp}!`d5_2W`Dr9&GEPMxy$Ay}0uO5n3C+j>Ij$8l`{OpB#~j~d zZeO2t{qzpB7&0L<|0zC2N@LHwxglb!&>O{_ZBD)}t|%R9MKp&=^Hj1Zx;Ml{sjwgZ zTPSKv7me^jAqXYe`&v3wn`lObrnO`f(s_??w&WxHw=XuC9vWd{Nh3;f2iO0xKcr*& zr3}MPoWEdy^Dn>EdYe>H6FfyI1Jf9@XcFA{s=#i6f$3n>4bc8EI0 zm*u9WrZXYLGKVF>Dfu@Jg71i83+>nS%R`#xQ1H5DLMkOJc`||JmEqm=v2*~)*9zmb z(&NvKus^?);927uC*5Ki;?v#+*aXP52z*u+oGABhB^ zu4!n-peI9z>d|Hu)^5S=uT7@Cc9mW%iT~ta(a?B;@h!GSFR7d9g?KF*|9zUGpuEJ( z`)6E*a8)a?TMQ|jOZW%lgcieEQC4npi9TDT;`~>9cfO;i7fP={8dXAQNM zZZR4Fgu6nVPS*VP91a2*Kd=v8+l02_7UsC}Pr)h91uUCR_7lcl%EVIQ-||J_3BeDfM!1S+OU#32 zi!G@3SnowJJ5u;bVNh(2Cfpa)1Kmu|rB)5VA08uU2BC0%bf|;D-TfgJ9S@RD=HTL_ zuJZ!FTy3d#$FjpnI0LfzkL5{vP#Q(8(kBRBPv44|IG^d@O{>vd0@+cz?E$O_nbm7e zK6oHE?02RJ-w@?*=i)AK^#vzP2$ZIE8k;%@tkosO^Qy zx!DAcwc_gQeVru^gzzf4cl7hT&g7!N;>OZ7D+)T*C$$g$qH7gv7_+a5FZ+hCxvz@g z9`v>lUeKx*jAzBhl3_6*@{J^_Qf>Q_Qk5{Fy`*3)sSlvaOYklR?d7@K6G3*rt>%MP z@k1SBg%ocbE<^4{5}OV`AT_`numIU_SR(E*%M6c9>5jmfX$^rrZrEH0fqUqNwTNBy zklr=V9S|@iZk2v$B*>CpMZgGUv5E&>hLCO-13xLriq{|@zs{p9^U4MT*~>%C{YB{8 zv8o>L&CE5nzXXNnkqhrlYr91XddTo>du&LKKE>F@SDQIy0Q6cc=j559lNF_DLHKC+ z^ux1aV&XIdyv)VN>#?F0)tdZGdX23nSY8h{6sR$OLiYB&QHGAiQte9Wir&-8y`H2 z^>6p+a%IJZw@AQayw7%(QsvjITlJ707u4AQ_4IZ`%2@{W=Ab|~KAFb=g zkNKYGz4^f32X`n>AJbRZLK-FC^&fzHmwp*vGl8$k6kaDMl3Aj($taqEQz#X(bYH(= zwG&>@Pm#vz7Df+}XP{s-c-U#dqxOV!2JLJUPaMxGoCGIgoH8%=;%mYVKA=83XPf|SAmt1D`E6GArrQeN;vV=QLL++kO z&-PZORq@wHSQ6L3*)bMGYk5fq(}&JngLnm>sYnBJ{3zt>E7(C$U5xq0;Z`1oc(My6 zUuj{iG+j;Dv%U^CM4mGi_uYw~aot6FC=xvlPE85k{BHR*sQ5y@dSaQft4_VgqGT0j znuz{TC4>Uv+5TDn>`x~7w~lwJrJfo{hbsP8>GmOC>kyOj=KU1NvU_N7*I&`pU$2P(#6|@L|Fa za&F>bVi%MjB^=_w8qdSgaIxW_4DeHXVu*DW-B4=Es*UL7ADeZXrZe&IENKszO!t} za2=USI(m5#`_?tqS}f2Ub%uAan2j3AVFh2W8JfQ=z))FKb~cCaV7mXpR~moeLUQ8v zX@g?&qC>fZp+H6zYViIxQe=Fji6N^aE8d5~G`n1(@kO{!0=3v!N0f(A7niUCFJ!kp zbQpQEl4*VOf=5&(w>sXWfFZbWK{+7un#m}O#p@pabF>xOJQVI-jnVb&OrvDI`1$af zoQ)84$EOdVxgI!~Qe|IplPXWQsBE+`#A}7n_Ycwi6+TdBnG?s_u}%-Y|9Y!xSW;`2a-4Zf2|t z4I`}*OabR-Z&a0k1Zp!_AfQSMrqAELsn^NYkG$^Ijl@>~be;mFi%VK*Ntjr)SmMUlnEOo3Z;+Lvs)wta`-NN1#+B(ZK&|;gX*&BM_Y*h>&OhaPlo?G zN{=d#(XZN&)SA4b!R~M=y|wNh_f%|hII5GuzfwAtMO>@198fWsH259#wRu8gtyhT< zZR{LE7Kv;**eGjtKfe<9t{$W0!pCoxtEtUg;r431S2oS&ZT3+7}0sXDO_!ZK#!22sG>kH?eMK4CMlx3ID_FW`&vskRgD~9y_hMKry89>SgjfG(LR@c#k#zJccj) z!$&fQE|yiGBx49@hCBa!+c^#BW}24Y8^4_ZbXUEBfkQ!XbA!j+b3>%i51-;vFuDUK zXKbz2XVHzhT=oy26)y|I+#f7yREzFHJOAKyEqgztsZWd=cmMA2{mXBa?qgwB<6>BT zM4k0SOMguVpeRCE)xZs_??5lA3te^~lSQ?VbKI7#j)iBdmQ*&;+wOoaX$Ho385)L4 zi;%?*%T7ZpMF#C)!abw+E+)89wjf_$5?r;)#?>067Q%STw@|YI=j6*?2!uC)pZUi1 z>V$7QMaJwH0ftIeWsUvN_N#dD#dEyb2K)JHIQ`(G6+Ib3C-~cBftk-r^sBm*WXLj2u(1a=`K0NHn!!6*cn5|^mm}Ey<&;HI@&H-YXwFQ-IlJ{HkFoC=y zF*&0zjXO@#&x}59YqZnfUdw+VU}0QQ`W!qe-(b-Fs0MwRT^!G(3bYQfdmSJ?H;$qM z^^T+G*tRqKriV8U8P1*ayy-O?QLAj|L&AU$a2b^=LkR{CXgE-~rLUF}H3a>~vFIJA z7vx4OOL!fp*?FvDpjxjtxzgc zX}Ntaf>)FsvxJZtd@-c7%Mp(q; z=^`}}+HnA<(0&ck!9J{@zY4~lMynAW{p9P;q{3B8H`59f_Tp;0uT80DdhbJsgg)G? zP+yDs#3cu1pbjFCfdUYN-j>sJ^-KOYf>e5=wGrY>1dZ)6J>5)&*+L&(r^CrTDj&{F z)XYpXtTq}qWh*ZT>JOJnBF?|N_1F(jMHz}e82vE?4VUsRY3m}o?Pg~4QYsQW1y zJRBuDITW_*WO?BcWSO7O9s+x29q!WL%6H^7TOqiI<~Cs1R1tywjsW^1jG?P#^UH#d zWoHkv#ya>MD@16j%xy=Nz<9aHSjp%>5b5iFhU0eqYJMR{&~}|zhf7tNA^TqX_#vxR zIR~YgX%%f85B?3GD}r7~O5_$rFEl4g+|2cy^YK?+*HACESXnD=vydy%A|xs783Um^HTC26%?Dp*_xC37Z9i2P6Cs!Z;2+D*_tN$#hGf;uvy> zeBGo!GQolA!&k@&B#V0Zq*sL{-rw-Mh>VM17x4`-xl3QmKJODKRZwxx3YqeE zc2bNh8u9uuBwOoL68SbL@+mockAvc2t!@KeAUxK2bTxU=IHVV9w}bV|F?#x~Zauob zW5PdD%RMW%MFtX{MW?EW=D|fYS=}1Bwkvhm2y3KSHon!VdGaCDDJ|u93c_uK8}n(l z41|-6jVr8wU~`c%4F_eLIUg-U3L06uE0tc8zugvM2#&TN-+MtN~R zW4E<>>$iw97A^Of2S(`3iyz^0g=}<`&ZaC`Y*0A>lnr9;>&SBadci7*zolvwzyo027 zk0JF$K2yPPt#UlV)KZQ;jvS6DnwW8qas0Vtn{q1n=6gx1Kbg{K);r?RSTVcaMw~A* z)f_!Fd8Dc0R|^p3hC}2Ot+6a+eNAx>|RXVJ~6n9QV$F zECkf(LJTH%BbF(xaq|~nR9(V{Loa9_ohNI2GTcaRYTJfGY%%fvS6zFjFU7{wWa2f4n}%)ZpqUh?j*;9S!i zCsgLioO${jCqEOA1J;`yf|vxE+#h8K%wRS`(EVGBdo2<*pyg=eGXXjeW)}w+-SlM> z5SPYO2pu;w-Nc7K8;}yPy2Ue3H}O#IuPJHrjUnL;i|)U3{l#rk5eBSS^2RZ?sh(x|v&cxDJJ$W&syFpk~uv09Hl#8#VmHkyz! zkSd|}@O85kufRcPUmJx!W7kV4nKAj$?)X+WGxG)8VsN$i>ATBBl9 zyG)XtIl%L7eZ8N1Uux|!my{nO!&}bAH-(=DNA$xoEF0weukuCZ?g0vPDOPKb z^WyLdkhXls5}7az2=3VNjP=}kg^lf84`_8KMOb!geqP6U3<1pW*)mDwgF_^Bb2>D0 zi^pzU-?80hvJ@FF?Dk8;(H!%eQ-c-tlIUS~F7H%m7Kg5ME)}|h-S}?v{(mH3chF`B zztb#QK&iRdoRS#Lf2Ub(`M94ERfEt&<4>d&jmj3ljm}Xou#=JgyPl;BUk#Sx$C^J_ ziVL)5=b0Z4zlxqOyL6UAR@o1wYp7bh8)3-}wicE(VBV4BJEGWrSnU$W8@xRPgoYUm z4=LrhRkOO?AA{)6aseo+&m4vuKr@T0V{!$ZapHU&_igbOWm6;&7-(Z1hT12Qb}3A z`)?oeX3W_zGV0e2mOWp|Vyb|>4_`@7X44IbZ1_`uC?+;*-gkCBf4mz7!pt(I6PDd9 z!2K~o|94ZpUpn`$+F!nrz$d{w6k+}Yz&B!WKK`&C^lg)A z_m<8ao=~vZUB1sUC(FrL2s*RU(CLM(dq#$!mC1#U!U{^gy^+~c$;5qcW#&!dw@{_( zk2{J6-sf(%GJj_~WCcRls9lrc!YY+K`ZyyfWN3IzKxBPOz%xqf*pu73NlGjmEZehR zC-q(5@P5<^kGq#O965z5VFmwl>FPX6V=eHZi@G>T&*y`ilEuuY?4@v~_Rl-ks{$dx zD>LBSqKxR$bFL1X?)}-WGByWrUPMnG=S+m?%>hfUR~c*g$3&^5?FN-Pboy?xDHOc} z-%BK;;X1!UWJ}!C_hd^vYo9eASZ#T$HRGT?e>9V83mNgrUzWg}N72#pY>wRfBxK^9 zppyd*D|oiev4!f+mJ2Jp{c-fG+=_O4t=+6S6LEV(Qv8P%Fh4H*J#}G(^s3A>?bq;3 zsbJ24&Z|lLx(Y>4LLKW$*m$cdyJWSAH2Wvz6mrke_*%&-+kM^N_#BF}){L8-EA*h~A5TiS!c|O* zDEMVDoD(8H&AEby%sxFkMPY%?#i%HU&w4SHm}M zb+

vQ0ml|AZ@^j^#j9@p%p|e+Qzip_^XQ`nY8SlyLBUD$bjtfqm~m(xw4P@%q&S zK|r&h4W*W~fNWs^vS*I_(10h-apHcqvi@#s0{}grhBcx@vUW zZLV&~@toeEde>X$S#moEPQ&VYqM=x{@_8{Sbfxa1{g-48C^#S)U8JdVi?VhXSw12F zJLO0cZS~L&e>0%K?o=c~;vnsA%pg{Ou(Yl&C{gib{OGR2U+V$QF7q%~lDnXqvVnad zXT=5S((ju={>>BgX+!lc9fOx=Z-01xE^iJv31^ z55HT&3y}VGhodLzf6(<7P;mx9f^cwm_rcxW-Q8UVC%C&i1b25QVUPg9-GheUPH-9A z$xn7~ckkZ)_ujnfuI{dy>aLpaebd!l^(^t-nQ#`Id){-f$1|5JvddQ&e9lKkIP9t- z(qb~`yaDFbx^(7L%3WYS1X^$u>0LhMa^22*PVD6&BB2Ok#C(+TLiPI_>($=1%4qDh zzyKfp*Eg{f2}#M%X(OF6@j_fB^Coh`8t+!M%Z#-^Vk(wEb&+kXRq@?{r~@mZ)a~QV zfhe!YqCVS<*SN7yew;fWHw-?G>;zk^9orvi9{vN`wGbMJbIZz|)SYPIQmQHqT+|1*WCF#JT>; zr{~Hx22eD^g`WB3X#q9au9@WjXtEJe@x@?xvgn*5DOSI@JGz<9 z?P8sEx|alTuhLH71X_EjrZYY(b%9RkP?S4>+fueZD4xV~O+rHK>vPxTCH?+x-YraM zXa@z?{X{J6-nK=4YCe|uBE2u9JOMB-Z8J$mPl zxx93y?>^mw;~Lr@tV#iLee05lLQ051)+MAjZJ7n}`kGA}2n1J@09hh7@Sz>K&Twge zKSp%TyJQm>3ItOF|44l=n)@~=nPOV|R_~1yP0?%gjI_asGD$yNm^Z&q|L7a~xeaLT zydDvB<#8TA!2bHqWP)$}q;?c-sy-NBIHAQy^nzM7_9f{L_P`LdQO zX4I2-P+X+&SQ1)N-0P-RH)U_-6K&|$=NnA0ro$dYP~G)MCG#5U^|9JzyGm&U>b0fE zX}f9;dAo|K0q)Vx;RCt)5n5FFjPjaJ{rIUjLk2QP!4wUw`Ro_6r9cTGI&3Aa@ zQX>Y71WF>KLW|I!k**1g1T&f@C@VB{L^K$r(+*WC>bbV$Dwy`S;CL)9L}MHd9>>OL zlB5vKSPWtbz>u#rl#6C zSCsC0ZH?DJ($;*-cwW+jbSn0H#RIkA#gR!6af}r z5Wj~}krKkle;oUg*E$Xih53LK?_pGyB3SZDzLr$#!-5x&86!c&ln)v9!M%!UKO%w# z%_@8&c}^;KiEPUV#^Cx#h$RRdw2tM&U>s(5d?9!?HM+x5ibR9Ha4f-WY+k1R(q>1* zx=8(%5w(j^bfLLmXA$kT;Dw;jsQp?Pbvatl%sNpO7|~QYc*LS+o<8<1VEN@E1Qbuf zHCsD-97ju!**S=NY+L5QKDnC}`y-VVlsKO={N(X}TOf=Ifr1ThJ{)VHw^zzSv z8`L!ue4^Af2BxGl(XZoeOfHnSXkCmJ_!!^3mMK^cI;lLZ2B+mt0)JD)Bg=7l$Wf@g z(8(Lq+a|cizU7^+e$ru@T*m_l(zC_L!~b+nuVfQ>4J2PuG|ZTjH^V%h$yUz$)YH1poqALzeEHbhnlpu5$-9ssMjn;p=BS;E=DQdtY}BW;3(v!$Jr4S-eN(#pxh&cl~k z4v|&L%H6`s$r9+~@sGwGV93t?@iiL8h^$)9+D>*KBvt@+j(<%bzWRSM{fh(fA7lPk zK#2cclz%M#F?6$Y@o;tnu>X@-bf{6i`$dPG)LH%lwGe^aFUuPJ^sYI?f3I9NG; zOc}uTZw{=w07Eu_Aul(ZF@T*HV93GC%fiRWVGQ8p`Do?fU}5LsGX`+60}OfC**<#s z`2f8CHH7 z4qfW%6KlE;dDuiUAWCsBkkx0qw77|(uDZT`7Y2A>xN)O|D&_qUxV0tPi^lm|b8$m^ zV^Mo?af4G58uAka2pM|J*1?TOgt$?N*sPkUPtEYM+P&xani$454kD(&=&-$2oebTL zFeF=D;hST%M_7;Z>}Nwy`QpoYl#kj3sb!=z_XRZigmQO0pnk12EBV=lB4(PWOaV? z6CPgV`!I?<0z*KCA;&6PPhO6n^aTa^9%PlvVV6%!X51|Zf}T?op=_pa^Et6U9{?I~ zLtS74YDfKO`hB4C`S)_n>9kJ#P3O&Y!e@fTS+v5Sv5}KepFY*bMj5o=Xwy@8q(d5( zDEeR#4qL-Z(`k%Kx?cd^)@o(vn;zdWJ}O!%Wd*-!{Yun_|( zBq#>7;dL3*zGPX@h+?sf-fNX#CR_V;pL{PK5fFPO{w=%x*v46aLA(3- z)4chCYJPnA0ouVd)(3X?IIhQ9C3xEM(P)KOmy__k8T!;&GCK|gyTj0emNp~CkCeGY zK@#F;hjEG#CKbZBKVFd)LgY%AC~A_!3-;X;@;i)pb}wK-o~nF$gYY!J5eNnI4D<(e z6rkC|iE(1X5B~VOoDYRDGTR0uUL77hpeA{EpZY|3!!~Wt8b+nl&xYdDR$AyguhLF5 znFIfWI5#wgs6?DEL1YpA^s(2mZ$8Z0XI*ytV(@>S#%0{~X8&d)@)rAIZ~BR1$j}y;sGVSGE;Q!zr9Z zdC5*(l?{<-w-e(2@PuQa9t3eO{m3UG`p5>^IwP4U5(y?AxKV-=1@mc!JbcH2?pS<* z+GUHNi)h1evbc&;3!RX!5=!HG$_=)Lae957Zsq%S8u)-;Oz+-#Q+~JAc z(0lpPJSx)V+d}sMx!$OG=5QVvXD3L{;xy$e0!-Fzo_%eTjD~s;dXncCY0>+&QarK{ z|Hhe|+BG(n-cb2nSvGkeBpUg6jIG3*wFFN-vMclh5O#L`aTNK{~^|yPzS~iL)GR*x9JwRwxLWTt7|X;`bD-@d<&M+RkRwEMOJbVi`@`NOV2q z0xBzT&=ZyPq@U?L1^v|>=LO-ud8P>v??z%}6S*=yLtLQ2*9Q+_y$sKVWDAOBzAlqB z99rOVGF?(+qRZ}ftc8Q{b|_)q6Tt){7Q&{z?YyAZsRy{JIe7cPs}tcBqN0XDnW47F zy_+sRWf4}9#lz7*Pub_pOuaYVFiZ3&jlV^MGs*N5E%;Oa@V7oZJ5EsPtqt>+WBoE@J2HUB8_VM6(0>UE#(4Ezi|}SDc;Gc>`o|Z4!2%)@#VD( zVjj4^1-)eN;nF~ZaI=tvUNK@cFuDq%`}u7Bf?}(ocl7QGWn>vykD8#n_xdq-;1p+i zoA4QS2@fjFaR;3~5b9yaz+m`%Z_s}E8KWoE1(%Yq_xM%lgXpnrwtN+qx2}Wz0(Hm4 z1^(?GW!g&Qi_|lZYu*FXHLf{X-^j=_{cFI$J$>f`z>xLCiLLIOPvn}7e*k4-5(k3bu_glxgri|G-n$Ancn{cn)QPe8=)auq z?zP`vga_=f&&DqL1U(XFE_mqCD-E;@#`;Hvxar|)0zodAagnBv@68(X;;4u5eSxox zbQqIrJw&xbsMVNVa(NiPErd$*f59}DD+t*bhriK@$rgXXRnW3eR(TNVe|*I6^`XR% zi6>8+yo35yN0vQ}cDLXwym}A&1+J2H{_W1H^g*PDK$lULIn2H~B+AJu@A!3@yLY4FpA~zB;~zCU}-SzXRT(VACs<^d>?o zcXQZqf+qidvW#6aC!H4ar+`N)i3nwYC;UZdTU-Pbo|He#xax7wTK*LNULf)pbqr!` zo98yP`!Dg^yBP5mS>Ifco;CiGF5z0y-!_5wwvcz@IJT^{UXrM5{Aa}T23Y15xTRzZ z{9vLTL|BC%VQV)KMh|gCpdg$E(V1X;M#_j9Gtpks<3Gr+e{SBMFDI5S@87qRphT-V zub9FBL1C}(hgq(oAKtFGVLb*`)0sZg3zCNiGXcwzf{&a8NsTuX3{Wi`#R+;nM0)v4 z>_^P1qwb3yrU4vcYaRH-D4`I)e_DmKLcxx8hKY-IW^#1yW-H;ozF&<#J-q|{Vp++z z@O-QbQPmIkHXn7aheNW9cH$R(mL@Lmp8nGIeZPSnsDp2K4Hm|o z$ z+C2TvjLUCR)`zIX_)C<1hPxG2CQ@M2ArIr*O2DXCXE{y!f}BFRwc! zqe*i`y$mjdJA$s?ABYR-7hj1JZ1jT;tPf>jZf@?$g9@o8Y~qNUn>hu0oBjcc&6r1- z<4-3G$hshZkp(14Q#C*oEBvg=Fs4R_2Ksl(Wt)~W3pM=Mmt_q%#vcVja_jqaSMwat z41onfOz7h1EfQTIDO)TjTvF_JhLhp(-NjvpE7U8NtDq}jqvkqe5M5!imm;fzsbZD3 zpLWYcDdE@L+T4a(O=dM_^^Nil!@HOh*F~>I!&=E&Q@9PjBVB$y)GWAhSGQ6jt?A_G z$+AC2zV$``-Z8~Lviqa^wDqanQ-r{TW*UQNgDS&@#&2y-v1P8c?c43M?cnxmKE?Iq z_Vo72U1Ha1A42Dtw$(lvN58y&>y{ix_Eu=aTIR`2`?~MU&0j%Zn>HLe2k(o|5e%ap zlN^iAsn_yap<2-#|H@4%x6mdk*2~o^)vM(H&A82Ys++4issY*@>pz{vH2<&}E&M5J z=;C#~($u_EWbE$srtdtH@9ul}{^$7W-lzKv%f{rHu5 z#yJdF2$-|zlQfuk(U>3D44h6sO)58=SQ#(b|3GgsvRnrYyWzNMue)7e>U}F;ayxZ6 zZ~fYx=?}&1jnB;6vE(*AmH;Il1xT#xryFQ~4_@|0o|=$|sw4 zR9p$>Uwt@it%rMsWx{a&n`ivLZ=7ZzMpoQaXEuaYZ-rtm1^qJb>yPv|_f7~|ZImi9+7{E0lZ_DOpKEewHsO1C)D-`(*NbnBGHvmTFqwzjCU3 zi3*3)Hk(g=p1dp)Ek4v-YHyYG7`}Bp%io2cD7Oa}s3m0lRsd))i`7w*j^{DO514;&5UPZ=Hwz0o!j0gO}Yyn6SHA6cs1cw6Ek<7v6^7@YS%lfqD?Uh)aF2k8tbIZGD98WTeru;F#Gc>o=rzh^lS>sd?{y z1|NvT0dMdphe)z*Qx`(gFxqWyADF@jel>*CV#Li)v|We|gQh#mK1iEUtR;k-p-?X9 zg<(Iys_iJ7(cU}4Uly^zDD_dk+hI>?(Q}4q=d+)XiZODF2@FbLEc`G6V3y21SffPy z@9+<2`ocljnSC7X5Is@OK;9Lx3yO4jDJsnTNK!Z6AJOZI7vb*P50$6YVQs;~q@)ruw9c zIZ&k#1K`F5_l>VfeaY^VsW`Hc$;ZJk#I(`dP>ec{)O63tJ`0^SAV>Z;7LRQ7?UabaOF^Ez!F%i zMxlgZ4bK_L7^wlwWg`iqGM#t2=SNzHqIIPv&JD5j)fLxd4bf? zsWS}97GaK{!oyI=OMoL__%wbF1_2tcWa-wCVcKQPG*~ow6?tW%+(LFHcORYmnEaG7 znWXVXcwKo0Ki}LbX0mhwpG(+j+cim?iE^Tg?8zC1cwf4co<)8Tcw(&_xSdB#)p z??60j_v=JmL-+ego^$ux5)|A{f%lgG-&wd|!PgyG*~FmN@hZHT#Q)dl@5;)YQ`=!e zHZBZ5Zf0((kfn=8DKojqG=m}R@Q9SGbgnYS&Io80UD8FZSVXL9 zv06KoP7Sgq@=6+~Nw9H%nP$Y)OP`Q&R1h>U*HAjZY@t0TM%rf7zp@{i@3mhqlC*HK z*Ye^y!>+~+*mV}v2{5^!Xs%4B=xzn{D0vm_-%6=8?0Ep@br=!HA`fyQSCkp{ zCA{Yba$5)DAkUm+L!1I$eX3ZPalDQolaa+H9OInV#g!}&Xsc^nZc!0)#k3|!a7@*X z62GvT5w^n^mXu8xb5yi|0IQ#w6%{?xt}?FN*U?3%N%9Bu&4kECz7k&Q+yE`{@=)qi zyvx1XHrA~|_;Ki;Yaba#7b~3l)kj9)<@;Tc)+0CkQ^hev!>!4 z9T}~>AJXUFzv|pGF%F8TWv<3Ctj4(<<;}$mFIW;MZ0hNp(D3dibL1i9WjF^-KT7Cm zva5R8?~M*Gwfw9TTaxfRueEn%0|OsxHn0s%LD!YF@&CEB5! zZ5?r}!y^MF-Jfp~N$u;aS2&JQbM=Meozz667;8a8zf3TKuh&AkQ7dGuIA7tvSh~&= zeO|u!hyzW3v2Te>adLH}`OB4w)uyU~<*bx)mk5P}f{QG@*pbAlY+>$IhtVCJ%OW_{ zALhT3u3@Sl#Xm=4rm9NszJU09{@fX%sNW#-$UIvcbFbc z^P<^>Bu6aeH?)^ccFzBu*gu~0BmnF(J7DUU09ca$hCfMEito||l~sBHwX1kSj@xPn z(F=c&p`m?dqgdkD=g<#(KoXFXHRH_LtU;MF%4!%@ZX^k%&=vM-j1@hX2!!*B*uz0& zM-O$piPVFaHKIa3fp7D;#JSjZXlpic>1rg&-9h$*g|+fJw2y3DJ{i#r=7l73G`86t zQyNpnhLwbclrCeD{!IUZ(LEcbMbuG)V!ROb%}$beRV?!GY;n1X(04xav{?OPG0=I) zna#6yNNoEz^5yXJe;CGKPuU_2nzWGguU4x#qAyIXV-Sw4OcgvAPj$UXa4N|ZR|hCo zjpwg`bma~-<KS5IH-qgcEed_Qwm%+Oc?Izvc7d$qPcxOD^?b4aDG3+ooI{ zZYd_4-$;&rWXr5WzmyRozA8jLwa;W(U5thI>}{ZA3scVQSx!m)hK`#jkzI;I!9~Hj zC1d(=w*=YiigyShrl@KIJ^GP<$a-j_1*U->_nGTw6nl&}!y|P^(lA$G)J0fIjG$mx zHsS>;x8$0j7P9anM307ihZR|1oKrZRWndE}JEDD?2Mvv0*n@d+gouKnfH`x?lfW~8 zp1LZfh6I%px#YGak@)v`M`B=;6|b@E(#lVMuRP(OJ^EuHt8yy`T+6(Cn!~ZYKLJqP zFjQQmEP5TVMf?EGFQm|d0|Ff#l2fjSMT7i}zUq2WGhQm%+s!GOA*zzjr}Qd+=7@kC zki#OG048a@C>N-h$1c&6L{ z*D98RQTPY4s>Nav)BgdM{@btr3iU8dYPx^Gs1%x7LG2%CmqJiTFP1{c0+mIOP@mBO z4>7&c|Fdp1kod3e9*dklCiV|(V-nD%g#H8l)EsIQP}Cgl5nnJl6vpVt-4O_=Ez^_L zB@X@zhf=^-8wkOKWdVvI{71cy8hKsxE4 z_!Dc{m^e0n3ifkUceqZ-almG1ApHObG|gH<(+1r?VAPryt;-0R z=2;d&M7>V8bAaiT?(!dXqk)8fb@y1WbX(bf;1KhiZZ7d3n5JI!-EO5|i~TD?_t$=D zmPoD!mRGmq3-XQ+eZb@uCg0T69VmQHbkI8SFZ3jT52_oV33M#6MKEz&h_s{t6Mz2` zMm8i3<&=!A*m4AjWQQM`y9yN$5Vu{ia?i)EC#>Hs-);PsZ?O7Jq_VQVcDz=(_Pq8` zYu;;-9Vg?a%nmS9mTj=D^KKM3bBXSP)o(*ub#05Am>22GL$syOHA(8G@y?P%EJrg# zopwIH|JyRPp&|=2#YTX@4gAfDyuvGRS8oy+kv~-4n4ROaa`$z7K80^!Ui$uN7Tq6l z*BQeL;2@)ym>ixQn;c~(2?BRmet5x0>5VTW3MIsc0+KQf87& zCaFIp@{$_}pj&bL}E&P`wAhabP>vtXL5 z>J(`-Xqr}#R*{w^>W168!Esu4RE-Q0$D!(%!)TdC?{azuwN{$BT~XySLiJ z!6wTz#d#Q+h(e&PUcUF`4>v6jP2YX?WCv`0S7!Ri-fg@1IG2mPij& zd*f@07qJqGp4nKn9g}9z$e9U`Y^FDpowkUbTz$5t{vc@Z*eSsqw>@hJZ%i<%+b8q@ zPyYt=7)Am%-?^P9VO=eNAsIXyMBF2Ru`2NWQSk&dVz42oV|M`~Flkkg(bRZ-%{rSu zlY6tQ_vBC?z@t+roRp+}WbSIJ(IE-7Hvxv6bfra^1tRN88BUH!;}ecHg=`9sd^W-C zmW6JDfdmUQB#|AWN|`C!J=s0gBPudSfTwS)xvYG^y)AUi{3c`?knK$mBYU1zPa z@``pz1t$YfGf?zmTM+|HDvRe?e)Ga%{Ocp*@IzMS^sNKg$P4`-$%osHxP%i)GmQcj~t4cy4#j&rrYdoq&-!Z$)P{rAI#`Gu^tEJlTO>(xS}3p$ksX-2AEU z{EM7Cy`mk>x=AlL&lOSk66mKw!_Tf4++N9myPw+@_uQ2&&yfZ=7@Nxz(#h%fV#v{> z02!qrBy68vaQNt-x3KT*O1&doB&Se+79mvow#LjTBx(7fXdWY_Qh{s`l9x6@S&q@} zAX3)T1t1i{Fp@s>R0x8UR3UbI&ibM8kuXG7-#Iw>fuL!<6KBmiQg<8g*EZHVth*WJ z^~@*QeDp-goClNHJ_Yzbq%|$fkC>&OwnR%&z55ElOi#@Zc*KkFla0MbW6=SZH=Cq# zQF+6j^T6{37osz22Iq>U$k8G`+cNr1w5}oviuC?LF!V{fM91)CAL7l9OnaUbVguki zDJcZn+``BItXtwlE(mK0VWu#iWkly>H4O@qOcme1`)`<%OvtchDlB~BP2*C})zuG) zcR|p){Z2k8hm)HdyV|=JAKBuCF5fosz@Oic=TZXqed#qj{;Z>H!gZ^LlHL(S;pRl_ zO@;RM-GBuG(EgSxb}R0D+(D)6RWgrN!kpVBo0aQ`V8Pf7FX3Br`_+oW6G|l91b06W z)bbO94|X&*MJjD_oY;-%32$+M!B4DRb(y}474yoq#an23R)!f&cecAVPXfTaFA`T6 z6Po|n?0QJE_@&%iAsSD3#k#A1m7jjZf53lQu>Xd^`b*sf#t2Svt`+U(lKverdfOBX z^(uvINmaAco5?5BQ(-_TOS>S`0nsp`ikKF$7I8T2;F`~88esS;*CB6E=2a~$oG0w< zQ+V687dY~QI--JpUj#30J4eLEKpJ_tEqwMS`G_xKaTWfSMBY5c&8NVrFMUC!nws^o zgJqvj4fZA`kwkL&1-lyIWuS6d?7e#gw>*k}OZm+H7WGaiCy~!4p$8D6Yma2vDF&m> z0cheRhk?76u9-*DGnNWj3Jc}T^E}5e$LS2Z+7+L*0xLdpv6uLs8eRWT722-3Y8NiR z?MoLav6flfmzZd}TZ`bRF`xLZ@|y2=I3 zLrV0RCLdHjly=z#Jy(%R8R|(nu~=Ecwh>9fFF2x%p)x_cTrYdQR21j4I-zF)p1 z(qHOf$Hz?uV)RX{9_$`WmC_}pT|lFDmr9pxfYLgB52Z2r;?4{CF@KGrpX^DBpOzL* zMVxv?-9b%jwTNcjFUnukCyOT!i=wuK^2qWsKmWX&VlT)Ug@+&Me8$d7P1D1O*NrpF z_eFdL5`K##1E_+FO~dpAPjFQzhuLUqn&O@AL!noYR`#v*YD*ytpNA}F#S`Z(cH?g` z`J@=+${(}USi8l)f>4_01Uix~sQ1ihe^ziF%Rr2|u3slwbeSS~r#_Vvw5Ft4a{VaW zPp$xqB%oKO;3S_0#>7b}(j;q!5cZ*%;~3qHZq=lzyI`yMkfK8ki&_MogrOYG(Qa#A z^W~GLJ{V3pL0VJ7S?A=jZ#qL~-c7g`Fc071#}bt5cz*I~+ES@;*RmKs-Nt@Jq%9Z5 zSKnhx3p#dr2u_?8}NtKaLe%?;ep+z#o}yC zQ8GfKt%#f#u&Zm)^waS(^Fe z1rAD9GEB8J+w>lBMA~@rc}|?bqsjNbaG<-tsMS}HA*U*j$6Mi#acElOf8A%daZHI2 zi}UuO-s(l*EoT?t#JNS_3lYum55tooLuUIw?WljkC|lW@`~i=*LtzvV?}EWHAl?3d zA+M-v_jQg4TYwP4k@NDQG7axJA%A?iQrS-h2Dj)jzpAOwaPCv2W-)o$!}*87nioIT z<)apUV9orIAf1mQ4M3g*Md?X8JOTg`NlYFO%PXSosl!h9M2UjnE9lsvM$EGv52__Zl=DzmIVSTo|ALurE?Q5L2ju^~r=Od0-8==rIvhWnk!9XP4^qpf}C zFKf~YzbdgdZT}2sV^eeX*f$UKkL5;K1G7V?4qMuRSd2w6-O7dDgtBD|B))5v< z!1owQSIa0u7w9n7evcR{=7Fr%Hc(He&QMR)A}kPQ@TyNl!9d5!t7ujiSBpKXlD`LJ zo_$0$im|NlW3C$15e6vQ4@K`f3+1UQ3^Gz{>T&Fo5u(uh+^C z<9OkQS$0dUYodna*g_RMRaK&l84^j+4H&>TF-7~5@xrPUsSHca$z0^0?zyVm<{j1r zQJ=X0F+2*@xLhYfidkKj>EvhqGfrYFEhrR)E8%`kLlb@PqoZ z?kvk-U1$UTyY=HdI@M6MrCH*4se~r8OFn8$jI_%6sMTV`9kj;erghlxh&Kl`Vn0z-Q8RCR^rO!(#W(Cq&C0HKDw+AYxsmeY2tl2)P%DKv$ zF3yESY(OY{_e)| z!hckL@eE;fYpkaGy(-vu+HYEAY&F~j7;JHkck99?HAvB^OnuIC!Gn~gVEOGfR7z6P zAyix(-O-E`6;@el-isTkSg%PG5+A?mkOcbJDA5_>8DWXfFh}WsDCKuz2qEe z)YM(ytFwi6;Q8u4!}wb=*gY7_6s(0@kY=iZRXrHWYv!~ZeS-kU_-bw{!r&tT&N%Uf zxVbg_o&@uK^b%2^ezAs&p+Fw1ga|_mTMN7XH+~EG2*3g~$3|f*1+M7-Djtt=c?|h& z(S+y+n^%UH_P%l>gJXnx9U5vb?k~a?ilgb`=(*z+1f26T0ER;{rWSPHLL_8VGkfICBe|g z_O&wi++QEc!*ovSqWtXs>*7ea%q0GIomS`$M(tP()nYQZX-xW-rbGX(YP%7 ze4we)t@0{d;j5F_O;YTXilp3p=_Up?+Vxa%cmk0jfi)=+?v(M$l}gw&vpC|53EfQZ zg`u=6ynH4bd!b?7=P<`OTJ~e}Ws7$ee&fPxm9{U>;zw2gwp*m-LyF4)8W)}6_=ear>w3% zTB~m#Z{grJl|H~;=q<_lQ~rmAwKdHF6<%rq|6^FQ$l4) zn_BA=Yndsotl`&s9jkZ?$k7NZS-iO6&|zTYoOkWz+{vvtG#ak@S(|$5>KGFVE;>50 zK$N^zPU|g>OW_aAB-XIg!f1O!nD0(S8iYo; z2Ds{2XNaECM$RV0zIfe=f8%y=ZnlugvQ?{9Y*iu(;QvbF@NhU-KK?1mlEu!Myk+V3 z^RQKsO4Qe*Vn#Hf-@xt?a8o?8!043pmbMM|d6l+dfAy$lXSDSJ)Ws5~54*a^>1>ly z?2cCtlM-Tn^0I^xyzU2k;z#L$)tTm{# zBk2pyMc@EQ3Y8w0{GdF3$7e^w2j{L@^0yT{ISP!}++|V=*R-ToMTs>6@+hG$B4m7C zX^66mxlN+>evq`EQk!lkFNjg0j6HSosG^o)V_)15(!0h>y1Uk2`ETe2=pFBFCNz}2 zpMB^4Xu00SUrkQO<<#3Ygytd_A35$Tb%(1Hs2@A~b}lyS|If zf1kOtQdUnh6Unxwe$SozjL;1dV?vPJr<<9RmyNjytg|J4L3)aKRcQH4FQHIeLC#H% zNzOHek4Z>_Pmf?t8iVu8pkNyRTl^>ZS?bR%Ui{?totZx0rHHJo-{WHx#|&(oco$@D z=sDk}zqDamm`m|_Gheqxr)lm`3C*WxolY_;(@*SR)fA;;ie`XN?)R(J$!QAHxS41z z6lj=gkV|*6jhh}Z)!9=W48LzRy1LaLD*8@POWzF&x7!yA_8d2jXtx!1CZPH|=y>&? zUL zuU|5}HTY%q;o8>;FOZ-cr*5nU($%pHG-i|Ma~~!y3mi(q3tbYK^yhXOV`7QC6C3oe zXb&E0vY{@Ab6|H--Ll@A8pg76iw1*G@k}dBcjg2%;(pC@3{HV^Fsu$bk2Y#MxxJ4& zTcVFJj;rsJ-xZG)7j;9UM!B5l+q76eo6ISG&0Va;AtaADuk84Q#hIe6MI>8O#!yOB zX6KQN16v-jHR7A98ts^i`ZW%z`a>W3G#)PL5V(Cx?r*UQP|uXX`WTje~5{ z7EzdYhhNcRYFv4R_~;jcOx58i49(hD{B$1h7e2A%y*!WwqFZCfj$6e6M0fx8vGDBN|;x&-&V~FA*RV7W%oi!5na|bklRIC{SXWvbo5~ zky7HJ>%}d0o8Qwy`|bn>b@aT@fY9~0MLRDh^i$(h$H_aV^uTm9ukik6(3U3C!dGZ? zw^>NoWplECEU6Z8%Umty?+OxrwsvaDY8KVi3$UlmetvbEbydF4<9@Z5rSWHyzdnC) z3$67C7q_3awk7&r%JbRos8UTp(8KQN{^OFrc{K!PY~KNy*fGa^qHNt*a@PhG>?>U5 zX7PKCklQYv*}=*-)yQMDk|~Gbr(vfBuKUS1{R%;k97=ZRdZ8wESf$UTC*`-N^A?`|*clU5wXvF;fjH)z0Iy^jFhqZfF z^Yht0A$|}K)lS(2r-$Nc6MQ!=xaTzj$3oeb>%_D55z(bdP#b6IPhhjV3x^400RH5t z030>wx=}?jtbv$dc=Fzs>Hsbk!&TbV8*Y=}L1X>JmBBt=L=C%4FVUY+6`L);fLTD< z&R1$Bz?XkIa~LGdgQ$v46}~KwfWbJ%;T8uUlAC21L-(hQv268GC^w(m z4uKc+B4J;*91T+TI{RfgGd<4HD40C%L==X0#=C5}vxG9oMLChE&&n5xst^Vnh__Im6u`_Wag1(z7!?HZ}Rx4FtEH~8N zmamZL$yXt7l>Or&+QEa4GE|)W$=RQ^-da*vCx{A z%#@w#7ncHmZ}}q&9}-yGWEzbVVRPwxHDtDHyZ=!PEufRpC4IA+YH+jqB? znvCS{(OZ9Ek)(ANkL4J9?EX{J{K8hCPhxqMQUN<3jWH^36n0X9uj*Ql;csLj!D&nk zQ97Z@=j+`(r{E=&vn0jl}N*d-JMQW6heHGPl(2l14I6koQ);Flq6 zg@hp{+eLxzn^VLj$dz7cYgAf*laWqyJPbiF{LjQ}^l1@O$40ngsFvdnj}Q+IB3&-G zWw^XZDb1rVV4*#lJ+Cq~ppmtajgYO7ReGN-+r$WRSxoec{n}gGw7ECxw`2C#LD8}{ ze@p!fse6L6<7=bAW4TONmFpR+lshCLk|<=f5xhw_Dj|`fF~`s5GJ|W@UUC8VU;st%byV_nVZ3i?&7IvoR`T zW82G|Tk^gk=MXiCW46@FX|w!9p|2b!H`U-4os5iuz$ixICG<4V&Sqy9BG%(Tiraqi zgT6A~e(&%3ws`AwMChg<$<-TNe&Yy#cz*cjw?k0Yh`d4I0x5e~%0T(ti}6jHi8uwb z<_(Lz0!>Pu_0$jRQ`prDb~5+Gob_jz;}J>B;Geh+P+O)La9}WP8M}L0Bcif6Wgz5x zFOSK@yQzFWZa-aj-7IEyMFX}0jCF>7$s#zFL9&E9x!R<1*1#5Ny>wRK3PyyZ49ZMJ zty1E&MBqpz5>@M5t3c-n(EbtA_Oj^Mh)@XY_3w-<{iAGTB_QhgZ&4Ctc~qjBZC7B_ z0euVf^Xw#!LF6rQq|RXetY74BF+IHq3HuT@3jGq#Ul;*$B~WI1QsEM(Kp&}22{^=N z2{Z-|bmRn|q&H75b+L^wrBFm>iS$^P+&3~h&q}IDT=D_zpLRIE>y)zSTHs46f%!U| zz1cL3T+ahsg$soBOL+`92GGsN3M%$Ef1hoOf&*J-dPP$?{zg@ z*LkkrU-$F9pYQ$L&-3r^^RRcNOEJT85Vl#S58DkkPlF;BR3cADkYo!K5t}-ZKI_Ru zWGdr&wR0ADTN?=>-?~I8d4KLmX1PZCwMt{i1|LDK_pEQLwu7<6F;OS@TA!-#P7X423`0TOk>zcA`^S_hpe0Vr3x(_=xjrgZ93}( zo$oYUTE{5)>PX(c6Pr5{T_duzi_9q}@C(i~ws?0_bJ#-na$k3zQXM+T1V1Hr_ zq-4B8V?pL+Qa_!-bdy+V9B?m(d?fR!rFmV0-CZT8oG}LTl|2D*@J~q1Sx)K_x*5H8 ztQ{7%zsgY&qZD-&JA5RqS!-lk>hgVMZ%O){a7Vx4`G^=n4QyVf=rf|Ly#M{sXm^L$ zYu#8ZOSOdhG)~QGjP!fUSd@5L|k+yucT`4I(WUG6ja5dr;IELXkE!PC5u z&5f5hnfF7-W;VYKqzP#E4G*jY=qi+Jv0o6QXdC#jiO-zkYk}Moef`*LLq$0IksPho zUC~*FsWzDKbDv0?r4#EDdh|L(BIgM~x0dFuw1N|a8eBxO2g&-hZ_uwCtHejB1M{Tb zV_&X4T3Z=hS(!j5SdP60`=gduR?vpuOS-Qh%SOo?!SZYR<8XFQ_hurU?6@(#LyQd+ zgA!Gbj{^tP1p3e1l#k#!*QXyT9+e>aV;|2UB+Y$8CYp?!{t?X1*uoIUZlg;zB&kQT zZVXTDI({try5H+e(D@qfSJD~9*yPw*({=sFv$JBTa3ywlU6mE6=z2NJv?gGr7eLF2j8dy1%Mb6+t^;D6uYFa7XN8_sYQdhXvx}A!>r>^tg3q)_ z?2hr`z}l6zahs8oR+7_W>vi%Tdkn=^L#Ug_b5{F1*-^~F+5)Z1fsQ9tsq>%b7iA0d zDza*`wib`h0W4ww7O_B-NzH&08ey;L{{p@NutW0CW%OA0x8PF8b}p&F{JILGb;2)5 zB`)CcV^7Pfp&IS;_E;LtH;aY~Hz7ySVuLzgI9(2Yvb*P1(U>ZhhQCZMN*@{}cC{on zjrn_cxtS!`mHC#Ex?ZKCM_Xjg5^|nz-I|JaO>8Hc$K<#-(iudP%BiBKT-m z%h9q^*_FO?)|9kdk1SgA4G!Lyh;-pSyT8+(KC&;Z?|eI;yHFXG^=Bm=c&1EE3d6mA z>m{$)B@Ml5pN0l7*iyM4;|8zV$3c@hu&t-IurU%+ z640ZYZ7MrjT^4D{s;`-Pi0_aiQl&4UW{XY*n|b9(w>x1b-x-CE2ZOzuS@;0$Al3(l zHmmd?)wT6|A*q|VKvxOiltiQM?yFbteoTkki{;B~6Z>mWxxpKm}W92E1sxKfic?ew4?{a)n|q(MUFc z=R$aJ==a;%J3lWJSfvPmf^q#F1NI0X&h?EXZsgfXNE`=RrXDVo=&~p`!meIwmuo;a zP7c3DzdRDQ^rn&vs%q|VqNT(&yv=mRu{=WqOwjv^Z=yZ5tHVmEiIq~4Ns#lr>>l&TuV())R2+yz9=xLN%?$e z9Ep(F7441)RB2pK?*UH9cKg)_^%9KFv`6HD)~{N0sI__8yMHOX6x8r(Pz zRd(S-d~L6xM0&!Aoa;WR>go>X$)aF~jo@)@mx>IgN0VI5;(gch=FGY=v=USut zB>-~w`n`;UhDM!O==VIo@rLvCl0e;VHONKV#qqXwD4?{`L8e&3jb&Q<`%2m*#9%R3 zdf~JK#&e3QSj6B2c$;U&^}UmO(9?ZjCaHKp?a2j+JViNXhjuEsEhR~-p!t@W&zNL9 z=}yeK)~1%G_c-i6FYiVnfy;Td?O=xDM0C^PM156j6sNNjj?^c2WSg4s*JI253i)-) z=|rOhuRXM_nnw2GS5rTLwxhoM-sS>G8L{g-c8Tclr`){GcFdvnm)MxLb!sgK@4~}z z_%1y8AlBla<9$*|$L~CE*|a_%PXpqqo9?$0JNKU?&TKi7d~W23zRsLliq{|J5-%S8 zF)(f=DST?3*|-5UZDCpnc()ziMhq(rBf@81e!5sT6dkhx1{e)G+3#S|H)sgY9HMzzzcsgKAPEFpR{*O@t$LZzuzZR!Vo9o`nGISIknh5 z%qYco@!4*#PpJdV01wk6jL<;HxjP<$dY2)ypoQ(RM#i5YM-Rujh?Vv{2@muWy{DJu+DCTMXYeJ1L8o#fTqx zTPJ&!`CMft2~pqs!Nb#m5xTsGM*1p&nO+LKeKM&g6foE;jUhwbMrWo5c152TV>`Q% z047t5ov*kNzhz|kedMLUJej#={=G!FP>01&HCs@@N>D8v9?ylmtIc~NR z&IHMIU75A3++BfWM@=AYY>p-V)0bxS2oe9g>cbZKPRG6S_hc7e2PUJ+`!6!YDAUc} z<{#$I_TDUMH51p}H{fCYfd89JZoE>nQaedYQ~Dv+!!@KN|r42l2=**kyDDqQ|w46Y4^ z{u~p+XdpsjkBANd))mo#pl}_C=z1`SkQpSJ3)h8;>fkV4knS%!xUP`x^JDy7A3_%b z74a3;2L;1_)`7LPVM3zLPq{)JLR-Wa41$88qQ1f`(H{I*NLM5m1VZRSMfV0mz;s|g z`wDe%A=l`qT0mf-2SOwU0zsi7H4AlMT~QBhunt_bFWO+Zj_7(iFp#$BdO9#2A>!%h zTA(nvwrCH)Al+Yj4+g=1_0@(#ME5J~{V%yXFoX~h^|P-I94rJ&{T&M=dI%X!L~$YF r%FZ{;`^VMQvLz4#6@DDXe>hyEKo4TzUpE`94H2HPxRR2o)#-l&X5~%^ From 3b3250339a12a5ec90923718ced3240d9728e841 Mon Sep 17 00:00:00 2001 From: Stephen Twigg Date: Tue, 3 Feb 2015 18:15:01 -0800 Subject: [PATCH 284/688] Explicitely convert results of Bits Muxes to UInt Chisel updated to emit SInt result instead of UInt so this commit addresses this change. --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 794e3447..9040594b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -145,7 +145,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en) + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) From 7b86ea17cfc3dedb5989728847c8bf86493340ad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Feb 2015 19:37:32 -0800 Subject: [PATCH 285/688] rename L2HellaCache to L2HellaCacheBank --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9040594b..a033be2f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -328,7 +328,7 @@ class L2DataArray extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCache(bankId: Int, innerId: String, outerId: String) extends +class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { require(isPow2(nSets)) From 0c66e70f147273726f3b823249a7f20a520029af Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 6 Feb 2015 13:20:44 -0800 Subject: [PATCH 286/688] cleanup of conflicts; allocation bugfix --- uncore/src/main/scala/cache.scala | 13 ++++++------- uncore/src/main/scala/coherence.scala | 2 -- uncore/src/main/scala/tilelink.scala | 3 +++ uncore/src/main/scala/uncore.scala | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a033be2f..859e0bc3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -397,7 +397,7 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => arb.valid := acq.ready acq.bits := acquire.bits - acq.valid := acquire.valid && (acquire_idx === UInt(i)) + acq.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires @@ -475,7 +475,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val cacq = io.inner.acquire.bits val crel = io.inner.release.bits val cgnt = io.inner.grant.bits - val c_ack = io.inner.finish.bits val mgnt = io.outer.grant.bits val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) @@ -498,7 +497,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) io.has_release_match := !crel.payload.isVoluntary() && - co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + crel.payload.conflicts(xact_addr_block) && (state === s_probe) val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) @@ -824,12 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St } //TODO: Allow hit under miss for stores - io.has_acquire_conflict := (co.isCoherenceConflict(xact.addr_block, cacq.payload.addr_block) || - xact.addr_block(idxMSB,idxLSB) === cacq.payload.addr_block(idxMSB,idxLSB)) && + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + cacq.payload.addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && (state != s_idle) && !collect_cacq_data - io.has_acquire_match := xact.hasMultibeatData() && - (xact.addr_block === cacq.payload.addr_block) && + io.has_acquire_match := xact.conflicts(cacq.payload) && collect_cacq_data io.has_release_match := !crel.payload.isVoluntary() && (xact.addr_block === crel.payload.addr_block) && diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 201e67ab..82f72dea 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -135,8 +135,6 @@ abstract class CoherencePolicy(val dir: DirectoryRepresentation) { def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool } class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3fc0e4e7..afbf4dc0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -49,6 +49,9 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) + + def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block + def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cc4755f0..1230771c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -300,10 +300,10 @@ class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: Stri val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) probe_initial_flags := ~(io.tile_incoherent | myflag) - io.has_acquire_conflict := co.isCoherenceConflict(xact_addr_block, cacq.payload.addr_block) && - (state != s_idle) && - !collect_inner_data - io.has_release_match := co.isCoherenceConflict(xact_addr_block, crel.payload.addr_block) && + io.has_acquire_conflict := xact.conflicts(cacq.payload) && + !collect_inner_data && + (state != s_idle) + io.has_release_match := xact.conflicts(crel.payload) && !crel.payload.isVoluntary() && (state != s_idle) From 0a8722e881056c75ece113243f4544e702895997 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Feb 2015 00:35:18 -0800 Subject: [PATCH 287/688] bugfix for indexing DataArray of of small L2 --- uncore/src/main/scala/cache.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 859e0bc3..4380e255 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -281,9 +281,9 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasCacheBlockAddress with HasTileLinkBeatId with HasL2Id { + val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) } @@ -311,8 +311,8 @@ class L2DataArray extends L2HellaCacheModule { val wmask = FillInterleaved(8, io.write.bits.wmask) val reg_raddr = Reg(UInt()) val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_block, io.write.bits.addr_beat) - val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_block, io.read.bits.addr_beat) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) when (io.write.bits.way_en.orR && io.write.valid) { array.write(waddr, io.write.bits.data, wmask) @@ -524,7 +524,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) @@ -658,7 +658,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, ou io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) @@ -881,12 +881,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: St io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_block := xact_addr_block + io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := read_data_cnt io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_block := xact_addr_block + io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := write_data_cnt io.data.write.bits.wmask := SInt(-1) io.data.write.bits.data := xact_data(write_data_cnt) From 1bed6ea498de86d51815b0c8b95f861bbe8f6ad0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sat, 28 Feb 2015 17:02:13 -0800 Subject: [PATCH 288/688] New metadata-based coherence API --- uncore/src/main/scala/broadcast.scala | 380 ++++++++ uncore/src/main/scala/cache.scala | 1236 ++++++++++++------------- uncore/src/main/scala/coherence.scala | 985 +++++++++----------- uncore/src/main/scala/directory.scala | 43 + uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/memserdes.scala | 314 +++++-- uncore/src/main/scala/metadata.scala | 188 ++++ uncore/src/main/scala/tilelink.scala | 307 +++--- uncore/src/main/scala/uncore.scala | 508 +++------- 9 files changed, 2117 insertions(+), 1851 deletions(-) create mode 100644 uncore/src/main/scala/broadcast.scala create mode 100644 uncore/src/main/scala/directory.scala create mode 100644 uncore/src/main/scala/metadata.scala diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala new file mode 100644 index 00000000..bd8a3f24 --- /dev/null +++ b/uncore/src/main/scala/broadcast.scala @@ -0,0 +1,380 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object L2StoreDataQueueDepth extends Field[Int] + +trait BroadcastHubParameters extends CoherenceAgentParameters { + val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation extends Bundle with BroadcastHubParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + +class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent + with BroadcastHubParameters { + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + // Create SHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + + // Propagate incoherence flags + trackerList.map(_.io.incoherent := io.incoherent.toBits) + + // Queue to store impending Put data + val acquire = io.inner.acquire + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() + val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } + when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + + // Handle acquire transaction initiation + val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + val block_acquires = any_acquire_conflict + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + alloc_arb.io.in(i).valid := t.acquire.ready + t.acquire.bits := acquire.bits + t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + t.acquire.valid := alloc_arb.io.in(i).ready + } + acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires + alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + + // Queue to store impending Voluntary Release data + val release = io.inner.release + val voluntary = release.bits.payload.isVoluntary() + val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + + // Handle releases, which might be voluntary and might have data + val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} + for( i <- 0 until trackerList.size ) { + val t = trackerList(i).io.inner + t.release.bits := release.bits + t.release.bits.payload.data := (if (i < nReleaseTransactors) + DataQueueLocation(rel_data_cnt, inVolWBQueue) + else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits + t.release.valid := release.valid && (release_idx === UInt(i)) + } + release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) + + // Wire finished transaction acks + val ack = io.inner.finish + trackerList.map(_.io.inner.finish.valid := ack.valid) + trackerList.map(_.io.inner.finish.bits := ack.bits) + ack.ready := Bool(true) + + // Wire probe requests and grant reply to clients, finish acks from clients + // Note that we bypass the Grant data subbundles + io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + { case TLId => params(OuterTLId) + case TLDataBits => internalDataBits }) + outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + // Get the pending data out of the store data queue + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.payload.hasData() && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } +} + +class BroadcastXactTracker extends XactTracker { + val io = new ManagerXactTrackerIO +} + +class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = ManagerMetadata.onReset + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + + io.has_acquire_conflict := Bool(false) + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + + io.outer.acquire.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + + //TODO: Use io.outer.release instead? + io.outer.acquire.bits.payload := Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(UInt(0)) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := Mux(io.irel().hasData(), s_outer, + Mux(io.irel().requiresAck(), s_ack, s_idle)) + } + } + is(s_outer) { + io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) + when(oacq_data_done) { + state := Mux(xact.requiresAck(), s_grant, s_idle) + } + } + is(s_grant) { // Forward the Grant.voluntaryAck + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(io.inner.grant.fire()) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val coh = ManagerMetadata.onReset + + assert(!(state != s_idle && xact.isBuiltInType() && + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + + val release_count = Reg(init=UInt(0, width = log2Up(nClients))) + val probe_flags = Reg(init=Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(probe_flags) + val probe_initial_flags = Bits(width = nClients) + probe_initial_flags := Bits(0) + // issue self-probes for uncached read xacts to facilitate I$ coherence + val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() + val myflag = Mux(probe_self, Bits(0), + UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) + probe_initial_flags := ~(io.incoherent.toBits | myflag) + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ognt_ack = Reg(init=Bool(false)) + val pending_outer_write = xact.hasData() + val pending_outer_read = io.ignt().hasData() + val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + + io.has_acquire_conflict := xact.conflicts(io.iacq()) && + !collect_iacq_data && + (state != s_idle) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state != s_idle) + io.has_acquire_match := Bool(false) + + val outer_write_acq = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + val outer_write_rel = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = io.irel().addr_beat, + data = io.irel().data))(outerTLParams) + val outer_read = Bundle(GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block))(outerTLParams) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := outer_read //default + io.outer.grant.ready := Bool(false) + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := coh.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + when(pending_ognt_ack) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } + //TODO add finish queue if this isnt the last level manager + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(UInt(0)) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + probe_flags := probe_initial_flags + release_count := PopCount(probe_initial_flags) + state := Mux(probe_initial_flags.orR, s_probe, + Mux(pending_outer_write_, s_mem_write, + Mux(pending_outer_read_, s_mem_read, s_make_grant))) + } + } + is(s_probe) { + // Generate probes + io.inner.probe.valid := probe_flags.orR + when(io.inner.probe.ready) { + probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + } + + // Handle releases, which may have data to be written back + io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready + when(io.inner.release.valid) { + when(io.irel().hasData()) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits.payload := outer_write_rel + when(io.outer.acquire.ready) { + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } .otherwise { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_write) { // Write data to outer memory + io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.bits.payload := outer_write_acq + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + state := Mux(pending_outer_read, s_mem_read, s_mem_resp) + } + } + is(s_mem_read) { // Read data from outer memory (possibly what was just written) + io.outer.acquire.valid := !pending_ognt_ack + io.outer.acquire.bits.payload := outer_read + when(io.outer.acquire.fire()) { state := s_mem_resp } + } + is(s_mem_resp) { // Wait to forward grants from outer memory + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(ignt_data_done) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4380e255..75eb897d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,7 +2,6 @@ package uncore import Chisel._ -import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -11,6 +10,7 @@ case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -140,7 +140,6 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip val resp = Vec.fill(nWays){rstVal.clone.asOutput} } - val metabits = rstVal.getWidth val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) @@ -148,17 +147,14 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } + val metabits = rstVal.getWidth val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) when (rst || io.write.valid) { tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) - for (w <- 0 until nWays) { - val m = tags(metabits*(w+1)-1, metabits*w) - io.resp(w) := rstVal.clone.fromBits(m) - } - + io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } @@ -166,35 +162,17 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val refillCyclesPerBeat = tlDataBits/rowBits - val refillCycles = refillCyclesPerBeat*tlDataBeats + val blockAddrBits = params(TLBlockAddrBits) + val refillCyclesPerBeat = outerDataBits/rowBits + val refillCycles = refillCyclesPerBeat*outerDataBeats require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) - require(amoAluOperandBits <= tlDataBits) + require(amoAluOperandBits <= innerDataBits) + require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states } -abstract class L2HellaCacheBundle extends TLBundle with L2HellaCacheParameters - -abstract class L2HellaCacheModule extends TLModule with L2HellaCacheParameters { - def connectDataBeatCounter[S <: HasTileLinkData](inc: Bool, data: S) = { - val (cnt, cnt_done) = - Counter(inc && data.hasMultibeatData(), tlDataBeats) - val done = (inc && !data.hasMultibeatData()) || cnt_done - (cnt, done) - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload)._2 - } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData](in: DecoupledIO[T]) = { - connectDataBeatCounter(in.fire(), in.bits) - } - def connectIncomingDataBeatCounter[T <: HasTileLinkData](in: ValidIO[T]) = { - connectDataBeatCounter(in.valid, in.bits)._2 - } -} +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -206,22 +184,29 @@ trait HasL2InternalRequestState extends L2HellaCacheBundle { val way_en = Bits(width = nWays) } -trait HasL2Data extends HasTileLinkData { +trait HasL2BeatAddr extends L2HellaCacheBundle { + val addr_beat = UInt(width = log2Up(refillCycles)) +} + +trait HasL2Data extends L2HellaCacheBundle + with HasL2BeatAddr { + val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) + def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) +} + +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: ManagerMetadata) = { + def apply(tag: Bits, coh: HierarchicalMetadata) = { val meta = new L2Metadata meta.tag := tag meta.coh := coh meta } } -class L2Metadata extends Metadata with L2HellaCacheParameters { - val coh = new ManagerMetadata -} class L2MetaReadReq extends MetaReadReq with HasL2Id { val tag = Bits(width = tagBits) @@ -250,7 +235,8 @@ class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaW class L2MetadataArray extends L2HellaCacheModule { val io = new L2MetaRWIO().flip - val meta = Module(new MetadataArray(() => L2Metadata(UInt(0), co.managerMetadataOnFlush))) + def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) + val meta = Module(new MetadataArray(onReset _)) meta.io.read <> io.read meta.io.write <> io.write @@ -259,7 +245,7 @@ class L2MetadataArray extends L2HellaCacheModule { def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && co.isValid(meta.io.resp(w).coh)).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) @@ -281,7 +267,7 @@ class L2MetadataArray extends L2HellaCacheModule { } class L2DataReadReq extends L2HellaCacheBundle - with HasTileLinkBeatId + with HasL2BeatAddr with HasL2Id { val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) @@ -289,7 +275,7 @@ class L2DataReadReq extends L2HellaCacheBundle class L2DataWriteReq extends L2DataReadReq with HasL2Data { - val wmask = Bits(width = tlWriteMaskBits) + val wmask = Bits(width = rowBits/8) } class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data @@ -305,7 +291,7 @@ trait HasL2DataWriteIO extends L2HellaCacheBundle { class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO -class L2DataArray extends L2HellaCacheModule { +class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip val wmask = FillInterleaved(8, io.write.bits.wmask) @@ -320,23 +306,22 @@ class L2DataArray extends L2HellaCacheModule { reg_raddr := raddr } - io.resp.valid := ShiftRegister(io.read.fire(), 1) - io.resp.bits.id := ShiftRegister(io.read.bits.id, 1) - io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, 1) - io.resp.bits.data := array(reg_raddr) + io.resp.valid := ShiftRegister(io.read.fire(), delay+1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) + io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) io.read.ready := !io.write.valid io.write.ready := Bool(true) } -class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) with L2HellaCacheParameters { - +class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent + with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) - val tshrfile = Module(new TSHRFile(bankId, innerId, outerId)) + val tshrfile = Module(new TSHRFile(bankId)) val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray) + val data = Module(new L2DataArray(1)) tshrfile.io.inner <> io.inner tshrfile.io.meta <> meta.io @@ -345,45 +330,28 @@ class L2HellaCacheBank(bankId: Int, innerId: String, outerId: String) extends io.incoherent <> tshrfile.io.incoherent } +class TSHRFileIO extends HierarchicalTLIO { + val meta = new L2MetaRWIO + val data = new L2DataRWIO +} -class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - val meta = new L2MetaRWIO - val data = new L2DataRWIO - } - - // Wiring helper funcs - def doOutputArbitration[T <: Data]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]], - count: Int = 1, - lock: T => Bool = (a: T) => Bool(true)) { - val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, count, lock)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o, i) => o.valid := in.valid && (UInt(i) === in.bits.id) } - } +class TSHRFile(bankId: Int) extends L2HellaCacheModule + with HasCoherenceAgentWiringHelpers { + val io = new TSHRFileIO // Create TSHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId, innerId, outerId)) + Module(new L2VoluntaryReleaseTracker(id, bankId)) } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId, innerId, outerId)) + Module(new L2AcquireTracker(id, bankId)) } - val wb = Module(new L2WritebackUnit(nTransactors, bankId, innerId, outerId)) + val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.tile_incoherent) :+ wb.io.tile_incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation val acquire = io.inner.acquire @@ -414,29 +382,14 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac } release.ready := Vec(releaseList.map(_.ready)).read(release_idx) - // Wire finished transaction acks - val finish = io.inner.finish - val finish_idx = finish.bits.payload.manager_xact_id - trackerList.zipWithIndex.map { case (t, i) => - t.io.inner.finish.valid := finish.valid && finish_idx === UInt(i) - } - trackerList.map(_.io.inner.finish.bits := finish.bits) - finish.ready := Vec(trackerList.map(_.io.inner.finish.ready)).read(finish_idx) - - // Wire probe requests to clients + // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) - - // Wire grant reply to initiating client - doOutputArbitration( - io.inner.grant, - trackerList.map(_.io.inner.grant), - tlDataBeats, - (m: LogicalNetworkIO[Grant]) => m.payload.hasMultibeatData()) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(outerList.size), - {case TLId => outerId}) + val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -449,10 +402,495 @@ class TSHRFile(bankId: Int, innerId: String, outerId: String) extends L2HellaCac doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } + +class L2XactTrackerIO extends HierarchicalXactTrackerIO { + val data = new L2DataRWIO + val meta = new L2MetaRWIO + val wb = new L2WritebackIO +} + +abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { + if(data.refillCycles > 1) { + val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) + } else { (UInt(0), inc) } + } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + in: DecoupledIO[T], + beat: UInt = UInt(0), + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.fire(), in.bits, beat, full_block) + } + def connectInternalDataBeatCounter[T <: HasL2Data]( + in: ValidIO[T], + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 + } +} + +class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.release.bits.header.src.clone) + val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = xact_meta.coh + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := io.irel().isVoluntary() + + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + + io.data.read.valid := Bool(false) + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.wb.req.valid := Bool(false) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact_src := io.inner.release.bits.header.src + xact := io.irel() + data_buffer(io.irel().addr_beat) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := Mux(io.meta.resp.bits.tag_match, + Mux(xact.hasData(), s_data_write, s_meta_write), + Mux(xact.requiresAck(), s_inner_grant, s_idle)) + } + } + is(s_data_write) { + io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) + when(write_data_done) { state := s_meta_write } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? + } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val state = Reg(init=s_idle) + + val xact_src = Reg(io.inner.acquire.bits.header.src.clone) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + Reg(io.iacq().data.clone) + } + val xact_tag_match = Reg{ Bool() } + val xact_meta = Reg{ new L2Metadata } + val xact_way_en = Reg{ Bits(width = nWays) } + val pending_coh = Reg{ xact_meta.coh.clone } + val pending_finish = Reg{ io.outer.finish.bits.clone } + + val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val do_allocate = xact.allocate() + val needs_writeback = !xact_tag_match && do_allocate && + xact_meta.coh.outer.requiresVoluntaryWriteback() + val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src), + pending_coh.outer.onHit(M_XWR)) // WB is a write + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.ognt(), xact.op_code())) + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + + val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val mask_self = Mux(xact.requiresSelfProbe(), + full_sharers | (UInt(1) << xact_src), + full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) + val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) + + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data //default + amoalu.io.rhs := data_buffer(0) // default + + // TODO: figure out how to merge the following three versions of this func + def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) + def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { + val old_data = incoming.data + val new_data = buffer(incoming.addr_beat) + val amoOpSz = UInt(amoAluOperandBits) + val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> offset*amoOpSz + amoalu.io.rhs := new_data >> offset*amoOpSz + val wmask = + Mux(xact.is(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), + FillInterleaved(8, xact.write_mask()), + UInt(0, width = innerDataBits))) + buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + } + + //TODO: Allow hit under miss for stores + val in_same_set = xact.addr_block(idxMSB,idxLSB) === + io.iacq().addr_block(idxMSB,idxLSB) + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := !io.irel().isVoluntary() && + (xact.addr_block === io.irel().addr_block) && + (state === s_probe) + + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits.payload := Mux(do_allocate, + xact_meta.coh.outer.makeAcquire( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + io.outer.acquire.bits.header.src := UInt(bankId) + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.outer.finish.valid := Bool(false) + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_cnt, + data = Mux(xact.is(Acquire.putAtomicType), + data_buffer(innerDataBeats), + data_buffer(ignt_data_cnt))) + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + io.data.read.valid := Bool(false) + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.read.bits.addr_beat := read_data_cnt + io.data.write.valid := Bool(false) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(write_data_cnt) + io.meta.read.valid := Bool(false) + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := Bool(false) + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh := pending_coh + + io.wb.req.valid := Bool(false) + io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.way_en := xact_way_en + io.wb.req.bits.id := UInt(trackerId) + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + data_buffer(io.iacq().addr_beat) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + irel_had_data := Bool(false) + ognt_had_data := Bool(false) + state := s_meta_read + } + } + is(s_meta_read) { + io.meta.read.valid := Bool(true) + when(io.meta.read.ready) { state := s_meta_resp } + } + is(s_meta_resp) { + when(io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + pending_coh := io.meta.resp.bits.meta.coh + val _coh = io.meta.resp.bits.meta.coh + val _tag_match = io.meta.resp.bits.tag_match + val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _needs_writeback = !_tag_match && do_allocate && + _coh.outer.requiresVoluntaryWriteback() + val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + when(_is_hit) { pending_coh := pending_coh_on_hit } + when(_needs_probes) { + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + } + state := Mux(_tag_match, + Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade + Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + } + } + is(s_wb_req) { + io.wb.req.valid := Bool(true) + when(io.wb.req.ready) { state := s_wb_resp } + } + is(s_wb_resp) { + when(io.wb.resp.valid) { state := s_outer_acquire } + } + is(s_probe) { + // Send probes + io.inner.probe.valid := pending_probes != UInt(0) + when(io.inner.probe.ready) { + pending_probes := pending_probes & ~UIntToOH(curr_p_id) + } + // Handle releases, which may have data being written back + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + pending_coh := pending_coh_on_irel + // Handle released dirty data + //TODO: make sure cacq data is actually present before accpeting + // release data to merge! + when(io.irel().hasData()) { + irel_had_data := Bool(true) + mergeDataInner(data_buffer, io.irel()) + } + // We don't decrement release_count until we've received all the data beats. + when(!io.irel().hasMultibeatData() || irel_data_done) { + release_count := release_count - UInt(1) + } + } + when(release_count === UInt(0)) { + state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + } + } + is(s_outer_acquire) { + io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + when(oacq_data_done) { + state := s_outer_grant + } + } + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().hasData()) { + mergeDataOuter(data_buffer, io.ognt()) + ognt_had_data := Bool(true) + } + when(ognt_data_done) { + pending_coh := pending_coh_on_ognt + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := Mux(!do_allocate, s_inner_grant, + Mux(io.ognt().hasData(), s_data_write, s_data_read)) + } + } + } + } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { + state := Mux(!do_allocate, s_inner_grant, + Mux(ognt_had_data, s_data_write, s_data_read)) + } + } + is(s_data_read) { + io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(read_data_done) { state := s_data_resp } + } + is(s_data_resp) { + when(io.data.resp.valid) { + mergeDataInternal(data_buffer, io.data.resp.bits) + } + when(resp_data_done) { + state := Mux(xact.hasData(), s_data_write, s_inner_grant) + } + } + is(s_data_write) { + io.data.write.valid := Bool(true) + when(write_data_done) { state := s_inner_grant } + } + is(s_inner_grant) { + io.inner.grant.valid := Bool(true) + when(ignt_data_done) { + val meta = pending_coh_on_ignt != xact_meta.coh + when(meta) { pending_coh := pending_coh_on_ignt } + state := Mux(meta, s_meta_write, + Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + } + } + is(s_meta_write) { + io.meta.write.valid := Bool(true) + when(io.meta.write.ready) { + state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + } + } + is(s_inner_finish) { + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + class L2WritebackReq extends L2HellaCacheBundle with HasL2Id { - val addr_block = UInt(width = tlBlockAddrBits) - val coh = new ManagerMetadata + val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size + val coh = new HierarchicalMetadata val way_en = Bits(width = nWays) } @@ -463,58 +901,63 @@ class L2WritebackIO extends L2HellaCacheBundle { val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val wb = new L2WritebackIO().flip - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - } - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val mgnt = io.outer.grant.bits +class L2WritebackUnitIO extends HierarchicalXactTrackerIO { + val wb = new L2WritebackIO().flip + val data = new L2DataRWIO +} - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_write :: Nil = Enum(UInt(), 5) +class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { + val io = new L2WritebackUnitIO + + val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_coh = Reg{ new ManagerMetadata } + val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) + val xact_coh = Reg{ new HierarchicalMetadata } val xact_way_en = Reg{ Bits(width = nWays) } - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } + val pending_finish = Reg{ io.outer.finish.bits.clone } - val crel_had_data = Reg(init = Bool(false)) + val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = co.dir.flush) - val curr_p_id = co.dir.next(pending_probes) + val pending_probes = Reg(init = Bits(0, width = nClients)) + val curr_p_id = PriorityEncoder(pending_probes) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val (read_data_cnt, read_data_done) = Counter(io.data.read.fire(), tlDataBeats) - val resp_data_done = connectIncomingDataBeatCounter(io.data.resp) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) + val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) + val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - io.has_release_match := !crel.payload.isVoluntary() && - crel.payload.conflicts(xact_addr_block) && + val pending_icoh_on_irel = xact_coh.inner.onRelease( + incoming = io.irel(), + src = io.inner.release.bits.header.src) + + io.has_acquire_conflict := Bool(false) + io.has_acquire_match := Bool(false) + io.has_release_match := !io.irel().isVoluntary() && + io.irel().conflicts(xact_addr_block) && (state === s_probe) - val next_coh_on_rel = co.managerMetadataOnRelease(crel.payload, xact_coh, crel.header.src) - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) // default + io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( client_xact_id = UInt(trackerId), addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - io.outer.grant.ready := Bool(false) // Never gets mgnts + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) + io.outer.release.bits.header.src := UInt(bankId) + io.outer.grant.ready := Bool(false) // default + io.outer.finish.valid := Bool(false) // default + io.outer.finish.bits := pending_finish + val pending_finish_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := Probe.onVoluntaryWriteback(xact_coh, xact_addr_block) + io.inner.probe.bits.payload := + xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) @@ -540,528 +983,75 @@ class L2WritebackUnit(trackerId: Int, bankId: Int, innerId: String, outerId: Str xact_coh := io.wb.req.bits.coh xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id + irel_had_data := Bool(false) val coh = io.wb.req.bits.coh - val needs_probes = co.requiresProbesOnVoluntaryWriteback(coh) + val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = co.dir.full(coh.sharers) & ~io.tile_incoherent - pending_probes := mask_incoherent - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) + val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits + pending_probes := mask_incoherent(nCoherentClients-1,0) + release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) } state := Mux(needs_probes, s_probe, s_data_read) } } is(s_probe) { // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) + io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - xact_coh := next_coh_on_rel + xact_coh.inner := pending_icoh_on_irel // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - xact_data(crel.payload.addr_beat) := crel.payload.data + when(io.irel().hasData()) { + irel_had_data := Bool(true) + data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasData() || crel_data_done) { + when(!io.irel().hasData() || irel_data_done) { release_count := release_count - UInt(1) } } when(release_count === UInt(0)) { - state := Mux(crel_had_data, s_outer_write, s_data_read) + state := Mux(irel_had_data, s_outer_release, s_data_read) } } is(s_data_read) { io.data.read.valid := Bool(true) - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { xact_data(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_write } + when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + when(resp_data_done) { state := s_outer_release } } - is(s_outer_write) { - io.outer.acquire.valid := Bool(true) - when(macq_data_done) { - io.wb.resp.valid := Bool(true) - state := s_idle + is(s_outer_release) { + io.outer.release.valid := Bool(true) + when(orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) } } - } -} - -abstract class L2XactTracker(innerId: String, outerId: String) extends L2HellaCacheModule { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - val data = new L2DataRWIO - val meta = new L2MetaRWIO - val wb = new L2WritebackIO - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cack = io.inner.finish.bits - val mgnt = io.outer.grant.bits -} - -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.release.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_crel_data = Reg(init=Bool(false)) - val crel_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectOutgoingDataBeatCounter(io.data.write) - - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId), xact_meta.coh) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := co.managerMetadataOnRelease(xact, - xact_meta.coh, - xact_src) - io.wb.req.valid := Bool(false) - - when(collect_crel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(crel.payload.addr_beat) := crel.payload.data - crel_data_valid(crel.payload.addr_beat) := Bool(true) - } - when(crel_data_done) { collect_crel_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_addr_beat := crel.payload.addr_beat - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_crel_data := crel.payload.hasMultibeatData() - crel_data_valid := Bits(1) - state := s_meta_read + is(s_outer_grant) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { + when(io.ognt().requiresAck()) { + pending_finish.payload := pending_finish_on_ognt + pending_finish.header.dst := io.outer.grant.bits.header.src + state := s_outer_finish + }.otherwise { + state := s_wb_resp + } } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_crel_data || crel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } -} - - -class L2AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends L2XactTracker(innerId, outerId) { - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_read :: s_outer_resp :: s_data_read :: s_data_resp :: s_data_write :: s_meta_write :: s_grant :: s_ack :: Nil = Enum(UInt(), 14) - val state = Reg(init=s_idle) - - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats+1) { // Extra entry holds AMO result - Reg(io.inner.acquire.bits.payload.data.clone) - } - val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) - - val collect_cacq_data = Reg(init=Bool(false)) - val cacq_data_valid = Reg(init=Bits(0, width = tlDataBeats)) - val crel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = UInt(0, width = nCoherentClients)) - val curr_p_id = co.dir.next(pending_probes) - val full_sharers = co.dir.full(io.meta.resp.bits.meta.coh.sharers) - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) - val mask_incoherent = mask_self & ~io.tile_incoherent - - val cacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val crel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (macq_data_cnt, macq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val mgnt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val cgnt_data_cnt = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val cgnt_data_max = Reg(init = UInt(0, width = tlBeatAddrBits+1)) - val read_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val read_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val write_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_cnt = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - val resp_data_max = Reg(init = UInt(0, width = log2Up(refillCycles)+1)) - - val needs_writeback = !xact_tag_match && co.isValid(xact_meta.coh) // TODO: dirty bit - val is_hit = xact_tag_match && co.isHit(xact, xact_meta.coh) - val needs_probes = co.requiresProbes(xact, xact_meta.coh) - //val do_allocate = !xact_builtin_type || xact.allocate() - - val amoalu = Module(new AMOALU) - amoalu.io.addr := xact.addr() - amoalu.io.cmd := xact.op_code() - amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := xact.data(0) // default - - def mergeData[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.uncachedAtomic), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.uncachedWriteBlock) || xact.is(Acquire.uncachedWrite), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = tlDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.uncachedAtomic), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.uncachedAtomic)) { buffer(tlDataBeats) := old_data } // For AMO result - } - - //TODO: Allow hit under miss for stores - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - cacq.payload.addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(cacq.payload) || in_same_set) && - (state != s_idle) && - !collect_cacq_data - io.has_acquire_match := xact.conflicts(cacq.payload) && - collect_cacq_data - io.has_release_match := !crel.payload.isVoluntary() && - (xact.addr_block === crel.payload.addr_block) && - (state === s_probe) - - val next_coh_on_rel = co.managerMetadataOnRelease( - incoming = crel.payload, - meta = xact_meta.coh, - src = crel.header.src) - val next_coh_on_gnt = co.managerMetadataOnGrant( - outgoing = cgnt.payload, - meta = xact_meta.coh, - dst = cgnt.header.dst) - - val outer_write = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = macq_data_cnt, - data = xact_data(macq_data_cnt)), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := Bool(true) //grant.data -> xact.data - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe(xact_meta.coh) - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant( - manager_xact_id = UInt(trackerId), - meta = xact_meta.coh, - addr_beat = cgnt_data_cnt, - data = Mux(xact.is(Acquire.uncachedAtomic), - xact_data(tlDataBeats), - xact_data(cgnt_data_cnt))) - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := xact_data(write_data_cnt) - io.meta.read.valid := Bool(false) - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh := next_coh_on_gnt - - io.wb.req.valid := Bool(false) - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact_addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh - io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) - - when(collect_cacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(cacq.payload.addr_beat) := cacq.payload.data - cacq_data_valid(cacq.payload.addr_beat) := Bool(true) - } - when(cacq_data_done) { collect_cacq_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - when( io.inner.acquire.valid ) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_cacq_data := cacq.payload.hasMultibeatData() - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - val coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _needs_writeback = !_tag_match && co.isValid(coh) //TODO: dirty bit - val _needs_probes = _tag_match && co.requiresProbes(xact, coh) - val _is_hit = _tag_match && co.isHit(xact, coh) - val full_block = !xact.builtin_type || - xact.hasMultibeatData() || - cgnt.payload.hasMultibeatData() - read_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - read_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - write_data_cnt := Mux(full_block || !_is_hit, UInt(0), xact_addr_beat) - write_data_max := Mux(full_block || !_is_hit, UInt(refillCycles-1), xact_addr_beat) - resp_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - resp_data_max := Mux(full_block, UInt(refillCycles-1), xact_addr_beat) - cgnt_data_cnt := Mux(full_block, UInt(0), xact_addr_beat) - cgnt_data_max := Mux(full_block, UInt(tlDataBeats-1), xact_addr_beat) - when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := co.dir.count(mask_incoherent) - crel_had_data := Bool(false) - } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_read)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_read)) // Evict ifneedbe - } - } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } + is(s_outer_finish) { + io.outer.finish.valid := Bool(true) + when(io.outer.finish.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_read } - } - is(s_probe) { - // Send probes - io.inner.probe.valid := !co.dir.none(pending_probes) - when(io.inner.probe.ready) { - pending_probes := co.dir.pop(pending_probes, curr_p_id) - } - // Handle releases, which may have data being written back - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_meta.coh := next_coh_on_rel - // Handle released dirty data - when(crel.payload.hasData()) { - crel_had_data := Bool(true) - mergeData(xact_data, crel.payload) - } - // We don't decrement release_count until we've received all the data beats. - when(!crel.payload.hasMultibeatData() || crel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(crel_had_data, s_data_write, s_data_read), s_outer_read) - } - } - is(s_outer_read) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { - state := s_outer_resp - } - } - is(s_outer_resp) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - //TODO make sure cacq data is actually present before merging - mergeData(xact_data, mgnt.payload) - when(mgnt_data_done) { - state := Mux(mgnt.payload.hasData(), s_data_write, s_data_read) - } - } - } - is(s_data_read) { - io.data.read.valid := !collect_cacq_data || cacq_data_valid(read_data_cnt) - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(io.data.read.ready) { - read_data_cnt := read_data_cnt + UInt(1) - when(read_data_cnt === read_data_max) { state := s_data_resp } - } - } - is(s_data_resp) { - when(io.data.resp.valid) { - mergeData(xact_data, io.data.resp.bits) - resp_data_cnt := resp_data_cnt + UInt(1) - } - when(resp_data_cnt === resp_data_max) { - state := Mux(xact.hasData(), s_data_write, s_meta_write) - } - } - is(s_data_write) { - io.data.write.valid := Bool(true) - when(io.data.write.ready) { - write_data_cnt := write_data_cnt + UInt(1) - when(write_data_cnt === write_data_max) { - state := s_meta_write - } - } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { state := s_grant } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - cgnt_data_cnt := cgnt_data_cnt + UInt(1) - when(cgnt_data_cnt === cgnt_data_max) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - } - is(s_ack) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + io.wb.resp.valid := Bool(true) + state := s_idle } } } diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 82f72dea..2524f3d1 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -3,202 +3,148 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers -abstract class CoherenceMetadata extends Bundle with CoherenceAgentParameters +/** The entire CoherencePolicy API consists of the following three traits: + * HasCustomTileLinkMessageTypes, used to define custom messages + * HasClientSideCoherencePolicy, for client coherence agents + * HasManagerSideCoherencePolicy, for manager coherence agents + */ +abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends + HasCustomTileLinkMessageTypes with + HasClientSideCoherencePolicy with + HasManagerSideCoherencePolicy -class ClientMetadata extends CoherenceMetadata { - val state = UInt(width = co.clientStateWidth) - def ===(right: ClientMetadata): Bool = this.state === right.state - override def clone = new ClientMetadata().asInstanceOf[this.type] -} -object ClientMetadata { - def apply(state: UInt) = { - val meta = new ClientMetadata - meta.state := state - meta - } -} - -class ManagerMetadata extends CoherenceMetadata { - val state = UInt(width = co.masterStateWidth) - val sharers = UInt(width = co.dir.width) - override def clone = new ManagerMetadata().asInstanceOf[this.type] -} -object ManagerMetadata { - def apply(state: UInt, sharers: UInt) = { - val meta = new ManagerMetadata - meta.state := state - meta.sharers := sharers - meta - } - def apply(state: UInt): ManagerMetadata = apply(state, new ManagerMetadata().co.dir.flush) -} - -// This class encapsulates transformations on different directory information -// storage formats -abstract class DirectoryRepresentation(val width: Int) { - def pop(prev: UInt, id: UInt): UInt - def push(prev: UInt, id: UInt): UInt - def flush: UInt - def none(s: UInt): Bool - def one(s: UInt): Bool - def count(s: UInt): UInt - def next(s: UInt): UInt - def full(s: UInt): UInt -} - -class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { - def pop(prev: UInt, id: UInt) = UInt(0) - def push(prev: UInt, id: UInt) = UInt(0) - def flush = UInt(0) - def none(s: UInt) = Bool(false) - def one(s: UInt) = Bool(false) - def count(s: UInt) = UInt(nClients) - def next(s: UInt) = UInt(0) - def full(s: UInt) = SInt(-1, width = nClients).toUInt -} - -class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { - def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) - def push(prev: UInt, id: UInt) = prev | UIntToOH(id) - def flush = UInt(0, width = width) - def none(s: UInt) = s === UInt(0) - def one(s: UInt) = PopCount(s) === UInt(1) - def count(s: UInt) = PopCount(s) - def next(s: UInt) = PriorityEncoder(s) - def full(s: UInt) = s -} - -// Coherence policy inferface for clients and managers -abstract class CoherencePolicy(val dir: DirectoryRepresentation) { - def nClientStates: Int - def nManagerStates: Int - def nAcquireTypes: Int - def nProbeTypes: Int - def nReleaseTypes: Int - def nGrantTypes: Int - def clientStateWidth = log2Up(nClientStates) - def masterStateWidth = log2Up(nManagerStates) +/** This API defines the custom, coherence-policy-defined message types, + * as opposed to the built-in ones found in tilelink.scala. + * Policies must enumerate the custom messages to be sent over each + * channel, as well as which of them have associated data. + */ +trait HasCustomTileLinkMessageTypes { + val nAcquireTypes: Int def acquireTypeWidth = log2Up(nAcquireTypes) + val nProbeTypes: Int def probeTypeWidth = log2Up(nProbeTypes) + val nReleaseTypes: Int def releaseTypeWidth = log2Up(nReleaseTypes) + val nGrantTypes: Int def grantTypeWidth = log2Up(nGrantTypes) - val clientStatesWithReadPermission: Vec[UInt] - val clientStatesWithWritePermission: Vec[UInt] - val clientStatesWithDirtyData: Vec[UInt] val acquireTypesWithData = Nil // Only built-in Acquire types have data for now val releaseTypesWithData: Vec[UInt] val grantTypesWithData: Vec[UInt] +} +/** This API contains all functions required for client coherence agents. + * Policies must enumerate the number of client states and define their + * permissions with respect to memory operations. Policies must fill in functions + * to control which messages are sent and how metadata is updated in response + * to coherence events. These funtions are generally called from within the + * ClientMetadata class in metadata.scala + */ +trait HasClientSideCoherencePolicy { + // Client coherence states and their permissions + val nClientStates: Int + def clientStateWidth = log2Ceil(nClientStates) + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + + // Transaction initiation logic def isValid(meta: ClientMetadata): Bool - def isValid(meta: ManagerMetadata): Bool - def isHit(cmd: UInt, meta: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), clientStatesWithWritePermission.contains(meta.state), clientStatesWithReadPermission.contains(meta.state)) } - //TODO: Use outer protocol's clientState instead, remove this function: - def isHit(incoming: Acquire, meta: ManagerMetadata) = isValid(meta) - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool + //TODO: Assumes all states with write permissions also have read permissions + def requiresAcquireOnSecondaryMiss( + first_cmd: UInt, + second_cmd: UInt, + meta: ClientMetadata): Bool = { + isWriteIntent(second_cmd) && + !isWriteIntent(first_cmd) || + (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: - def needsTransactionOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = clientStatesWithDirtyData.contains(meta.state) - def needsWriteback(meta: ClientMetadata): Bool = - needsTransactionOnCacheControl(M_FLUSH, meta) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + // Determine which custom message type to use + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + + // Mutate ClientMetadata based on messages or cmds + def clientMetadataOnReset: ClientMetadata def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnFlush: ClientMetadata - - def managerMetadataOnRelease(incoming: Release, meta: ManagerMetadata, src: UInt): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, meta: ManagerMetadata, dst: UInt): ManagerMetadata - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnFlush: ManagerMetadata - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt - def getReleaseType(p: Probe, meta: ClientMetadata): UInt - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt - - def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata): Bool - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool - def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool - def requiresProbesOnVoluntaryWriteback(meta: ManagerMetadata): Bool = requiresProbes(M_FLUSH, meta) + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata } -class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 2 - def nManagerStates = 2 - def nAcquireTypes = 1 - def nProbeTypes = 2 - def nReleaseTypes = 4 - def nGrantTypes = 1 +/** This API contains all functions required for manager coherence agents. + * Policies must enumerate the number of manager states. Policies must fill + * in functions to control which Probe and Grant messages are sent and how + * metadata should be updated in response to coherence events. These funtions + * are generally called from within the ManagerMetadata class in metadata.scala + */ +trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { + val nManagerStates: Int + def masterStateWidth = log2Ceil(nManagerStates) - val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Transaction probing logic + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + + // Determine which custom message type to use in response + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + + // Mutate ManagerMetadata based on messages or cmds + def managerMetadataOnReset: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached + meta.sharers, + dir.push(meta.sharers, dst))) + //state = meta.state) TODO: Fix 0-width wires in Chisel +} + +/** The following concrete implementations of CoherencePolicy each provide the + * functionality of one particular protocol. + */ + +/** A simple protocol with only two Client states. + * Data is always assumed to be dirty. + * Only a single client may ever have a copy of a block at a time. + */ +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 1 + val nProbeTypes = 2 + val nReleaseTypes = 4 + val nGrantTypes = 1 val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientValid) - val clientStatesWithWritePermission = Vec(clientValid) - val clientStatesWithDirtyData = Vec(clientValid) val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 2 + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata): Bool = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (outstanding.a_type != acquireExclusive) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata(Mux(incoming.builtin_type, clientInvalid, clientValid)) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata(Mux(incoming.p_type === probeInvalidate, - clientInvalid, meta.state)) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers, src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = acquireExclusive + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) @@ -213,112 +159,85 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A simple protocol with only three Client states. + * Data is marked as dirty when written. + * Only a single client may ever have a copy of a block at a time. + */ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 1 - - val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 1 val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantExclusive) + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def isValid (meta: ClientMetadata) = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusiveDirty)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - Mux(outstanding.a_type === acquireExclusiveDirty, clientExclusiveDirty, - clientExclusiveClean))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusiveDirty, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -333,75 +252,113 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), grantExclusive) + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - probeInvalidate) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), Bool(true)))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } +/** A protocol with only three Client states. + * Data is always assumed to be dirty. + * Multiple clients may share read permissions on a block at the same time. + */ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 3 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) val grantTypesWithData = Vec(grantShared, grantExclusive) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid(meta: ManagerMetadata) = meta.state != managerInvalid + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -409,9 +366,9 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, @@ -424,30 +381,82 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) } +} - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) +/** A protocol with four Client states. + * Data is marked as dirty when written. + * Multiple clients may share read permissions on a block at the same time. + */ +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + // Client states and functions + val nClientStates = 4 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -462,80 +471,11 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) - - def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = - MuxLookup(cmd, probeCopy, Array( - M_FLUSH -> probeInvalidate, - M_PRODUCE -> probeDowngrade)) - - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) - - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) - - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) - - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) -} - -class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 4 - def nManagerStates = 2 - def nAcquireTypes = 2 - def nProbeTypes = 3 - def nReleaseTypes = 6 - def nGrantTypes = 3 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) - - val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) - val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) - val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) - val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) - - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && (outstanding.a_type != acquireExclusive)) + def clientMetadataOnReset = ClientMetadata(clientInvalid) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( @@ -544,13 +484,12 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { clientShared, meta.state), M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, + Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, - grantExclusive -> Mux(outstanding.a_type === acquireExclusive, - clientExclusiveDirty, clientExclusiveClean), + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty)))) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = @@ -560,188 +499,80 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> clientShared, probeCopy -> meta.state))) - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next - )) - } + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) - - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), acquireExclusive, outstanding.a_type) - - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) - MuxLookup(cmd, releaseCopyAck, Array( - M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), - M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), - M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) - } - - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = - MuxLookup(incoming.p_type, releaseInvalidateAck, Array( - probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) - - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) - - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive)) - - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate))) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), Bool(true)) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } } class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { - def nClientStates = 7 - def nManagerStates = 2 - def nAcquireTypes = 3 - def nProbeTypes = 4 - def nReleaseTypes = 10 - def nGrantTypes = 4 - - val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - //val managerInvalid :: masterShared :: masterExclusive :: Nil = Enum(UInt(), nManagerStates) - val managerInvalid :: managerValid :: Nil = Enum(UInt(), nManagerStates) + // Message types + val nAcquireTypes = 3 + val nProbeTypes = 4 + val nReleaseTypes = 10 + val nGrantTypes = 4 val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + // Client states and functions + val nClientStates = 7 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid - def isValid (meta: ManagerMetadata) = meta.state != managerInvalid - def needsTransactionOnSecondaryMiss(cmd: UInt, outstanding: Acquire): Bool = - (isRead(cmd) && outstanding.builtin_type) || - (isWriteIntent(cmd) && !Vec(acquireExclusive, acquireInvalidateOthers).contains(outstanding.a_type)) - - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( - clientExclusiveClean -> clientExclusiveDirty, - clientMigratoryClean -> clientMigratoryDirty)), - meta.state)) - - def clientMetadataOnFlush = ClientMetadata(clientInvalid) - - def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(cmd, meta.state, Array( - M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), - clientShared, meta.state), - M_CLEAN -> MuxLookup(meta.state, meta.state, Array( - clientExclusiveDirty -> clientExclusiveClean, - clientMigratoryDirty -> clientMigratoryClean))))) - - def clientMetadataOnGrant(incoming: Grant, outstanding: Acquire) = - ClientMetadata( - Mux(incoming.builtin_type, clientInvalid, - MuxLookup(incoming.g_type, clientInvalid, Array( - grantShared -> clientShared, - grantExclusive -> MuxLookup(outstanding.a_type, clientExclusiveDirty, Array( - acquireExclusive -> clientExclusiveDirty, - acquireShared -> clientExclusiveClean)), - grantExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> MuxLookup(outstanding.a_type, clientMigratoryDirty, Array( - acquireInvalidateOthers -> clientMigratoryDirty, - acquireExclusive -> clientMigratoryDirty, - acquireShared -> clientMigratoryClean)))))) - - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = - ClientMetadata( - MuxLookup(incoming.p_type, meta.state, Array( - probeInvalidate -> clientInvalid, - probeInvalidateOthers -> clientInvalid, - probeCopy -> meta.state, - probeDowngrade -> MuxLookup(meta.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid))))) - - def managerMetadataOnRelease(r: Release, meta: ManagerMetadata, src: UInt) = { - val next = ManagerMetadata(managerValid, dir.pop(meta.sharers,src)) - MuxBundle(meta, Array( - r.is(releaseInvalidateData) -> next, - r.is(releaseInvalidateAck) -> next, - r.is(releaseInvalidateDataMigratory) -> next, - r.is(releaseInvalidateAckMigratory) -> next)) - } - - def managerMetadataOnGrant(g: Grant, meta: ManagerMetadata, dst: UInt) = - Mux(g.builtin_type, - ManagerMetadata(managerValid, meta.sharers), - ManagerMetadata(managerValid, dir.push(meta.sharers, dst))) - - def managerMetadataOnFlush = ManagerMetadata(managerInvalid) - - def managerMetadataOnCacheControl(cmd: UInt, meta: ManagerMetadata) = - ManagerMetadata(Mux(cmd === M_FLUSH, managerInvalid, meta.state)) - - def getAcquireTypeOnPrimaryMiss(cmd: UInt, meta: ClientMetadata): UInt = + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), acquireShared) - def getAcquireTypeOnSecondaryMiss(cmd: UInt, meta: ClientMetadata, outstanding: Acquire): UInt = - Mux(isWriteIntent(cmd), - Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), - outstanding.a_type) - def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) MuxLookup(cmd, releaseCopyAck, Array( @@ -769,43 +600,89 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(dirty, with_data, without_data) } - def isCoherenceConflict(addr1: UInt, addr2: UInt): Bool = (addr1 === addr2) + def clientMetadataOnReset = ClientMetadata(clientInvalid) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, Grant.getGrantTypeForUncached(a), - MuxLookup(a.a_type, grantShared, Array( - acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - acquireExclusive -> grantExclusive, - acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = - Mux(a.builtin_type, - MuxLookup(a.a_type, probeCopy, Array( - Acquire.uncachedReadBlock -> probeCopy, - Acquire.uncachedWriteBlock -> probeInvalidate, - Acquire.uncachedRead -> probeCopy, - Acquire.uncachedWrite -> probeInvalidate, - Acquire.uncachedAtomic -> probeInvalidate)), - MuxLookup(a.a_type, probeCopy, Array( - acquireShared -> probeDowngrade, - acquireExclusive -> probeInvalidate, - acquireInvalidateOthers -> probeInvalidateOthers))) + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def requiresOuterRead(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterRead(acq.a_type), acq.a_type != acquireInvalidateOthers) + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) - def requiresOuterWrite(acq: Acquire, meta: ManagerMetadata) = - Mux(acq.builtin_type, Acquire.requiresOuterWrite(acq.a_type), Bool(false)) + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? - def requiresProbes(a: Acquire, meta: ManagerMetadata) = - Mux(dir.none(meta.sharers), Bool(false), - Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.builtin_type, a.hasData(), a.a_type != acquireShared))) + def managerMetadataOnReset = ManagerMetadata() - def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped, + incoming.is(releaseInvalidateDataMigratory) -> popped, + incoming.is(releaseInvalidateAckMigratory) -> popped)) + } } diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/directory.scala new file mode 100644 index 00000000..db555ad3 --- /dev/null +++ b/uncore/src/main/scala/directory.scala @@ -0,0 +1,43 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// This class encapsulates transformations on different directory information +// storage formats +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt + def full(s: UInt): UInt +} + +abstract trait HasDirectoryRepresentation { + val dir: DirectoryRepresentation +} + +class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(nClients) + def next(s: UInt) = UInt(0) + def full(s: UInt) = SInt(-1, width = nClients).toUInt +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s +} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5e5053be..4d5e1135 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -14,7 +14,6 @@ case object HTIFNCores extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) val dataBeats = params(TLDataBeats) - val co = params(TLCoherence) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) val offsetBits = params(HTIFOffsetBits) @@ -197,12 +196,12 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, - UncachedWriteBlock( + PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), - UncachedReadBlock(addr_block = init_addr)) + GetBlock(addr_block = init_addr)) io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid @@ -258,7 +257,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLAddrBits)) >> 20) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e58dd08d..f10b3257 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -202,13 +202,15 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } +//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary + + //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemIO } - val co = params(TLCoherence) val mifTagBits = params(MIFTagBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) @@ -216,121 +218,241 @@ class MemIOUncachedTileLinkIOConverter(qDepth: Int) extends Module { val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - //require(params(TLClientXactIdBits) <= params(MIFTagBits)) + require(params(TLClientXactIdBits) <= params(MIFTagBits)) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + io.tl.grant <> gnt_arb.io.out + + val acq_has_data = io.tl.acquire.bits.payload.hasData() + val rel_has_data = io.tl.release.bits.payload.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - - io.uncached.acquire.ready := Bool(false) - io.uncached.grant.valid := Bool(false) - io.mem.resp.ready := Bool(false) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - - //TODO: Assert that only WriteUncachedBlock and ReadUncachedBlock are - //acceptable Acquire types - val acq_has_data = io.uncached.acquire.bits.payload.hasData() - val (tl_cnt_out, tl_wrap_out) = Counter(io.uncached.acquire.fire() && acq_has_data, tlDataBeats) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tl_done_out = Reg(init=Bool(false)) - val mif_done_out = Reg(init=Bool(false)) val tag_out = Reg(Bits()) val addr_out = Reg(Bits()) val has_data = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.uncached.acquire.bits.payload.data.clone) } - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + val grant_for_rel = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = tag_out, + manager_xact_id = UInt(0)) + val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_out(0), + a_type = tag_out >> UInt(1), + client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) - when(!active_out){ - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := io.uncached.acquire.bits.payload.client_xact_id - addr_out := io.uncached.acquire.bits.payload.addr_block - has_data := acq_has_data - tl_done_out := tl_wrap_out - mif_done_out := Bool(false) - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := io.tl.release.bits.payload.client_xact_id + addr_out := io.tl.release.bits.payload.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + addr_out := io.tl.acquire.bits.payload.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + } } - } - when(active_out) { - when(!cmd_sent_out) { - mem_cmd_q.io.enq.valid := Bool(true) - } - when(has_data) { - when(!tl_done_out) { - io.uncached.acquire.ready := Bool(true) - when(io.uncached.acquire.valid) { - tl_buf_out(tl_cnt_out) := io.uncached.acquire.bits.payload.data + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out } } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) } } - when(mem_cmd_q.io.enq.fire()) { - cmd_sent_out := Bool(true) + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.payload.data, + io.tl.acquire.bits.payload.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tag_out := io.mem.req_cmd.bits.tag + addr_out := io.mem.req_data.bits.data + has_data := io.mem.req_cmd.bits.rw + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := Bool(true) + io.mem.req_cmd.bits.rw := rel_has_data + io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.release.bits.payload.data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + io.mem.req_cmd.bits.rw := acq_has_data + io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.a_type, + io.tl.acquire.bits.payload.is_builtin_type) + io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block + io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + } + } } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || mif_done_out)) { - active_out := Bool(false) + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } } } - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - // Aggregate incoming MemIO responses into TL Grants - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val (tl_cnt_in, tl_wrap_in) = Counter(io.uncached.grant.fire(), tlDataBeats) val active_in = Reg(init=Bool(false)) - val mif_done_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) - val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.uncached.acquire.bits.payload.data.clone } - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - - when(active_in) { - io.uncached.grant.valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = tag_in(0), + a_type = tag_in >> UInt(1), + client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { io.mem.resp.ready := Bool(true) when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data } } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits.payload := + ManagerMetadata.onReset.makeGrant( + acq = Acquire( + is_builtin_type = io.mem.resp.bits.tag(0), + a_type = io.mem.resp.bits.tag >> UInt(1), + client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), + addr_block = UInt(0)), //DNC + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } - - io.uncached.grant.bits.payload := Grant(builtin_type = Bool(true), - g_type = Grant.uncachedReadBlock, - client_xact_id = tag_in, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module @@ -428,15 +550,15 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOUncachedTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { val io = new Bundle { - val uncached = new UncachedTileLinkIO().flip + val tl = new TileLinkIO().flip val mem = new MemPipeIO } - val a = Module(new MemIOUncachedTileLinkIOConverter(2)) + val a = Module(new MemIOTileLinkIOConverter(1)) val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) - a.io.uncached <> io.uncached + a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) a.io.mem.resp <> b.io.cpu.resp diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala new file mode 100644 index 00000000..8f5d7e90 --- /dev/null +++ b/uncore/src/main/scala/metadata.scala @@ -0,0 +1,188 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// Classes to represent coherence information in clients and managers +abstract class CoherenceMetadata extends Bundle { + val co = params(TLCoherencePolicy) + val id = params(TLId) +} + +/* The ClientMetadata stores the client-side coherence information, + such as permissions on the data and whether the data is dirty. + Its API can be used to make TileLink messages in response to + memory operations or TileLink Probes. +*/ +class ClientMetadata extends CoherenceMetadata { + val state = UInt(width = co.clientStateWidth) + + def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + + def isValid(dummy: Int = 0): Bool = co.isValid(this) + def isHit(cmd: UInt): Bool = co.isHit(cmd, this) + def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) + def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) + def requiresReleaseOnCacheControl(cmd: UInt): Bool = + co.requiresReleaseOnCacheControl(cmd: UInt, this) + def requiresVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresReleaseOnCacheControl(M_FLUSH, this) + + def makeAcquire( + client_xact_id: UInt, + addr_block: UInt, + op_code: UInt): Acquire = { + Bundle(Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true))), + { case TLId => id }) + } + + def makeVoluntaryWriteback( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def makeRelease( + prb: Probe, + client_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(false), + r_type = co.getReleaseType(prb, this), + client_xact_id = client_xact_id, + addr_block = prb.addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onGrant(incoming: Grant, pending: UInt): ClientMetadata = + Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + def onProbe(incoming: Probe): ClientMetadata = + Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + def onHit(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) + def onCacheControl(cmd: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) +} + +object ClientMetadata { + def apply(state: UInt) = { + val meta = new ClientMetadata + meta.state := state + meta + } + def onReset = new ClientMetadata().co.clientMetadataOnReset +} + +/* The ManagerMetadata stores manager-side information about the status + of a cache block, including whether it has any known sharers. Its + API can be used to create Probe and Grant TileLink messages. +*/ +class ManagerMetadata extends CoherenceMetadata { + // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + val sharers = UInt(width = co.dir.width) + + def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel + this.sharers === rhs.sharers + def full(dummy: Int = 0) = co.dir.full(this.sharers) + + def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresProbes(M_FLUSH, this) + + def makeProbe(cmd: UInt, addr_block: UInt): Probe = + Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + + def makeProbe(acq: Acquire): Probe = + Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + + def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = + makeProbe(M_FLUSH, addr_block) + + def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + Bundle(Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = rel.client_xact_id, + manager_xact_id = manager_xact_id), { case TLId => id }) + } + + def makeGrant( + acq: Acquire, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Grant = { + Bundle(Grant( + is_builtin_type = acq.isBuiltInType(), + g_type = Mux(acq.isBuiltInType(), + acq.getBuiltInGrantType(), + co.getGrantType(acq, this)), + client_xact_id = acq.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + def onRelease(incoming: Release, src: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + + def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) +} + +object ManagerMetadata { + def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + val meta = new ManagerMetadata + //meta.state := state TODO: Fix 0-width wires in Chisel + meta.sharers := sharers + meta + } + def apply() = { + val meta = new ManagerMetadata + //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel + meta.sharers := meta.co.dir.flush + meta + } + def onReset = new ManagerMetadata().co.managerMetadataOnReset +} + +/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + that is a Manager with respect to some inner caches and a Client with + respect to some outer cache. +*/ +class HierarchicalMetadata extends CoherenceMetadata { + val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) + val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) + def ===(rhs: HierarchicalMetadata): Bool = + this.inner === rhs.inner && this.outer === rhs.outer + def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) +} + +object HierarchicalMetadata { + def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + val m = new HierarchicalMetadata + m.inner := inner + m.outer := outer + m + } + def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) +} + +case object InnerTLId extends Field[String] +case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index afbf4dc0..c8996ddf 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -8,30 +8,35 @@ import scala.math.max // external requirements or design space exploration // case object TLId extends Field[String] // Unique name per network -case object TLCoherence extends Field[CoherencePolicy] -case object TLAddrBits extends Field[Int] +case object TLCoherencePolicy extends Field[CoherencePolicy] +case object TLBlockAddrBits extends Field[Int] case object TLManagerXactIdBits extends Field[Int] case object TLClientXactIdBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] +case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLAddrBits) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlClientXactIdBits = params(TLClientXactIdBits) val tlManagerXactIdBits = params(TLManagerXactIdBits) val tlDataBits = params(TLDataBits) val tlDataBeats = params(TLDataBeats) + val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) - val tlAtomicOpcodeBits = M_SZ - val tlUncachedOperandSizeBits = MT_SZ - val tlSubblockUnionBits = max(tlWriteMaskBits, + val tlMemoryOpcodeBits = M_SZ + val tlMemoryOperandSizeBits = MT_SZ + val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes), + tlCoh.acquireTypeWidth) + val tlAcquireUnionBits = max(tlWriteMaskBits, (tlByteAddrBits + - tlUncachedOperandSizeBits + - tlAtomicOpcodeBits)) + 1 - val co = params(TLCoherence) - val networkPreservesPointToPointOrdering = false //TODO: check physical network type + tlMemoryOperandSizeBits + + tlMemoryOpcodeBits)) + 1 + val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), + tlCoh.grantTypeWidth) + 1 + val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -50,8 +55,8 @@ trait ClientToClientChannel extends TileLinkChannel // Unused for now trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) - def conflicts[T <: HasCacheBlockAddress](that: T) = this.addr_block === that.addr_block - def conflicts[T <: HasCacheBlockAddress](addr: UInt) = this.addr_block === addr + def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block + def conflicts(addr: UInt) = this.addr_block === addr } trait HasTileLinkBeatId extends TLBundle { @@ -66,8 +71,9 @@ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } -abstract trait HasTileLinkData extends HasTileLinkBeatId { +trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) + def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } @@ -79,95 +85,81 @@ class Acquire extends ClientToManagerChannel with HasClientTransactionId with HasTileLinkData { // Actual bundle fields - val builtin_type = Bool() - val a_type = UInt(width = max(log2Up(Acquire.nBuiltinAcquireTypes), co.acquireTypeWidth)) - val subblock = Bits(width = tlSubblockUnionBits) + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) + val union = Bits(width = tlAcquireUnionBits) // Utility funcs for accessing subblock union - val opSizeOff = tlByteAddrBits + 1 - val opCodeOff = tlUncachedOperandSizeBits + opSizeOff - val opMSB = tlAtomicOpcodeBits + opCodeOff - def allocate(dummy: Int = 0) = subblock(0) - def addr_byte(dummy: Int = 0) = subblock(opSizeOff-1, 1) - def op_size(dummy: Int = 0) = subblock(opCodeOff-1, opSizeOff) - def op_code(dummy: Int = 0) = subblock(opMSB-1, opCodeOff) - def write_mask(dummy: Int = 0) = subblock(tlWriteMaskBits, 1) - def addr(dummy: Int = 0) = Cat(addr_block, addr_beat, this.addr_byte(0)) + val opCodeOff = 1 + val opSizeOff = tlMemoryOpcodeBits + opCodeOff + val addrByteOff = tlMemoryOperandSizeBits + opSizeOff + val addrByteMSB = tlByteAddrBits + addrByteOff + def allocate(dummy: Int = 0) = union(0) + def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) + def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs def is(t: UInt) = a_type === t - def hasData(dummy: Int = 0): Bool = builtin_type && Acquire.typesWithData.contains(a_type) + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type - def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && builtin_type && + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + + // Assumes no custom types have data + def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = builtin_type && Acquire.requiresSelfProbe(a_type) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType - def makeProbe(meta: ManagerMetadata = co.managerMetadataOnFlush): Probe = - Probe(co.getProbeType(this, meta), this.addr_block) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { - Grant( - builtin_type = this.builtin_type, - g_type = co.getGrantType(this, meta), - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id, - addr_beat = addr_beat, - data = data - ) + def getBuiltInGrantType(dummy: Int = 0): UInt = { + MuxLookup(this.a_type, Grant.ackType, Array( + Acquire.getType -> Grant.dataBeatType, + Acquire.getBlockType -> Grant.dataBlockType, + Acquire.putType -> Grant.ackType, + Acquire.putBlockType -> Grant.ackType, + Acquire.putAtomicType -> Grant.dataBeatType)) } } object Acquire { - val nBuiltinAcquireTypes = 5 + val nBuiltInTypes = 5 //TODO: Use Enum - def uncachedRead = UInt(0) - def uncachedReadBlock = UInt(1) - def uncachedWrite = UInt(2) - def uncachedWriteBlock = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedWrite, uncachedWriteBlock, uncachedAtomic) - def typesWithMultibeatData = Vec(uncachedWriteBlock) - def requiresOuterRead(a_type: UInt) = a_type != uncachedWriteBlock - def requiresOuterWrite(a_type: UInt) = typesWithData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(a_type: UInt) = a_type === uncachedReadBlock + def getType = UInt("b000") + def getBlockType = UInt("b001") + def putType = UInt("b010") + def putBlockType = UInt("b011") + def putAtomicType = UInt("b100") + def typesWithData = Vec(putType, putBlockType, putAtomicType) + def typesWithMultibeatData = Vec(putBlockType) + def typesOnSubBlocks = Vec(putType, getType, putAtomicType) def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt // Most generic constructor def apply( - builtin_type: Bool, + is_builtin_type: Bool, a_type: Bits, client_xact_id: UInt, addr_block: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0), - subblock: UInt = UInt(0)): Acquire = { + union: UInt = UInt(0)): Acquire = { val acq = new Acquire - acq.builtin_type := builtin_type + acq.is_builtin_type := is_builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id acq.addr_block := addr_block acq.addr_beat := addr_beat acq.data := data - acq.subblock := subblock + acq.union := union acq } - // For cached types - def apply(a_type: Bits, client_xact_id: UInt, addr_block: UInt): Acquire = { - apply( - builtin_type = Bool(false), - a_type = a_type, - client_xact_id = client_xact_id, - addr_block = addr_block) - } // Copy constructor def apply(a: Acquire): Acquire = { val acq = new Acquire @@ -177,58 +169,73 @@ object Acquire { } // Asks for a single TileLink beat of data -object UncachedRead { +object Get { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedRead, + is_builtin_type = Bool(true), + a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - subblock = alloc) + union = Cat(M_XRD, alloc)) } } // Asks for an entire cache block of data -object UncachedReadBlock { +object GetBlock { def apply( client_xact_id: UInt = UInt(0), addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedReadBlock, + is_builtin_type = Bool(true), + a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - subblock = alloc.toUInt) + union = Cat(M_XRD, alloc)) } } -object UncachedWrite { +// Writes up to a single TileLink beat of data, using mask +object Put { def apply( client_xact_id: UInt, addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask, - alloc: Bool = Bool(true)): Acquire = { + write_mask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWrite, + is_builtin_type = Bool(true), + a_type = Acquire.putType, addr_block = addr_block, addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - subblock = Cat(write_mask, alloc)) + union = Cat(write_mask, Bool(true))) } } -// For full block of data -object UncachedWriteBlock { +// Writes an entire cache block of data +object PutBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + write_mask: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + } def apply( client_xact_id: UInt, addr_block: UInt, @@ -236,17 +243,18 @@ object UncachedWriteBlock { data: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedWriteBlock, + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(Acquire.fullWriteMask, alloc)) + union = Cat(Acquire.fullWriteMask, alloc)) } } -object UncachedAtomic { +// Performs an atomic operation in the outer memory +object PutAtomic { def apply( client_xact_id: UInt, addr_block: UInt, @@ -256,48 +264,30 @@ object UncachedAtomic { operand_size: UInt, data: UInt): Acquire = { Acquire( - builtin_type = Bool(true), - a_type = Acquire.uncachedAtomic, + is_builtin_type = Bool(true), + a_type = Acquire.putAtomicType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - subblock = Cat(atomic_opcode, operand_size, addr_byte, Bool(true))) + union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) } } class Probe extends ManagerToClientChannel with HasCacheBlockAddress { - val p_type = UInt(width = co.probeTypeWidth) + val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t - def makeRelease( - client_xact_id: UInt, - meta: ClientMetadata = co.clientMetadataOnFlush, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(false), - r_type = co.getReleaseType(this, meta), - client_xact_id = client_xact_id, - addr_block = this.addr_block, - addr_beat = addr_beat, - data = data) - } } object Probe { - val co = new Probe().co def apply(p_type: UInt, addr_block: UInt) = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } - - def onVoluntaryWriteback(meta: ManagerMetadata, addr_block: UInt): Probe = { - apply(co.getProbeType(M_FLUSH, meta), addr_block) - } } @@ -305,31 +295,19 @@ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - val r_type = UInt(width = co.releaseTypeWidth) + val r_type = UInt(width = tlCoh.releaseTypeWidth) val voluntary = Bool() // Helper funcs def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = co.releaseTypesWithData.contains(r_type) + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) //TODO: Assumes all releases write back full cache blocks: - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && co.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary - def requiresAck(dummy: Int = 0) = !Bool(networkPreservesPointToPointOrdering) - - def makeGrant( - manager_xact_id: UInt, - meta: ManagerMetadata = co.managerMetadataOnFlush): Grant = { - Grant( - g_type = Grant.voluntaryAck, - builtin_type = Bool(true), // Grant.voluntaryAck is built-in type - client_xact_id = this.client_xact_id, - manager_xact_id = manager_xact_id - ) - } + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } object Release { - val co = new Release().co def apply( voluntary: Bool, r_type: UInt, @@ -346,68 +324,52 @@ object Release { rel.voluntary := voluntary rel } - - def makeVoluntaryWriteback( - meta: ClientMetadata, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Release( - voluntary = Bool(true), - r_type = co.getReleaseType(M_FLUSH, meta), - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = addr_beat, - data = data) - } } class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { - val builtin_type = Bool() - val g_type = UInt(width = max(log2Up(Grant.nBuiltinGrantTypes), co.grantTypeWidth)) + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) // Helper funcs - def is(t: UInt) = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(builtin_type, + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), - co.grantTypesWithData.contains(g_type)) + tlCoh.grantTypesWithData.contains(g_type)) def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(builtin_type, + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), Grant.typesWithMultibeatData.contains(g_type), - co.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = builtin_type && (g_type === Grant.voluntaryAck) - def requiresAck(dummy: Int = 0): Bool = !Bool(networkPreservesPointToPointOrdering) && !isVoluntary() + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = new Finish + val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) f.manager_xact_id := this.manager_xact_id f } } object Grant { - val nBuiltinGrantTypes = 5 - //TODO Use Enum - def voluntaryAck = UInt(0) - def uncachedRead = UInt(1) - def uncachedReadBlock = UInt(2) - def uncachedWrite = UInt(3) - def uncachedAtomic = UInt(4) - def typesWithData = Vec(uncachedRead, uncachedReadBlock, uncachedAtomic) - def typesWithMultibeatData= Vec(uncachedReadBlock) + val nBuiltInTypes = 4 + def voluntaryAckType = UInt("b00") + def ackType = UInt("b01") + def dataBeatType = UInt("b10") + def dataBlockType = UInt("b11") + def typesWithData = Vec(dataBlockType, dataBeatType) + def typesWithMultibeatData= Vec(dataBlockType) def apply( - builtin_type: Bool, + is_builtin_type: Bool, g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Grant = { val gnt = new Grant - gnt.builtin_type := builtin_type + gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id gnt.manager_xact_id := manager_xact_id @@ -415,23 +377,12 @@ object Grant { gnt.data := data gnt } - - def getGrantTypeForUncached(a: Acquire): UInt = { - MuxLookup(a.a_type, Grant.uncachedRead, Array( - Acquire.uncachedRead -> Grant.uncachedRead, - Acquire.uncachedReadBlock -> Grant.uncachedReadBlock, - Acquire.uncachedWrite -> Grant.uncachedWrite, - Acquire.uncachedWriteBlock -> Grant.uncachedWrite, - Acquire.uncachedAtomic -> Grant.uncachedAtomic - )) - } } class Finish extends ClientToManagerChannel with HasManagerTransactionId - // Complete IO definitions for two types of TileLink clients -class UncachedTileLinkIO extends Bundle { +class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) @@ -456,11 +407,17 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } object TileLinkIOWrapper { - def apply[T <: Data](utl: UncachedTileLinkIO) = { + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { val conv = Module(new TileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: TileLinkIO) = tl } abstract trait HasArbiterTypes { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 1230771c..3c334376 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,437 +2,147 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] case object NIncoherentClients extends Field[Int] case object NCoherentClients extends Field[Int] -case object L2StoreDataQueueDepth extends Field[Int] -case object L2CoherencePolicy extends Field[DirectoryRepresentation => CoherencePolicy] -case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object L2CoherencePolicy extends Field[CoherencePolicy] -abstract trait CoherenceAgentParameters extends UsesParameters - with TileLinkParameters { +trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors val nCoherentClients = params(NCoherentClients) val nIncoherentClients = params(NIncoherentClients) val nClients = nCoherentClients + nIncoherentClients - val sdqDepth = params(L2StoreDataQueueDepth)*tlDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) - val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases + def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerDataBeats = outerTLParams(TLDataBeats) + val outerDataBits = outerTLParams(TLDataBits) + def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerDataBeats = innerTLParams(TLDataBeats) + val innerDataBits = innerTLParams(TLDataBits) + val innerBeatAddrBits = log2Up(innerDataBeats) + val innerByteAddrBits = log2Up(innerDataBits/8) + require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } +abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters +abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters -class DataQueueLocation extends Bundle with CoherenceAgentParameters { - val idx = UInt(width = dqIdxBits) - val loc = UInt(width = log2Ceil(nDataQueueLocations)) -} -object DataQueueLocation { - def apply(idx: UInt, loc: UInt) = { - val d = new DataQueueLocation - d.idx := idx - d.loc := loc - d +trait HasCoherenceAgentWiringHelpers { + def doOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( + out: DecoupledIO[S], + ins: Seq[DecoupledIO[S]]) { + def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter( + out.bits.clone, + ins.size, + out.bits.payload.tlDataBeats, + lock _)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } + + def doInputRouting[T <: HasManagerTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { + val idx = in.bits.payload.manager_xact_id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + in.ready := Vec(outs.map(_.ready)).read(idx) } } -abstract class CoherenceAgent(innerId: String, outerId: String) extends Module - with CoherenceAgentParameters { - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val incoherent = Vec.fill(nClients){Bool()}.asInput - } +trait HasInnerTLIO extends CoherenceAgentBundle { + val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val incoherent = Vec.fill(nClients){Bool()}.asInput + def iacq(dummy: Int = 0) = inner.acquire.bits.payload + def iprb(dummy: Int = 0) = inner.probe.bits.payload + def irel(dummy: Int = 0) = inner.release.bits.payload + def ignt(dummy: Int = 0) = inner.grant.bits.payload + def ifin(dummy: Int = 0) = inner.finish.bits.payload } -class L2BroadcastHub(bankId: Int, innerId: String, outerId: String) extends - CoherenceAgent(innerId, outerId) { - - val internalDataBits = new DataQueueLocation().getWidth - val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - - // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new VoluntaryReleaseTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new AcquireTracker(id, bankId, innerId, outerId), {case TLDataBits => internalDataBits})) - - // Propagate incoherence flags - trackerList.map(_.io.tile_incoherent := io.incoherent.toBits) - - // Queue to store impending UncachedWrite data - val acquire = io.inner.acquire - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val) - val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } - - // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready - } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires - - // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() - val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, tlDataBeats) //TODO Zero width - val vwbdq = Vec.fill(tlDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } - - // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) - } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) - - // Wire probe requests to clients - val probe_arb = Module(new Arbiter(new LogicalNetworkIO(new Probe), trackerList.size)) - io.inner.probe <> probe_arb.io.out - probe_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.probe } - - // Wire grant reply to initiating client - def hasData(m: LogicalNetworkIO[Grant]) = m.payload.hasMultibeatData() - val grant_arb = Module(new LockingArbiter(new LogicalNetworkIO(new Grant), trackerList.size, tlDataBeats, Some(hasData _))) - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat - io.inner.grant <> grant_arb.io.out - grant_arb.io.in zip trackerList map { case (arb, t) => arb <> t.io.inner.grant } - - // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), - { case TLId => outerId; case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) - val is_in_sdq = outer_data_ptr.loc === inStoreQueue - val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && - outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( - inStoreQueue -> sdq(outer_data_ptr.idx), - inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer <> outer_arb.io.out - - // Update SDQ valid bits - when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | - PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) - } +trait HasUncachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } - -abstract class XactTracker(innerId: String, outerId: String) extends Module { - val (co, tlDataBeats) = (params(TLCoherence), params(TLDataBeats)) - val nClients = params(NCoherentClients) + params(NIncoherentClients) - val io = new Bundle { - val inner = Bundle(new TileLinkIO, {case TLId => innerId}).flip - val outer = Bundle(new UncachedTileLinkIO, {case TLId => outerId}) - val tile_incoherent = Bits(INPUT, nClients) - val has_acquire_conflict = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) - } - - val cacq = io.inner.acquire.bits - val crel = io.inner.release.bits - val cgnt = io.inner.grant.bits - val cfin = io.inner.finish.bits - val macq = io.outer.acquire.bits - val mgnt = io.outer.grant.bits - +trait HasCachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new TileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits.payload + def oprb(dummy: Int = 0) = outer.probe.bits.payload + def orel(dummy: Int = 0) = outer.release.bits.payload + def ognt(dummy: Int = 0) = outer.grant.bits.payload + def ofin(dummy: Int = 0) = outer.finish.bits.payload } -class VoluntaryReleaseTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) - val state = Reg(init=s_idle) +class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact_r_type = Reg(io.inner.release.bits.payload.r_type) - val xact_addr_block = Reg(io.inner.release.bits.payload.addr_block.clone) - val xact_client_xact_id = Reg(io.inner.release.bits.payload.client_xact_id.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.release.bits.payload.data.clone) } - val xact = Release( - voluntary = Bool(true), - r_type = xact_r_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block) - - val collect_inner_data = Reg(init=Bool(false)) - // TODO: assert that all releases have full blocks of data - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.release.fire() && io.inner.release.bits.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasMultibeatData(), tlDataBeats) - - io.has_acquire_conflict := Bool(false) - io.has_release_match := crel.payload.isVoluntary() - - io.outer.grant.ready := Bool(false) - io.outer.acquire.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) - - io.outer.acquire.bits.payload := Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - - when(collect_inner_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_data(inner_data_cnt) := crel.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } - } - - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact_src := crel.header.src - xact_r_type := crel.payload.r_type - xact_addr_block := crel.payload.addr_block - xact_client_xact_id := crel.payload.client_xact_id - xact_data(UInt(0)) := crel.payload.data - collect_inner_data := crel.payload.hasMultibeatData() - state := Mux(crel.payload.hasData(), s_outer, - Mux(crel.payload.requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - when(outer_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) - } - } - is(s_grant) { - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } +abstract class CoherenceAgent extends CoherenceAgentModule { + def innerTL: TileLinkIO + def outerTL: TileLinkIO + def incoherent: Vec[Bool] } -class AcquireTracker(trackerId: Int, bankId: Int, innerId: String, outerId: String) extends XactTracker(innerId, outerId) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) +abstract class ManagerCoherenceAgent extends CoherenceAgent + with HasCoherenceAgentWiringHelpers { + val io = new ManagerTLIO + def innerTL = io.inner + def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def incoherent = io.incoherent +} - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact_builtin_type = Reg(io.inner.acquire.bits.payload.builtin_type.clone) - val xact_a_type = Reg(io.inner.acquire.bits.payload.a_type.clone) - val xact_client_xact_id = Reg(io.inner.acquire.bits.payload.client_xact_id.clone) - val xact_addr_block = Reg(io.inner.acquire.bits.payload.addr_block.clone) - val xact_addr_beat = Reg(io.inner.acquire.bits.payload.addr_beat.clone) - val xact_subblock = Reg(io.inner.acquire.bits.payload.subblock.clone) - val xact_data = Vec.fill(tlDataBeats){ Reg(io.inner.acquire.bits.payload.data.clone) } - val xact = Acquire( - builtin_type = xact_builtin_type, - a_type = xact_a_type, - client_xact_id = xact_client_xact_id, - addr_block = xact_addr_block, - addr_beat = xact_addr_beat, - data = UInt(0), - subblock = xact_subblock) +class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO - val collect_inner_data = Reg(init=Bool(false)) - //TODO: Assert that if xact.uncached, xact_a_type is ReadBlock or WriteBlock - val (inner_data_cnt, inner_data_done) = - Counter(io.inner.acquire.fire() && cacq.payload.hasMultibeatData(), tlDataBeats) - val (outer_data_cnt, outer_data_done) = - Counter(io.outer.acquire.fire() && macq.payload.hasMultibeatData(), tlDataBeats) - val (cgnt_data_cnt, cgnt_data_done) = - Counter(io.inner.grant.fire() && cgnt.payload.hasMultibeatData(), tlDataBeats) +abstract class HierarchicalCoherenceAgent extends CoherenceAgent { + val io = new HierarchicalTLIO + def innerTL = io.inner + def outerTL = io.outer + def incoherent = io.incoherent +} +trait HasTrackerConflictIO extends Bundle { + val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) +} - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) +class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO +class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO - val pending_outer_write = xact.hasData() - val pending_outer_read = co.requiresOuterRead(xact, co.managerMetadataOnFlush) - - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), UIntToOH(cacq.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.tile_incoherent | myflag) - - io.has_acquire_conflict := xact.conflicts(cacq.payload) && - !collect_inner_data && - (state != s_idle) - io.has_release_match := xact.conflicts(crel.payload) && - !crel.payload.isVoluntary() && - (state != s_idle) - - val outer_write_acq = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = outer_data_cnt, - data = xact_data(outer_data_cnt)), - { case TLId => outerId }) - val outer_write_rel = Bundle(UncachedWriteBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = crel.payload.addr_beat, - data = crel.payload.data), - { case TLId => outerId }) - val outer_read = Bundle(UncachedReadBlock( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block), - { case TLId => outerId }) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default - io.outer.grant.ready := io.inner.grant.ready - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := xact.makeProbe() - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := xact.makeGrant(UInt(trackerId)) // Data bypassed in parent - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - - when(collect_inner_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact_data(inner_data_cnt) := cacq.payload.data - } - when(inner_data_done) { collect_inner_data := Bool(false) } +abstract class XactTracker extends CoherenceAgentModule { + def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - val needs_outer_write = cacq.payload.hasData() - val needs_outer_read = co.requiresOuterRead(cacq.payload, co.managerMetadataOnFlush) - when(io.inner.acquire.valid) { - xact_builtin_type := cacq.payload.builtin_type - xact_a_type := cacq.payload.a_type - xact_addr_block := cacq.payload.addr_block - xact_addr_beat := cacq.payload.addr_beat - xact_client_xact_id := cacq.payload.client_xact_id - xact_data(UInt(0)) := cacq.payload.data - xact_subblock := cacq.payload.subblock - xact_src := cacq.header.src - collect_inner_data := cacq.payload.hasMultibeatData() - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, - Mux(needs_outer_write, s_mem_write, - Mux(needs_outer_read, s_mem_read, s_make_grant))) - } - } - is(s_probe) { - // Generate probes - io.inner.probe.valid := probe_flags.orR - when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) - } - - // Handle releases, which may have data to be written back - io.inner.release.ready := !crel.payload.hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { - when(crel.payload.hasData()) { - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel - when(io.outer.acquire.ready) { - when(outer_data_done) { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } .otherwise { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_read) { // Read data from outer memory (possibly what was just written) - io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_read - when(io.outer.acquire.ready) { state := s_mem_resp } - } - is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := (if(tlDataBeats == 1) Bool(true) - else !collect_inner_data || (outer_data_cnt < inner_data_cnt)) - io.outer.acquire.bits.payload := outer_write_acq - when(outer_data_done) { - state := Mux(pending_outer_read, s_mem_read, s_make_grant) - } - } - is(s_make_grant) { // Manufactor a local grant (some kind of permission upgrade) - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_mem_resp) { // Wait to forward grants from outer memory - when(io.outer.grant.valid && mgnt.payload.client_xact_id === UInt(trackerId)) { - io.inner.grant.valid := Bool(true) - } - when(cgnt_data_done) { - state := Mux(cgnt.payload.requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { // Wait for transaction to complete - when(io.inner.finish.valid && cfin.payload.manager_xact_id === UInt(trackerId)) { - state := s_idle - } - } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + beat: UInt = UInt(0)) = { + connectDataBeatCounter(in.fire(), in.bits.payload, beat) + } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } } From 8e41fcf6fc58f5194f70fbbf6ed057c9972b7340 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 13:13:12 -0800 Subject: [PATCH 289/688] reduce MemIFTag size, enable non pow2 HellaFLowQueue size --- uncore/src/main/scala/memserdes.scala | 68 ++++++++++++--------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index f10b3257..7542815b 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -244,20 +244,13 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { (io.tl.release.fire() && rel_has_data), tlDataBeats) val tl_done_out = Reg(init=Bool(false)) val make_grant_ack = Reg(init=Bool(false)) - val grant_for_rel = Grant( - is_builtin_type = Bool(true), - g_type = Grant.voluntaryAckType, - client_xact_id = tag_out, - manager_xact_id = UInt(0)) - val grant_for_acq_write = ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_out(0), - a_type = tag_out >> UInt(1), - client_xact_id = tag_out >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0)) + gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Mux(data_from_rel, grant_for_rel, grant_for_acq_write) + gnt_arb.io.in(1).bits.payload := Grant( + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -278,7 +271,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := io.tl.release.bits.payload.client_xact_id + tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) @@ -289,8 +283,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := Bool(true) cmd_sent_out := Bool(false) tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data data_from_rel := Bool(false) @@ -358,7 +351,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := io.tl.release.bits.payload.client_xact_id + io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -366,8 +360,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_cmd.bits.rw := acq_has_data io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.a_type, - io.tl.acquire.bits.payload.is_builtin_type) + io.tl.acquire.bits.payload.isBuiltInType()) io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data } @@ -409,12 +402,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = tag_in(0), - a_type = tag_in >> UInt(1), - client_xact_id = tag_in >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) @@ -442,23 +435,22 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := - ManagerMetadata.onReset.makeGrant( - acq = Acquire( - is_builtin_type = io.mem.resp.bits.tag(0), - a_type = io.mem.resp.bits.tag >> UInt(1), - client_xact_id = io.mem.resp.bits.tag >> UInt(io.tl.tlAcquireTypeBits+1), - addr_block = UInt(0)), //DNC - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.payload := Grant( + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), + Grant.dataBlockType, + UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module { val io = new QueueIO(data, entries) - require(isPow2(entries) && entries > 1) + require(entries > 1) val do_flow = Bool() val do_enq = io.enq.fire() && !do_flow @@ -466,7 +458,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val maybe_full = Reg(init=Bool(false)) val enq_ptr = Counter(do_enq, entries)._1 - val deq_ptr = Counter(do_deq, entries)._1 + val (deq_ptr, deq_done) = Counter(do_deq, entries) when (do_enq != do_deq) { maybe_full := do_enq } val ptr_match = enq_ptr === deq_ptr @@ -482,7 +474,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module when (do_enq) { ram(enq_ptr) := io.enq.bits } when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, deq_ptr + UInt(1), deq_ptr) + ram_addr := Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) } io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) From df79e7ff8dcadc09d91d6b2dede2b583487f2af0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Mar 2015 14:40:31 -0800 Subject: [PATCH 290/688] secondary miss bug --- uncore/src/main/scala/coherence.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2524f3d1..1e8d1b60 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -60,9 +60,7 @@ trait HasClientSideCoherencePolicy { first_cmd: UInt, second_cmd: UInt, meta: ClientMetadata): Bool = { - isWriteIntent(second_cmd) && - !isWriteIntent(first_cmd) || - (getAcquireType(first_cmd, meta) != getAcquireType(second_cmd, meta)) + isWriteIntent(second_cmd) && !isWriteIntent(first_cmd) } //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: From 002f1a1b393df17b4fe0e07f0a43dc36bdc0ac4f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 12:40:37 -0700 Subject: [PATCH 291/688] pin outer finish header --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75eb897d..aa2097af 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -831,6 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.ognt().requiresAck()) { pending_finish.payload := pending_finish_on_ognt pending_finish.header.dst := io.outer.grant.bits.header.src + pending_finish.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, From a1f04386f75664654a049c1d5b4078f402515c70 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 9 Mar 2015 16:34:59 -0700 Subject: [PATCH 292/688] Headerless TileLinkIO and arbiters --- uncore/src/main/scala/htif.scala | 7 +- uncore/src/main/scala/tilelink.scala | 123 ++++++++++++++++++++------- 2 files changed, 92 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 4d5e1135..644a8317 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new TileLinkIO + val mem = new HeaderlessUncachedTileLinkIO val scr = new SCRIO } @@ -195,19 +195,16 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val init_addr = addr.toUInt >> UInt(offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq - io.mem.acquire.bits.payload := Mux(cmd === cmd_writemem, + io.mem.acquire.bits := Mux(cmd === cmd_writemem, PutBlock( addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.acquire.bits.payload.data := mem_req_data io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - io.mem.probe.ready := Bool(false) - io.mem.release.valid := Bool(false) val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c8996ddf..3d46ae80 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import scala.math.max +import scala.reflect.ClassTag // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -290,7 +291,6 @@ object Probe { } } - class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -420,6 +420,42 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO) = tl } +// This version of TileLinkIO does not contain network headers for packets +// that originate in the Clients (i.e. Acquire and Release). These headers +// are provided in the top-level that instantiates the clients and network. +// By eliding the header subbundles within the clients we can enable +// hierarchical P&R while minimizing unconnected port errors in GDS. +class HeaderlessUncachedTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new Acquire) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new Release) +} + +class HeaderlessTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new HeaderlessUncachedTileLinkIO().flip + val out = new HeaderlessTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(false) + io.out.release.valid := Bool(false) +} + +object HeaderlessTileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId @@ -428,6 +464,7 @@ abstract trait HasArbiterTypes { HasClientTransactionId with HasTileLinkData } + // Utility functions for constructing TileLinkIO arbiters abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule with HasArbiterTypes { @@ -438,34 +475,65 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) - out <> arb.io.out - ins.zipWithIndex.zip(arb.io.in).map{ case ((req,id), arb) => { + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) req.ready := arb.ready }} + arb.io.out <> mngr } - def hookupManagerSource[M <: ManagerSourcedWithId] - (ins: Seq[DecoupledIO[LogicalNetworkIO[M]]], - out: DecoupledIO[LogicalNetworkIO[M]]) { - out.ready := Bool(false) + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + arb.io.out <> mngr + } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] + (clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) for (i <- 0 until arbN) { - ins(i).valid := Bool(false) - when (arbIdx(out.bits.payload) === UInt(i)) { - ins(i).valid := out.valid - out.ready := ins(i).ready + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready } - ins(i).bits := out.bits - ins(i).bits.payload.client_xact_id := managerSourcedClientXactId(out.bits.payload) + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := + managerSourcedClientXactId(mngr.bits.payload) } } + + def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] + (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_||_) + } } abstract class UncachedTileLinkIOArbiter(n: Int) @@ -474,13 +542,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) val in = Vec.fill(n){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { @@ -488,18 +552,11 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { val in = Vec.fill(n){new TileLinkIO}.flip val out = new TileLinkIO } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) hookupClientSource(io.in.map(_.release), io.out.release) - hookupManagerSource(io.in.map(_.grant), io.out.grant) - - io.in.map{ _.probe.valid := io.out.probe.valid } - io.in.map{ _.probe.bits := io.out.probe.bits } - io.out.probe.ready := io.in.map(_.probe.ready).reduce(_||_) - - val finish_arb = Module(new RRArbiter(new LogicalNetworkIO(new Finish), n)) - io.out.finish <> finish_arb.io.out - finish_arb.io.in zip io.in map { case (arb, req) => arb <> req.finish } + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } // Appends the port index of the arbiter to the client_xact_id From 17072a00416470e93e0b9ffc817f89b950753c23 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 10 Mar 2015 01:15:03 -0700 Subject: [PATCH 293/688] L2 Writeback bugfix --- uncore/src/main/scala/cache.scala | 27 +++++++++++-------- uncore/src/main/scala/coherence.scala | 2 ++ uncore/src/main/scala/memserdes.scala | 37 +++++++++++++++++---------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index aa2097af..ec25501c 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -566,17 +566,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && - xact_meta.coh.outer.requiresVoluntaryWriteback() - val needs_probes = xact_meta.coh.inner.requiresProbes(xact) + (xact_meta.coh.outer.requiresVoluntaryWriteback() || + xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( + val pending_icoh_on_irel = pending_coh.inner.onRelease( incoming = io.irel(), - src = io.inner.release.bits.header.src), - pending_coh.outer.onHit(M_XWR)) // WB is a write + src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, pending_coh.outer.onGrant(io.ognt(), xact.op_code())) @@ -768,7 +767,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) val _needs_writeback = !_tag_match && do_allocate && - _coh.outer.requiresVoluntaryWriteback() + (_coh.outer.requiresVoluntaryWriteback() || + _coh.inner.requiresProbesOnVoluntaryWriteback()) val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { @@ -796,12 +796,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - pending_coh := pending_coh_on_irel + pending_coh.inner := pending_icoh_on_irel // Handle released dirty data //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { irel_had_data := Bool(true) + pending_coh.outer := pending_ocoh_on_irel mergeDataInner(data_buffer, io.irel()) } // We don't decrement release_count until we've received all the data beats. @@ -931,8 +932,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val resp_data_done = connectInternalDataBeatCounter(io.data.resp) val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), + incoming = io.irel(), src = io.inner.release.bits.header.src) + val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) @@ -1008,6 +1010,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle released dirty data when(io.irel().hasData()) { irel_had_data := Bool(true) + xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } // We don't decrement release_count until we've received all the data beats. @@ -1016,7 +1019,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(irel_had_data, s_outer_release, s_data_read) + state := Mux(irel_had_data, // If someone released a dirty block + s_outer_release, // write that block back, otherwise + Mux(xact_coh.outer.requiresVoluntaryWriteback(), + s_data_read, // write extant dirty data back, or just + s_wb_resp)) // drop a clean block after collecting acks } } is(s_data_read) { diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 1e8d1b60..862eb484 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -383,6 +383,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), @@ -501,6 +502,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing // only a single sharer (also would need // notification msg to track clean drops) + // Also could avoid probes on outer WBs. def requiresProbes(a: Acquire, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7542815b..eafc44bd 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -212,6 +212,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mem = new MemIO } val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) val tlDataBits = params(TLDataBits) @@ -235,8 +236,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(Bits()) - val addr_out = Reg(Bits()) + val tag_out = Reg(UInt(width = mifTagBits)) + val addr_out = Reg(UInt(width = mifAddrBits)) val has_data = Reg(init=Bool(false)) val data_from_rel = Reg(init=Bool(false)) val (tl_cnt_out, tl_wrap_out) = @@ -343,26 +344,34 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid io.mem.req_cmd.valid := Bool(true) cmd_sent_out := io.mem.req_cmd.ready - tag_out := io.mem.req_cmd.bits.tag - addr_out := io.mem.req_data.bits.data - has_data := io.mem.req_cmd.bits.rw tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) make_grant_ack := Bool(true) - io.mem.req_cmd.bits.rw := rel_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - io.mem.req_cmd.bits.addr := io.tl.release.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.release.bits.payload.data + val tag = Cat(io.tl.release.bits.payload.client_xact_id, + io.tl.release.bits.payload.isVoluntary()) + val addr = io.tl.release.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := rel_has_data + tag_out := tag + addr_out := addr + has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data - io.mem.req_cmd.bits.rw := acq_has_data - io.mem.req_cmd.bits.tag := Cat(io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - io.mem.req_cmd.bits.addr := io.tl.acquire.bits.payload.addr_block io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_cmd.bits.rw := acq_has_data + val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + io.tl.acquire.bits.payload.isBuiltInType()) + val addr = io.tl.acquire.bits.payload.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := acq_has_data + tag_out := tag + addr_out := addr + has_data := acq_has_data } } } @@ -380,7 +389,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } when(tl_wrap_out) { tl_done_out := Bool(true) } when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) + gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } } when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { From 3ab1aca7decddfa7e1bf5ee35c864c43076e23ba Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Wed, 11 Mar 2015 01:56:47 -0700 Subject: [PATCH 294/688] L2 subblock access bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ec25501c..1d117ded 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -412,7 +412,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { - val (multi_cnt, multi_done) = Counter(inc, data.refillCycles) + val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } From b4ed1d912160964c5ea41a1b2614afa21c83f71f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 14:28:17 -0700 Subject: [PATCH 295/688] Add builtin prefetch types to TileLink --- uncore/src/main/scala/tilelink.scala | 48 ++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3d46ae80..ef0d3307 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -109,6 +109,8 @@ class Acquire extends ClientToManagerChannel def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) + // Assumes no custom types have data def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) @@ -124,7 +126,8 @@ class Acquire extends ClientToManagerChannel Acquire.getBlockType -> Grant.dataBlockType, Acquire.putType -> Grant.ackType, Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType)) + Acquire.putAtomicType -> Grant.dataBeatType, + Acquire.prefetchType -> Grant.ackType)) } } @@ -136,6 +139,7 @@ object Acquire { def putType = UInt("b010") def putBlockType = UInt("b011") def putAtomicType = UInt("b100") + def prefetchType = UInt("b101") def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -172,9 +176,9 @@ object Acquire { // Asks for a single TileLink beat of data object Get { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -189,8 +193,8 @@ object Get { // Asks for an entire cache block of data object GetBlock { def apply( - client_xact_id: UInt = UInt(0), - addr_block: UInt, + client_xact_id: UInt = UInt(0), + addr_block: UInt, alloc: Bool = Bool(true)): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -201,6 +205,22 @@ object GetBlock { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with read permissions +object GetPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XRD, Bool(true))) + } +} + // Writes up to a single TileLink beat of data, using mask object Put { def apply( @@ -275,6 +295,22 @@ object PutAtomic { } } +// Prefetch a cache block into the next level of the memory hierarchy +// with write permissions +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) From 059575c334dd34cc818231fdd5ff82769171c28f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 15:43:41 -0700 Subject: [PATCH 296/688] cleanup mergeData and prep for cleaner data_buffer in L2 --- uncore/src/main/scala/cache.scala | 72 +++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 2 + 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1d117ded..87473587 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -6,15 +6,16 @@ import Chisel._ case object CacheName extends Field[String] case object NSets extends Field[Int] case object NWays extends Field[Int] -case object BlockOffBits extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object CacheBlockBytes extends Field[Int] +case object CacheBlockOffsetBits extends Field[Int] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) - val blockOffBits = params(BlockOffBits) + val blockOffBits = params(CacheBlockOffsetBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits val tagBits = params(PAddrBits) - untagBits @@ -165,6 +166,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val blockAddrBits = params(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats + val internalDataBeats = params(CacheBlockBytes)*8/rowBits require(refillCyclesPerBeat == 1) val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) @@ -410,6 +412,14 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + class CacheBlockBuffer { + val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) + + def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) + def inner = Vec.fill(innerDataBeats){ Bits(width = innerDataBits) }.fromBits(buffer) + def outer = Vec.fill(outerDataBeats){ Bits(width = outerDataBits) }.fromBits(buffer) + } + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) @@ -554,9 +564,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats+1) { // Extra entry holds AMO result + val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) - } + } + val amo_result = Reg(io.iacq().data.clone) val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -612,44 +623,31 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer(0) // default + amoalu.io.rhs := data_buffer.head // default - // TODO: figure out how to merge the following three versions of this func - def mergeDataInternal[T <: HasL2Data](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) + def mergeData[T <: HasTileLinkData] + (byteAddrBits: Int, dataBits: Int) + (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + val old_data = incoming + val new_data = buffer(beat) val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) + val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = + val valid_beat = xact.addr_beat === beat + val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & + UInt(0, width = dataBits))) + buffer(beat) := ~wmask & old_data | wmask & Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result - } - def mergeDataInner[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) = mergeDataOuter(buffer, incoming) - def mergeDataOuter[T <: HasTileLinkData](buffer: Vec[UInt], incoming: T) { - val old_data = incoming.data - val new_data = buffer(incoming.addr_beat) - val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(innerByteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz - val wmask = - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), - UInt(0, width = innerDataBits))) - buffer(incoming.addr_beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) - when(xact.is(Acquire.putAtomicType)) { buffer(innerDataBeats) := old_data } // For AMO result + when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } + val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ + val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ + val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === @@ -694,7 +692,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { manager_xact_id = UInt(trackerId), addr_beat = ignt_data_cnt, data = Mux(xact.is(Acquire.putAtomicType), - data_buffer(innerDataBeats), + amo_result, data_buffer(ignt_data_cnt))) io.inner.acquire.ready := Bool(false) @@ -803,7 +801,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel()) + mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -824,7 +822,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt()) + mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -851,13 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits) + mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3c334376..05319ee9 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -21,6 +21,8 @@ trait CoherenceAgentParameters extends UsesParameters { def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerByteAddrBits = log2Up(outerDataBits/8) def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) From 1aff919c24a80b54e95efa5f1389da1df7b4f1aa Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 11 Mar 2015 17:32:06 -0700 Subject: [PATCH 297/688] added prefetchAck Grant type --- uncore/src/main/scala/memserdes.scala | 6 +++--- uncore/src/main/scala/tilelink.scala | 29 ++++++++++++++------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index eafc44bd..b9786b4c 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -249,7 +249,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(1).valid := Bool(false) gnt_arb.io.in(1).bits.payload := Grant( is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.ackType), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) @@ -414,7 +414,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = tag_in(0), g_type = Mux(tag_in(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = tag_in >> UInt(1), manager_xact_id = UInt(0), @@ -447,7 +447,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { gnt_arb.io.in(0).bits.payload := Grant( is_builtin_type = io.mem.resp.bits.tag(0), g_type = Mux(io.mem.resp.bits.tag(0), - Grant.dataBlockType, + Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol client_xact_id = io.mem.resp.bits.tag >> UInt(1), manager_xact_id = UInt(0), diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ef0d3307..531d3f52 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -121,13 +121,13 @@ class Acquire extends ClientToManagerChannel def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType def getBuiltInGrantType(dummy: Int = 0): UInt = { - MuxLookup(this.a_type, Grant.ackType, Array( - Acquire.getType -> Grant.dataBeatType, - Acquire.getBlockType -> Grant.dataBlockType, - Acquire.putType -> Grant.ackType, - Acquire.putBlockType -> Grant.ackType, - Acquire.putAtomicType -> Grant.dataBeatType, - Acquire.prefetchType -> Grant.ackType)) + MuxLookup(this.a_type, Grant.putAckType, Array( + Acquire.getType -> Grant.getDataBeatType, + Acquire.getBlockType -> Grant.getDataBlockType, + Acquire.putType -> Grant.putAckType, + Acquire.putBlockType -> Grant.putAckType, + Acquire.putAtomicType -> Grant.getDataBeatType, + Acquire.prefetchType -> Grant.prefetchAckType)) } } @@ -389,13 +389,14 @@ class Grant extends ManagerToClientChannel } object Grant { - val nBuiltInTypes = 4 - def voluntaryAckType = UInt("b00") - def ackType = UInt("b01") - def dataBeatType = UInt("b10") - def dataBlockType = UInt("b11") - def typesWithData = Vec(dataBlockType, dataBeatType) - def typesWithMultibeatData= Vec(dataBlockType) + val nBuiltInTypes = 5 + def voluntaryAckType = UInt("b000") + def putAckType = UInt("b001") + def prefetchAckType = UInt("b011") + def getDataBeatType = UInt("b100") + def getDataBlockType = UInt("b101") + def typesWithData = Vec(getDataBlockType, getDataBeatType) + def typesWithMultibeatData= Vec(getDataBlockType) def apply( is_builtin_type: Bool, From f75126c39c9925fc2073a82e03c270372e6bb06d Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 298/688] Require self probes for all built-in Acquire types This ensures that puts by the RoCC accelerator properly invalidates its tile's L1 D$, with which it currently shares the same TileLink port. --- uncore/src/main/scala/tilelink.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 531d3f52..7ea79ad1 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -117,8 +117,9 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache: - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() && a_type === Acquire.getBlockType + //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache + // and nbdcache invalidations after RoCC accelerator puts + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 8f8022379c11755367c0f35fce6a4775be91e06b Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 11 Mar 2015 23:24:58 -0700 Subject: [PATCH 299/688] Fix AMO opcode extraction --- uncore/src/main/scala/tilelink.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7ea79ad1..d353205e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -96,7 +96,9 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(hasData(), M_XWR, union(opSizeOff-1, opCodeOff)) + def op_code(dummy: Int = 0) = Mux(isBuiltInType() && + (a_type === Acquire.putType || a_type === Acquire.putBlockType), + M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) From 5e40c8ba77c7067fa686434db61cd9da4e3bb84f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 14:36:46 -0700 Subject: [PATCH 300/688] write back meta data when cache miss even when coherence meta data is clean --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 87473587..9815855e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val meta = pending_coh_on_ignt != xact_meta.coh - when(meta) { pending_coh := pending_coh_on_ignt } - state := Mux(meta, s_meta_write, + val coh_dirty = pending_coh_on_ignt != xact_meta.coh + when(coh_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(!is_hit || coh_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From 2c31ed64261cafefb0273cb41dde5d002052dc73 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 15:34:20 -0700 Subject: [PATCH 301/688] previous bug fix for meta data writeback wasn't quite right --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9815855e..7e601ba0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -868,9 +868,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { io.inner.grant.valid := Bool(true) when(ignt_data_done) { - val coh_dirty = pending_coh_on_ignt != xact_meta.coh - when(coh_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(!is_hit || coh_dirty, s_meta_write, + val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + when(meta_dirty) { pending_coh := pending_coh_on_ignt } + state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } } From dcc84c4dd3e429235a4739ebc3edaf18f891f149 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 09:33:56 -0700 Subject: [PATCH 302/688] arbiter probe ready bugfix --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d353205e..7da4cfab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -572,7 +572,7 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr: DecoupledIO[LogicalNetworkIO[M]]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } - mngr.ready := clts.map(_.ready).reduce(_||_) + mngr.ready := clts.map(_.ready).reduce(_&&_) } } From 8181262419de27535ce1fb396e82739533217fad Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 12 Mar 2015 16:22:14 -0700 Subject: [PATCH 303/688] clean up incoherent and probe flags --- uncore/src/main/scala/broadcast.scala | 30 ++++++++++++++------------- uncore/src/main/scala/cache.scala | 29 +++++++++++++------------- uncore/src/main/scala/uncore.scala | 2 +- 3 files changed, 32 insertions(+), 29 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index bd8a3f24..e28cbf1c 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -219,16 +219,15 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nClients))) - val probe_flags = Reg(init=Bits(0, width = nClients)) - val curr_p_id = PriorityEncoder(probe_flags) - val probe_initial_flags = Bits(width = nClients) - probe_initial_flags := Bits(0) - // issue self-probes for uncached read xacts to facilitate I$ coherence + val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val myflag = Mux(probe_self, Bits(0), - UIntToOH(io.inner.acquire.bits.header.src(log2Up(nClients)-1,0))) - probe_initial_flags := ~(io.incoherent.toBits | myflag) + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -307,18 +306,21 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat - probe_flags := probe_initial_flags - release_count := PopCount(probe_initial_flags) - state := Mux(probe_initial_flags.orR, s_probe, + val needs_probes = mask_incoherent.orR + when(needs_probes) { + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) + } + state := Mux(needs_probes, s_probe, Mux(pending_outer_write_, s_mem_write, Mux(pending_outer_read_, s_mem_read, s_make_grant))) } } is(s_probe) { // Generate probes - io.inner.probe.valid := probe_flags.orR + io.inner.probe.valid := pending_probes.orR when(io.inner.probe.ready) { - probe_flags := probe_flags & ~(UIntToOH(curr_p_id)) + pending_probes := pending_probes & ~UIntToOH(curr_p_id) } // Handle releases, which may have data to be written back diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e601ba0..dfe5b10a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -596,13 +596,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val mask_self = Mux(xact.requiresSelfProbe(), - full_sharers | (UInt(1) << xact_src), - full_sharers & ~UInt(UInt(1) << xact_src, width = nClients)) + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) @@ -770,8 +771,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } when(_needs_probes) { - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(_tag_match, Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade @@ -920,9 +921,11 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_finish = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nClients+1))) - val pending_probes = Reg(init = Bits(0, width = nClients)) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = io.wb.req.bits.coh.inner.full() + val mask_incoherent = full_sharers & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) @@ -985,12 +988,10 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val coh = io.wb.req.bits.coh - val needs_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_probes) { - val mask_incoherent = coh.inner.full() & ~io.incoherent.toBits - pending_probes := mask_incoherent(nCoherentClients-1,0) - release_count := PopCount(mask_incoherent(nCoherentClients-1,0)) + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) } state := Mux(needs_probes, s_probe, s_data_read) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 05319ee9..0d414f0d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -73,7 +73,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nClients){Bool()}.asInput + val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload From 3a78ca210d27e194c47ff3216539de5ef6f248c5 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Thu, 12 Mar 2015 16:33:35 -0700 Subject: [PATCH 304/688] bugfix in uncached TL to TL convertors --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7da4cfab..1aac8ce4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -443,7 +443,7 @@ class TileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } object TileLinkIOWrapper { @@ -484,7 +484,7 @@ class HeaderlessTileLinkIOWrapper extends TLModule { io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant io.out.finish <> io.in.finish - io.out.probe.ready := Bool(false) + io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } From 6e540825b214ac5c5a53a1f1e19fbaa213853145 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 14 Mar 2015 02:15:24 -0700 Subject: [PATCH 305/688] Use entire 12-bit CSR address --- uncore/src/main/scala/htif.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 644a8317..81d95453 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -34,7 +34,7 @@ class HostIO extends HTIFBundle class PCRReq extends Bundle { val rw = Bool() - val addr = Bits(width = 5) + val addr = Bits(width = 12) val data = Bits(width = 64) } @@ -206,7 +206,6 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.mem.finish.bits.payload.manager_xact_id := mem_gxid io.mem.finish.bits.header.dst := mem_gsrc - val pcr_reset = UInt(pcr_RESET)(pcr_addr.getWidth-1,0) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) @@ -214,7 +213,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != pcr_reset + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) cpu.pcr_req.bits.rw := cmd === cmd_writecr cpu.pcr_req.bits.addr := pcr_addr cpu.pcr_req.bits.data := pcr_wdata @@ -234,7 +233,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { when (cpu.pcr_req.valid && cpu.pcr_req.ready) { state := state_pcr_resp } - when (state === state_pcr_req && me && pcr_addr === pcr_reset) { + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) } From c03976896e8908fc306e108d83e5dd759ae2a5db Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 16:53:15 -0700 Subject: [PATCH 306/688] separate queues for resp tag and data --- uncore/src/main/scala/memserdes.scala | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b9786b4c..0a2b9c39 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -33,9 +33,9 @@ class MemReqCmd extends HasMemAddr with HasMemTag { val rw = Bool() } -class MemResp extends HasMemData with HasMemTag - +class MemTag extends HasMemTag class MemData extends HasMemData +class MemResp extends HasMemData with HasMemTag class MemIO extends Bundle { val req_cmd = Decoupled(new MemReqCmd) @@ -543,9 +543,20 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_dataq.io.enq <> io.mem.resp - io.cpu.resp <> resp_dataq.io.deq + // Have separate queues to allow for different mem implementations + val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) + resp_dataq.io.enq.valid := io.mem.resp.valid + resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + + val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tagq.io.enq.valid := io.mem.resp.valid + resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid + io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag + resp_dataq.io.deq.ready := io.cpu.resp.ready + resp_tagq.io.deq.ready := io.cpu.resp.ready inc := resp_dataq.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw From 23a6b007c1f6f357abfe2bad1209f8241bb31c9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Mar 2015 23:10:51 -0700 Subject: [PATCH 307/688] Fix BroadcastHub AcquiteTracker allocation bug and clean up tracker wiring --- uncore/src/main/scala/broadcast.scala | 100 +++++++++++++++----------- uncore/src/main/scala/cache.scala | 45 +++++++----- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index e28cbf1c..c8014f45 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -40,53 +40,53 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerList.map(_.io.incoherent := io.incoherent.toBits) // Queue to store impending Put data - val acquire = io.inner.acquire + val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) } val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = acquire.fire() && acquire.bits.payload.hasData() - val sdq = Vec.fill(sdqDepth){ Reg(io.inner.acquire.bits.payload.data) } - when (sdq_enq) { sdq(sdq_alloc_id) := acquire.bits.payload.data } + val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData() + when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val any_acquire_conflict = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - val block_acquires = any_acquire_conflict val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - alloc_arb.io.in(i).valid := t.acquire.ready - t.acquire.bits := acquire.bits - t.acquire.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - t.acquire.valid := alloc_arb.io.in(i).ready + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireMatchList = trackerList.map(_.io.has_acquire_match) + val any_acquire_matches = acquireMatchList.reduce(_||_) + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.bits.payload.data := + DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } - acquire.ready := trackerList.map(_.io.inner.acquire.ready).reduce(_||_) && sdq_rdy && !block_acquires - alloc_arb.io.out.ready := acquire.valid && sdq_rdy && !block_acquires + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + sdq_rdy && !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data - val release = io.inner.release - val voluntary = release.bits.payload.isVoluntary() - val vwbdq_enq = release.fire() && voluntary && release.bits.payload.hasData() + val voluntary = io.irel().isVoluntary() + val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Vec.fill(innerDataBeats){ Reg(release.bits.payload.data) } //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := release.bits.payload.data } + val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} - for( i <- 0 until trackerList.size ) { - val t = trackerList(i).io.inner - t.release.bits := release.bits - t.release.bits.payload.data := (if (i < nReleaseTransactors) - DataQueueLocation(rel_data_cnt, inVolWBQueue) - else DataQueueLocation(UInt(0), inClientReleaseQueue)).toBits - t.release.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(trackerList.map(_.io.inner.release.ready)).read(release_idx) - - // Wire finished transaction acks - val ack = io.inner.finish - trackerList.map(_.io.inner.finish.valid := ack.valid) - trackerList.map(_.io.inner.finish.bits := ack.bits) - ack.ready := Bool(true) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -107,7 +107,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val free_sdq = io.outer.acquire.fire() && io.outer.acquire.bits.payload.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, release.bits.payload.data, Array( + io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -216,8 +216,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && - Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType).contains(xact.a_type)), - "Broadcast Hub does not support PutAtomics or subblock Gets/Puts") // TODO + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType, + Acquire.prefetchType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) @@ -238,17 +239,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ognt_ack = Reg(init=Bool(false)) val pending_outer_write = xact.hasData() - val pending_outer_read = io.ignt().hasData() val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read = io.ignt().hasData() val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() io.has_acquire_conflict := xact.conflicts(io.iacq()) && - !collect_iacq_data && - (state != s_idle) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state != s_idle) - io.has_acquire_match := Bool(false) + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_probe) val outer_write_acq = Bundle(PutBlock( client_xact_id = UInt(trackerId), @@ -282,6 +284,18 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dfe5b10a..ef3cb3d7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -356,33 +356,34 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation - val acquire = io.inner.acquire val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val acquireList = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - acquireList.zip(alloc_arb.io.in).zipWithIndex.map { case((acq, arb), i) => - arb.valid := acq.ready - acq.bits := acquire.bits - acq.valid := arb.ready && (acquire_idx === UInt(i)) + trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { + case((tracker, arb), i) => + arb.valid := tracker.ready + tracker.bits := io.inner.acquire.bits + tracker.valid := arb.ready && (acquire_idx === UInt(i)) } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - acquire.ready := acquireList.map(_.ready).reduce(_||_) && !block_acquires - alloc_arb.io.out.ready := acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && + !block_acquires + alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires // Wire releases from clients - val release = io.inner.release val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).indexWhere{b: Bool => b} - val releaseList = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - releaseList.zipWithIndex.map { case(r, i) => - r.bits := release.bits - r.valid := release.valid && (release_idx === UInt(i)) + val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - release.ready := Vec(releaseList.map(_.ready)).read(release_idx) + io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -658,8 +659,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data - io.has_release_match := !io.irel().isVoluntary() && - (xact.addr_block === io.irel().addr_block) && + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && (state === s_probe) // If we're allocating in this cache, we can use the current metadata @@ -729,6 +730,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { From f6d1a2fb76cfe73ff87149264e9ac2f1237a82bb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:38 -0700 Subject: [PATCH 308/688] No more self-probes required --- uncore/src/main/scala/tilelink.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1aac8ce4..7bd218ad 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -119,9 +119,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - //TODO: This function is a hack to support Rocket icache snooping Rocket nbdcache - // and nbdcache invalidations after RoCC accelerator puts - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType() + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From b72230a9f04f18d9358ca427b69445ff090e6d95 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 00:09:55 -0700 Subject: [PATCH 309/688] PutBlock bugfix --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ef3cb3d7..8c65a20f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -636,7 +636,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.addr_beat === beat + val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), From f35a6a574f3386d282f9fcd9ef1fd692fe9e9c1c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:25:01 -0700 Subject: [PATCH 310/688] Add a queue on released data coming in to L2 --- uncore/src/main/scala/cache.scala | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8c65a20f..bc12cfeb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -322,14 +322,22 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nWays)) val tshrfile = Module(new TSHRFile(bankId)) - val meta = Module(new L2MetadataArray) - val data = Module(new L2DataArray(1)) - tshrfile.io.inner <> io.inner - tshrfile.io.meta <> meta.io - tshrfile.io.data <> data.io + //TODO: Expose queue depths and data array pipeline cycles as parameters? + tshrfile.io.inner.acquire <> io.inner.acquire + tshrfile.io.inner.probe <> io.inner.probe + tshrfile.io.inner.release <> Queue(io.inner.release) + tshrfile.io.inner.grant <> io.inner.grant + tshrfile.io.inner.finish <> io.inner.finish + io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent + + val meta = Module(new L2MetadataArray) + tshrfile.io.meta <> meta.io + + val data = Module(new L2DataArray(1)) + tshrfile.io.data <> data.io } class TSHRFileIO extends HierarchicalTLIO { From cf1df2d72a3b3cd687eed7f7ed8c3a17ac62758e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 311/688] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 145e15701e8e2dc6b34c252487201c0801b0d8cb Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 312/688] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 36fc67dc7c1011afa763350c819657a911fdd74b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 313/688] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..b6d22d5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -831,7 +831,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -852,7 +852,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -879,13 +879,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From f6fe037e303d708101c0d1d7b5716a9efb3919ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 314/688] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 260 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 114 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b6d22d5e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,38 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -819,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -829,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -839,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -853,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -864,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -873,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -915,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -952,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -988,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1029,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 40a5059cee6311f6bd5e74fc4a46aee5f34d99fe Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 315/688] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From 672fbb574f8d34066f946680d5c34addf6d2614e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 316/688] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From b21fcdfbe0836646a2a5a13f0a5b1369dab6c7e1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 317/688] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 2d33dd8e6ee2dbf257a046a2619f89c75b613fee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 318/688] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From fd8f6ff0269808bdc9bc2dc24a28d22e7f5890d0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 319/688] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 257dcedcb34b341197983eba6df4f64786d0e2ae Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 320/688] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From ce9d4b6e70192c0353056c59dda435c2e58ab740 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 13:27:05 -0700 Subject: [PATCH 321/688] further amo cleanups --- uncore/src/main/scala/cache.scala | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bc12cfeb..e1c8bce4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -572,11 +572,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats) { Reg(io.iacq().data.clone) } - val amo_result = Reg(io.iacq().data.clone) + val amo_result = xact.data val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -638,13 +638,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) (buffer: Vec[UInt], beat: UInt, incoming: UInt) { - val old_data = incoming - val new_data = buffer(beat) + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) amoalu.io.lhs := old_data >> offset*amoOpSz amoalu.io.rhs := new_data >> offset*amoOpSz - val valid_beat = xact.is(Acquire.putBlockType) || xact.addr_beat === beat + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && + xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits, UIntToOH(offset)), @@ -746,6 +747,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().client_xact_id != xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") + //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -765,6 +767,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() + xact.data := UInt(0) data_buffer(io.iacq().addr_beat) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat From 0e51fef200776fd411b9c92e3d5eca4dc0c4d81b Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Mon, 16 Mar 2015 18:47:16 -0700 Subject: [PATCH 322/688] bugfix where an in-progress acquire can be blocked by another acquire tracker being free'd up in between --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e1c8bce4..da000c5e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,23 +364,28 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => + arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready - tracker.bits := io.inner.acquire.bits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) - } val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && !block_acquires + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } // Wire releases from clients val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ From 1471d9debc04a81c98c3af32d1846bfde77cce7a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 05:40:05 -0700 Subject: [PATCH 323/688] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da000c5e..469adc94 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -754,6 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") From 9bedde9a8a7317e601bf920a013ccea01b08f85e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 12:22:57 -0700 Subject: [PATCH 324/688] re-merge mem resp queues --- uncore/src/main/scala/memserdes.scala | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 0a2b9c39..a5bea3a3 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,21 +544,11 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_dataq = Module((new HellaQueue(numEntries)) { new MemData }) - resp_dataq.io.enq.valid := io.mem.resp.valid - resp_dataq.io.enq.bits.data := io.mem.resp.bits.data + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) + resp_data_q.io.enq <> io.mem.resp + io.cpu.resp <> resp_data_q.io.deq - val resp_tagq = Module((new HellaQueue(numEntries)) { new MemTag }) - resp_tagq.io.enq.valid := io.mem.resp.valid - resp_tagq.io.enq.bits.tag := io.mem.resp.bits.tag - - io.cpu.resp.valid := resp_dataq.io.deq.valid && resp_tagq.io.deq.valid - io.cpu.resp.bits.data := resp_dataq.io.deq.bits.data - io.cpu.resp.bits.tag := resp_tagq.io.deq.bits.tag - resp_dataq.io.deq.ready := io.cpu.resp.ready - resp_tagq.io.deq.ready := io.cpu.resp.ready - - inc := resp_dataq.io.deq.fire() + inc := resp_data_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 6af48e168a7e2ab8de76b67477c3619aad034b41 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 15:06:46 -0700 Subject: [PATCH 325/688] cleanup mergeData buffer --- uncore/src/main/scala/cache.scala | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 469adc94..08ce9c3e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -642,23 +642,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeData[T <: HasTileLinkData] (byteAddrBits: Int, dataBits: Int) - (buffer: Vec[UInt], beat: UInt, incoming: UInt) { + (beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = buffer(beat) // Newly Put data is in the buffer + val new_data = data_buffer(beat) // Newly Put data is in the buffer val amoOpSz = UInt(amoAluOperandBits) - val offset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> offset*amoOpSz - amoalu.io.rhs := new_data >> offset*amoOpSz + val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) + amoalu.io.lhs := old_data >> amoOffset*amoOpSz + amoalu.io.rhs := new_data >> amoOffset*amoOpSz val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && xact.isBuiltInType() // Only custom a_types have data for now val wmask = Fill(dataBits, valid_beat) & Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(offset)), + FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, xact.write_mask()), + FillInterleaved(8, write_mask_buffer(beat)), UInt(0, width = dataBits))) - buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << offset*amoOpSz, new_data) + data_buffer(beat) := ~wmask & old_data | wmask & + Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ @@ -832,7 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(data_buffer, io.irel().addr_beat, io.irel().data) + mergeDataInner(io.irel().addr_beat, io.irel().data) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -853,7 +853,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { - mergeDataOuter(data_buffer, io.ognt().addr_beat, io.ognt().data) + mergeDataOuter(io.ognt().addr_beat, io.ognt().data) ognt_had_data := Bool(true) } when(ognt_data_done) { @@ -880,13 +880,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { - mergeDataInternal(data_buffer, io.data.resp.bits.addr_beat, io.data.resp.bits.data) + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } when(resp_data_done) { state := Mux(xact.hasData(), s_data_write, s_inner_grant) From 3f070eee1f849ff3c273c77ce732f650587241df Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Mar 2015 23:41:56 -0700 Subject: [PATCH 326/688] first cut of merging puts/gets --- uncore/src/main/scala/cache.scala | 261 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 24 ++- 2 files changed, 170 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 08ce9c3e..a1defb3b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -171,6 +171,9 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = 4 + val enableGetMerging = false + val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -316,6 +319,10 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } +class L2SecondaryMissInfo extends TLBundle + with HasTileLinkBeatId + with HasClientTransactionId + class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -570,6 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } + class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO @@ -578,15 +586,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats) { - Reg(io.iacq().data.clone) - } - val amo_result = xact.data + val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_finish = Reg{ io.outer.finish.bits.clone } + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -594,6 +601,29 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val curr_probe_dst = PriorityEncoder(pending_probes) + val full_sharers = io.meta.resp.bits.meta.coh.inner.full() + val probe_self = xact.requiresSelfProbe() + val mask_self = Mux(probe_self, + full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), + full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val irel_had_data = Reg(init = Bool(false)) + val ognt_had_data = Reg(init = Bool(false)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) + val curr_write_beat = PriorityEncoder(pending_writes) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -610,29 +640,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dst = io.inner.grant.bits.header.dst), pending_coh.outer) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, xact.addr_beat) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read, xact.addr_beat, !xact.isSubBlockType()) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write, xact.addr_beat, !xact.isSubBlockType() || ognt_had_data || irel_had_data) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp, !xact.isSubBlockType()) - + val amo_result = xact.data val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() @@ -640,42 +648,62 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeData[T <: HasTileLinkData] - (byteAddrBits: Int, dataBits: Int) - (beat: UInt, incoming: UInt) { + def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { + data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is in the buffer - val amoOpSz = UInt(amoAluOperandBits) - val amoOffset = xact.addr_byte()(byteAddrBits-1, log2Up(amoAluOperandBits/8)) - amoalu.io.lhs := old_data >> amoOffset*amoOpSz - amoalu.io.rhs := new_data >> amoOffset*amoOpSz - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) && - xact.isBuiltInType() // Only custom a_types have data for now - val wmask = Fill(dataBits, valid_beat) & - Mux(xact.is(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amoOffset)), - Mux(xact.is(Acquire.putBlockType) || xact.is(Acquire.putType), - FillInterleaved(8, write_mask_buffer(beat)), - UInt(0, width = dataBits))) - data_buffer(beat) := ~wmask & old_data | wmask & - Mux(xact.is(Acquire.putAtomicType), amoalu.io.out << amoOffset*amoOpSz, new_data) + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.is(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(log2Up(rowBits/8), rowBits) _ - val mergeDataInner = mergeData(innerByteAddrBits, innerDataBits) _ - val mergeDataOuter = mergeData(outerByteAddrBits, outerDataBits) _ + val mergeDataInternal = mergeData(rowBits) _ + val mergeDataInner = mergeData(innerDataBits) _ + val mergeDataOuter = mergeData(outerDataBits) _ + + val can_merge_iacq_get = Bool(enableGetMerging) && + (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + (xact_src === io.inner.acquire.bits.header.src) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish).contains(state) && + do_allocate && + ignt_q.io.enq.ready + //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((Bool(enablePutMerging) && + (xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType))) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + (xact_src === io.inner.acquire.bits.header.src) && + (xact.client_xact_id === io.iacq().client_xact_id) && + xact.conflicts(io.iacq()) && + Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, + s_probe, s_outer_acquire, s_outer_grant, + s_outer_finish, s_data_read, + s_data_resp).contains(state) && + do_allocate && + ignt_q.io.enq.ready - //TODO: Allow hit under miss for stores val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && (state === s_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && + (state != s_idle) && + !io.has_acquire_match // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -697,21 +725,25 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := Bool(false) + io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( acq = xact, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_cnt, + addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_cnt))) + data_buffer(ignt_data_idx))) + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + ignt_q.io.deq.ready := ignt_data_done - io.inner.acquire.ready := Bool(false) + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) @@ -719,14 +751,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt + io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) + io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -744,39 +776,16 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") - - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } - switch (state) { is(s_idle) { - io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - data_buffer(io.iacq().addr_beat) := io.iacq().data - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + wmask_buffer.foreach { w => w := UInt(0) } + pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_writes := UInt(0) + pending_resps := UInt(0) irel_had_data := Bool(false) ognt_had_data := Bool(false) state := s_meta_read @@ -820,7 +829,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) @@ -830,9 +839,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: make sure cacq data is actually present before accpeting // release data to merge! when(io.irel().hasData()) { - irel_had_data := Bool(true) pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) + pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -840,11 +849,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, Mux(irel_had_data, s_data_write, s_data_read), s_outer_acquire) + state := Mux(is_hit, + Mux(pending_writes.orR, s_data_write, s_data_read), + s_outer_acquire) } } is(s_outer_acquire) { - io.outer.acquire.valid := !iacq_data_done // collect all data before refilling + io.outer.acquire.valid := Bool(true) when(oacq_data_done) { state := s_outer_grant } @@ -854,7 +865,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - ognt_had_data := Bool(true) + pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt @@ -865,7 +876,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(io.ognt().hasData(), s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } } @@ -874,31 +885,39 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(ognt_had_data, s_data_write, s_data_read)) + Mux(pending_writes.orR, s_data_write, s_data_read)) } } is(s_data_read) { - io.data.read.valid := !collect_iacq_data || iacq_data_valid(read_data_cnt) + io.data.read.valid := pending_reads.orR + when(io.data.read.ready) { + pending_resps := pending_resps | UIntToOH(curr_read_beat) + pending_reads := pending_reads & ~UIntToOH(curr_read_beat) + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } - when(read_data_done) { state := s_data_resp } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(resp_data_done) { - state := Mux(xact.hasData(), s_data_write, s_inner_grant) + pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) + when(PopCount(pending_resps) <= UInt(1)) { + state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + } } } is(s_data_write) { - io.data.write.valid := Bool(true) - when(write_data_done) { state := s_inner_grant } + io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + when(io.data.write.ready) { + pending_writes := pending_writes & ~UIntToOH(curr_write_beat) + when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } + } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) - when(ignt_data_done) { + when(ignt_data_done && ignt_q.io.count === UInt(1)) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -916,6 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.finish.valid) { state := s_idle } } } + + ignt_q.io.enq.valid := io.inner.acquire.fire() && + (state === s_idle || !xact.hasMultibeatData()) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + + // Handle Get and Put merging + when(io.inner.acquire.fire()) { + val beat = io.iacq().addr_beat + when(io.iacq().hasData()) { + mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) + //iacq_data_valid(beat) := Bool(true) + pending_writes(beat) := Bool(true) + } + pending_reads(beat) := Bool(true) + } + + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.header.src != xact_src), + "AcquireTracker accepted data beat from different network source than initial request.") + + //TODO: Assumes in-order network + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle @@ -953,7 +998,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_p_id = PriorityEncoder(pending_probes) + val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits @@ -989,7 +1034,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id + io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) @@ -1030,7 +1075,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) + pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back io.inner.release.ready := Bool(true) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bd218ad..80f090d7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,6 +38,7 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val amoAluOperandBits = params(AmoAluOperandBits) } abstract class TLBundle extends Bundle with TileLinkParameters @@ -101,13 +102,22 @@ class Acquire extends ClientToManagerChannel M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def write_mask(dummy: Int = 0) = union(tlWriteMaskBits, 1) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + FillInterleaved(8, union(tlWriteMaskBits, 1)), + UInt(0, width = tlDataBits))) + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper funcs - def is(t: UInt) = a_type === t + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) @@ -119,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = Bool(false) + def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( @@ -229,7 +239,7 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt = Acquire.fullWriteMask): Acquire = { + wmask: UInt = Acquire.fullWriteMask): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -237,7 +247,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(write_mask, Bool(true))) + union = Cat(wmask, Bool(true))) } } @@ -248,7 +258,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - write_mask: UInt): Acquire = { + wmask: UInt): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -256,7 +266,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(write_mask, (write_mask != Acquire.fullWriteMask))) + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( client_xact_id: UInt, From 6d565d22e3e991ecb6abdfa75b1d19053024dbb3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:11 -0700 Subject: [PATCH 327/688] clean up acquire alloc in hub --- uncore/src/main/scala/broadcast.scala | 24 ++++++++++++++---------- uncore/src/main/scala/cache.scala | 3 +-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c8014f45..2564140b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,25 +48,29 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + + val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) + alloc_arb.io.out.ready := Bool(true) + trackerAcquireIOs.zip(alloc_arb.io.in).foreach { + case(tracker, arb) => arb.valid := tracker.ready + } + val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} + val acquireMatchList = trackerList.map(_.io.has_acquire_match) val any_acquire_matches = acquireMatchList.reduce(_||_) - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} + val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - trackerAcquireIOs.zip(alloc_arb.io.in).zipWithIndex.foreach { - case((tracker, arb), i) => - arb.valid := tracker.ready + val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) + io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := arb.ready && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && - sdq_rdy && !block_acquires - alloc_arb.io.out.ready := io.inner.acquire.valid && sdq_rdy && !block_acquires // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a1defb3b..4af26399 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -376,8 +376,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) alloc_arb.io.out.ready := Bool(true) trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => - arb.valid := tracker.ready + case(tracker, arb) => arb.valid := tracker.ready } val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} From d9598d26f250ec6863ca8d1c82956287848b1e9b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:23:35 -0700 Subject: [PATCH 328/688] fix assert --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4af26399..953aa2b6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -958,6 +958,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { //TODO: Assumes in-order network assert(!(state === s_idle && io.inner.acquire.fire() && + !io.iacq().isSubBlockType() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") } From 23f8033df55c59a85422b4213ed2bebc24f72437 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 00:24:59 -0700 Subject: [PATCH 329/688] turn off self probes again --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 80f090d7..56dfa7db 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -129,7 +129,7 @@ class Acquire extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - def requiresSelfProbe(dummy: Int = 0) = isBuiltInType()//Bool(false) + def requiresSelfProbe(dummy: Int = 0) = Bool(false) def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( From 730a13abf27d9fc15902b23970097f90860acfd7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 01:17:44 -0700 Subject: [PATCH 330/688] pending read fix --- uncore/src/main/scala/cache.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 953aa2b6..a2e78d1b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -782,7 +782,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), SInt(0), SInt(-1)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) irel_had_data := Bool(false) @@ -947,9 +949,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) - pending_writes(beat) := Bool(true) + pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } - pending_reads(beat) := Bool(true) + when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } } assert(!(state != s_idle && io.inner.acquire.fire() && From dc8861409449c6e1560da3d6389ddc1d71cfbdef Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 02:10:30 -0700 Subject: [PATCH 331/688] overlapping read/resps in l2 fix --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a2e78d1b..72ef4a79 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -900,6 +900,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) } + when(io.data.read.ready && io.data.resp.valid) { + pending_resps := (pending_resps & + ~UIntToOH(io.data.resp.bits.addr_beat)) | + UIntToOH(curr_read_beat) + } } is(s_data_resp) { when(io.data.resp.valid) { From 5c2461c74339da1698fcb08d539c95d8be5b122b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 04:58:54 -0700 Subject: [PATCH 332/688] merge data wmask bugfix --- uncore/src/main/scala/cache.scala | 11 ++++++----- uncore/src/main/scala/tilelink.scala | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 72ef4a79..c52566c3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -648,7 +648,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.rhs := data_buffer.head // default def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - data_buffer(beat) := ~wmask & data_buffer(beat) | wmask & put_data + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) + wmask_buffer(beat) := wmask | wmask_buffer(beat) } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { @@ -656,10 +658,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.is(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & wmask_buffer(beat) + val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) + val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.is(Acquire.putAtomicType), + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } @@ -952,7 +954,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val beat = io.iacq().addr_beat when(io.iacq().hasData()) { mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) //iacq_data_valid(beat) := Bool(true) pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 56dfa7db..42c17ca4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -106,10 +106,11 @@ class Acquire extends ClientToManagerChannel def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - FillInterleaved(8, union(tlWriteMaskBits, 1)), - UInt(0, width = tlDataBits))) + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) From fc0ae81a977b096bd2b94d6b2b031f206341a13f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 15:04:30 -0700 Subject: [PATCH 333/688] added finish counter --- uncore/src/main/scala/cache.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c52566c3..d4dbd75f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -610,12 +610,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) val mask_incoherent = mask_self & ~io.incoherent.toBits - val irel_had_data = Reg(init = Bool(false)) - val ognt_had_data = Reg(init = Bool(false)) val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) + when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -789,8 +789,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := UInt(0) pending_resps := UInt(0) - irel_had_data := Bool(false) - ognt_had_data := Bool(false) + ifin_cnt := UInt(0) state := s_meta_read } } @@ -940,7 +939,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_inner_finish) { io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - UInt(1) + when(ifin_cnt <= UInt(1)) { state := s_idle } + } } } From a52a729ab90bac736e6f5235518b475be26bb18a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 15:54:21 -0700 Subject: [PATCH 334/688] bugfix wmask handling --- uncore/src/main/scala/cache.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d4dbd75f..ed8d7051 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -658,13 +658,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val new_data = data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() - val valid_beat = (xact.isBuiltInType(Acquire.putBlockType) || xact.addr_beat === beat) - val wmask = Fill(dataBits, valid_beat) & FillInterleaved(8, wmask_buffer(beat)) + val wmask = FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) - when(xact.is(Acquire.putAtomicType) && valid_beat) { amo_result := old_data } + when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ val mergeDataInner = mergeData(innerDataBits) _ From 4fd01d82b86fffdca3364e1c47546311d66ef8e8 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 17:48:02 -0700 Subject: [PATCH 335/688] don't block finish messages when grant'ing --- uncore/src/main/scala/cache.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed8d7051..cd68a8df 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -929,6 +929,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(meta_dirty, s_meta_write, Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { + ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) + } } is(s_meta_write) { io.meta.write.valid := Bool(true) @@ -942,6 +946,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } + when(ifin_cnt === UInt(0)) { state := s_idle } } } From b08dced37cbe61589355c8505a4403544d35f311 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 17:51:00 -0700 Subject: [PATCH 336/688] first cut at pending scoreboarding --- uncore/src/main/scala/cache.scala | 90 +++++++++++++++++------------- uncore/src/main/scala/uncore.scala | 7 +++ 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index cd68a8df..0680442d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -446,17 +446,28 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) } else { (UInt(0), inc) } } + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( in: DecoupledIO[T], beat: UInt = UInt(0), full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.fire(), in.bits, beat, full_block) } + def connectInternalDataBeatCounter[T <: HasL2Data]( in: ValidIO[T], full_block: Bool = Bool(true)) = { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } + + def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + + def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -591,8 +602,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_finish = Reg{ io.outer.finish.bits.clone } - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -611,18 +620,38 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mask_incoherent = mask_self & ~io.incoherent.toBits val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ofin = Reg{ io.outer.finish.bits.clone } + + val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + ignt_q.io.enq.valid := Bool(false) + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + ignt_q.io.deq.ready := ignt_data_done + val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_reads := pending_reads | + addPendingBit(io.inner.acquire) | + dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) + + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) + pending_writes := pending_writes | + addPendingBit(io.inner.acquire) | + addPendingBit(io.inner.release) | + addPendingBit(io.outer.grant) & + dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + pending_resps := pending_resps | + addInternalPendingBit(io.data.read) & + dropInternalPendingBit(io.data.resp) + val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) @@ -720,8 +749,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) io.outer.finish.valid := Bool(false) - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -739,7 +768,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amo_result, data_buffer(ignt_data_idx))) io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id - ignt_q.io.deq.ready := ignt_data_done io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || @@ -789,6 +817,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_writes := UInt(0) pending_resps := UInt(0) ifin_cnt := UInt(0) + ignt_q.io.enq.valid := Bool(true) state := s_meta_read } } @@ -842,7 +871,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) - pending_writes := pending_writes | UIntToOH(io.irel().addr_beat) } // We don't decrement release_count until we've received all the data beats. when(!io.irel().hasMultibeatData() || irel_data_done) { @@ -866,14 +894,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - pending_writes := pending_writes | UIntToOH(io.ognt().addr_beat) } when(ognt_data_done) { pending_coh := pending_coh_on_ognt when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src - pending_finish.header.src := UInt(bankId) + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src + pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, @@ -892,18 +919,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_read) { io.data.read.valid := pending_reads.orR when(io.data.read.ready) { - pending_resps := pending_resps | UIntToOH(curr_read_beat) - pending_reads := pending_reads & ~UIntToOH(curr_read_beat) when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - } - when(io.data.read.ready && io.data.resp.valid) { - pending_resps := (pending_resps & - ~UIntToOH(io.data.resp.bits.addr_beat)) | - UIntToOH(curr_read_beat) } } is(s_data_resp) { @@ -918,7 +937,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_data_write) { io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present when(io.data.write.ready) { - pending_writes := pending_writes & ~UIntToOH(curr_write_beat) when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } @@ -950,20 +968,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } - ignt_q.io.enq.valid := io.inner.acquire.fire() && - (state === s_idle || !xact.hasMultibeatData()) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat // Handle Get and Put merging - when(io.inner.acquire.fire()) { - val beat = io.iacq().addr_beat - when(io.iacq().hasData()) { - mergeDataPut(beat, io.iacq().wmask(), io.iacq().data) - //iacq_data_valid(beat) := Bool(true) - pending_writes := pending_writes | UIntToOH(io.iacq().addr_beat) - } - when(state != s_idle) { pending_reads := pending_reads | UIntToOH(io.iacq().addr_beat) } + when(io.inner.acquire.fire() && io.iacq().hasData()) { + mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && @@ -1007,7 +1017,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_finish = Reg{ io.outer.finish.bits.clone } + val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -1043,8 +1053,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.release.bits.header.src := UInt(bankId) io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_finish - val pending_finish_on_ognt = io.ognt().makeFinish() + io.outer.finish.bits := pending_ofin + val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1133,8 +1143,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().requiresAck()) { - pending_finish.payload := pending_finish_on_ognt - pending_finish.header.dst := io.outer.grant.bits.header.src + pending_ofin.payload := pending_ofin_on_ognt + pending_ofin.header.dst := io.outer.grant.bits.header.src state := s_outer_finish }.otherwise { state := s_wb_resp diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 0d414f0d..16e976ad 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -139,12 +139,19 @@ abstract class XactTracker extends CoherenceAgentModule { val done = Mux(multi, multi_done, inc) (cnt, done) } + def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], beat: UInt = UInt(0)) = { connectDataBeatCounter(in.fire(), in.bits.payload, beat) } + def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } + + def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat)) + } } From 504eedbf893404b3ed3ee1caf7dcec82161d492f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:07:52 -0700 Subject: [PATCH 337/688] fixes in in bit manipulation --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0680442d..e3783d93 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -464,10 +464,10 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = - Fill(in.bits.refillCycles, in.fire()) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = - Fill(in.bits.refillCycles, in.valid) & ~UIntToOH(in.bits.addr_beat) + ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -634,22 +634,22 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := pending_reads | - addPendingBit(io.inner.acquire) | + pending_reads := (pending_reads | + addPendingBit(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := pending_writes | + pending_writes := (pending_writes | addPendingBit(io.inner.acquire) | addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant) & + addPendingBit(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := pending_resps | - addInternalPendingBit(io.data.read) & + pending_resps := (pending_resps | + addInternalPendingBit(io.data.read)) & dropInternalPendingBit(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( From 2d7375760dd6e5c89ef019a6809833ff1157f39a Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 18:35:04 -0700 Subject: [PATCH 338/688] set pending_writes for puts --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e3783d93..27c2e64f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -814,7 +814,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux(io.iacq().isSubBlockType(), UIntToOH(io.iacq().addr_beat), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := UInt(0) + pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) From d14efce0b4a6eaea52bc6d83f6cd89839a547e53 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 19:54:11 -0700 Subject: [PATCH 339/688] fix wmask_buffer initialization --- uncore/src/main/scala/cache.scala | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 27c2e64f..9dc35500 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -676,12 +676,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { amoalu.io.lhs := io.data.resp.bits.data //default amoalu.io.rhs := data_buffer.head // default - def mergeDataPut(beat: UInt, wmask: UInt, put_data: UInt) { - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & put_data) - wmask_buffer(beat) := wmask | wmask_buffer(beat) - } - def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data val new_data = data_buffer(beat) // Newly Put data is already in the buffer @@ -971,7 +965,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { - mergeDataPut(io.iacq().addr_beat, io.iacq().wmask(), io.iacq().data) + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } //iacq_data_valid(beat) := Bool(true) } From 9de5161d7ab8d351a3e86cbed7d8ad7faaac7b25 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 20:24:04 -0700 Subject: [PATCH 340/688] guard all writes to data ram with masks --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9dc35500..7e14c8f4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -686,6 +686,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) + wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } val mergeDataInternal = mergeData(rowBits) _ @@ -779,7 +780,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) io.meta.read.valid := Bool(false) io.meta.read.bits.id := UInt(trackerId) From 2d3f947a9c7247317aa9db5b4be30b9e2abe6e86 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:33:05 -0700 Subject: [PATCH 341/688] cleaned up finish counter --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7e14c8f4..c7d857d3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -632,7 +632,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + UInt(1) } + when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } + .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBit(io.inner.acquire)) & @@ -943,12 +945,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) } io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - Mux(ignt_data_done, UInt(0), UInt(1)) - } } is(s_meta_write) { io.meta.write.valid := Bool(true) + io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -956,14 +956,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_finish) { io.inner.finish.ready := Bool(true) when(io.inner.finish.valid) { - ifin_cnt := ifin_cnt - UInt(1) when(ifin_cnt <= UInt(1)) { state := s_idle } } when(ifin_cnt === UInt(0)) { state := s_idle } } } - // Handle Get and Put merging when(io.inner.acquire.fire() && io.iacq().hasData()) { val beat = io.iacq().addr_beat From b9591b297c94106de1b5bf4560a68a7b95e7751b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 19:59:22 -0700 Subject: [PATCH 342/688] added s_wait_puts to L2AcquireTracker --- uncore/src/main/scala/cache.scala | 33 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c7d857d3..70411a14 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -591,7 +591,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 15) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -602,12 +602,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_more_put_data = !pending_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -808,9 +811,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), - SInt(-1, width = innerDataBeats)).toUInt + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isSubBlockType(), + UIntToOH(io.iacq().addr_beat), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBit(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) @@ -876,7 +882,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } when(release_count === UInt(0)) { state := Mux(is_hit, - Mux(pending_writes.orR, s_data_write, s_data_read), + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read), s_outer_acquire) } } @@ -901,7 +909,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_outer_finish }.otherwise { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } } @@ -910,7 +920,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, s_data_write, s_data_read)) + Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_data_read)) } } is(s_data_read) { @@ -927,12 +939,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, s_data_write, s_inner_grant) + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) } } } + is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR //TODO make sure all acquire data is present + io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } From 638bace858b78f7a8bae7bccf6398ead12203d5f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:28:06 -0700 Subject: [PATCH 343/688] avoid reading data when write mask is full --- uncore/src/main/scala/cache.scala | 19 ++++++++++++------- uncore/src/main/scala/uncore.scala | 7 ++++++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 70411a14..b193fc8a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -603,7 +603,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBit(io.inner.acquire)) + pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) val do_allocate = xact.allocate() @@ -640,15 +640,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBit(io.inner.acquire)) & + addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & dropPendingBit(io.data.read) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) pending_writes := (pending_writes | - addPendingBit(io.inner.acquire) | - addPendingBit(io.inner.release) | - addPendingBit(io.outer.grant)) & + addPendingBitWhenHasData(io.inner.acquire) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant)) & dropPendingBit(io.data.write) val curr_write_beat = PriorityEncoder(pending_writes) @@ -815,9 +815,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { UInt(0), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), - UIntToOH(io.iacq().addr_beat), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_writes := addPendingBit(io.inner.acquire) + pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) @@ -933,6 +933,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } + when(PopCount(pending_reads) === UInt(0)) { + state := Mux(pending_writes.orR, + Mux(needs_more_put_data, s_wait_puts, s_data_write), + s_inner_grant) + } } is(s_data_resp) { when(io.data.resp.valid) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 16e976ad..14414b76 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -150,8 +150,13 @@ abstract class XactTracker extends CoherenceAgentModule { connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 } - def addPendingBit[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & UIntToOH(in.bits.payload.addr_beat)) } + + def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat)) + } } From d48775eecb3c7bc36b10959e9fdda6c8562483a1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:31:23 -0700 Subject: [PATCH 344/688] cleanup outdated comments --- uncore/src/main/scala/cache.scala | 9 --------- 1 file changed, 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193fc8a..8bcf3885 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -869,8 +869,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data - //TODO: make sure cacq data is actually present before accpeting - // release data to merge! when(io.irel().hasData()) { pending_coh.outer := pending_ocoh_on_irel mergeDataInner(io.irel().addr_beat, io.irel().data) @@ -990,18 +988,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - //iacq_data_valid(beat) := Bool(true) } assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") - - //TODO: Assumes in-order network - assert(!(state === s_idle && io.inner.acquire.fire() && - !io.iacq().isSubBlockType() && - io.iacq().addr_beat != UInt(0)), - "AcquireTracker initialized with a tail data beat.") } class L2WritebackReq extends L2HellaCacheBundle From 0e4cf74d8ab6c3fedebc0afae80404bf9978267b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 20:53:27 -0700 Subject: [PATCH 345/688] always merge Puts --- uncore/src/main/scala/cache.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8bcf3885..455f85ef 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -173,7 +173,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 val enableGetMerging = false - val enablePutMerging = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -709,11 +708,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { do_allocate && ignt_q.io.enq.ready //TODO: mix Puts and PutBlocks - val can_merge_iacq_put = ((Bool(enablePutMerging) && - (xact.isBuiltInType(Acquire.putType) && - io.iacq().isBuiltInType(Acquire.putType))) || - (xact.isBuiltInType(Acquire.putBlockType) && - io.iacq().isBuiltInType(Acquire.putBlockType))) && + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType)) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && (xact_src === io.inner.acquire.bits.header.src) && (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && From f4f59464df9bce2c8ccec3a9b7a9a698fa53ae7f Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 21:44:20 -0700 Subject: [PATCH 346/688] fix pending_puts initialization --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..514878e4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -810,7 +810,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), + addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt pending_reads := Mux(io.iacq().isSubBlockType(), addPendingBitWhenWmaskIsNotFull(io.inner.acquire), From aa5435800d7e7f757752817229fc8fd956d1c1f3 Mon Sep 17 00:00:00 2001 From: Yunsup Lee Date: Tue, 17 Mar 2015 22:43:00 -0700 Subject: [PATCH 347/688] fix get merging, and always turn it on --- uncore/src/main/scala/cache.scala | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 514878e4..27bb898b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -172,7 +172,6 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = 4 - val enableGetMerging = false } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -697,8 +696,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val mergeDataInner = mergeData(innerDataBits) _ val mergeDataOuter = mergeData(outerDataBits) _ - val can_merge_iacq_get = Bool(enableGetMerging) && - (xact.isBuiltInType(Acquire.getType) && + val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && @@ -979,13 +977,15 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } // Handle Get and Put merging - when(io.inner.acquire.fire() && io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - when(!xact.hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } + when(io.inner.acquire.fire()) { + when (io.iacq().hasData()) { + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) + } + when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } } assert(!(state != s_idle && io.inner.acquire.fire() && From 825c4b28509a81ad1d17e0df80c56d5384130f80 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:44:53 -0700 Subject: [PATCH 348/688] make ignts more eager --- uncore/src/main/scala/cache.scala | 52 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 455f85ef..39b5e7c2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -459,13 +459,16 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } - def addInternalPendingBit[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + def addPendingBitInternal[T <: HasL2BeatAddr](in: DecoupledIO[T]) = Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + def addPendingBitInternal[T <: HasL2BeatAddr](in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & UIntToOH(in.bits.addr_beat) + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) - def dropInternalPendingBit[T <: HasL2BeatAddr] (in: ValidIO[T]) = + def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) } @@ -626,6 +629,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) val pending_ofin = Reg{ io.outer.finish.bits.clone } + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + pending_ignt_data := pending_ignt_data | + addPendingBitInternal(io.data.resp) | + addPendingBitWhenHasData(io.inner.release) | + addPendingBitWhenHasData(io.outer.grant) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) @@ -634,8 +642,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { ifin_cnt := ifin_cnt + Mux(io.inner.finish.fire(), UInt(0), UInt(1)) } - .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + when(ignt_data_done) { + ifin_cnt := Mux(io.inner.finish.fire(), + Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), + Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) + } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | @@ -653,8 +664,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) pending_resps := (pending_resps | - addInternalPendingBit(io.data.read)) & - dropInternalPendingBit(io.data.resp) + addPendingBitInternal(io.data.read)) & + dropPendingBitInternal(io.data.resp) val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, @@ -755,7 +766,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - io.inner.grant.valid := state === s_inner_grant && ignt_q.io.deq.valid + io.inner.grant.valid := state === s_inner_grant && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) io.inner.grant.bits.header.src := UInt(bankId) io.inner.grant.bits.header.dst := xact_src io.inner.grant.bits.payload := pending_coh.inner.makeGrant( @@ -770,8 +784,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) + io.inner.release.ready := state === s_probe + io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, + s_data_write, s_wait_puts, s_data_resp).contains(state) io.data.read.valid := Bool(false) io.data.read.bits.id := UInt(trackerId) @@ -817,6 +832,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) + pending_ignt_data := UInt(0) ifin_cnt := UInt(0) ignt_q.io.enq.valid := Bool(true) state := s_meta_read @@ -863,7 +879,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) when(io.inner.release.valid) { pending_coh.inner := pending_icoh_on_irel // Handle released dirty data @@ -954,27 +969,26 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_grant) { - when(ignt_data_done && ignt_q.io.count === UInt(1)) { + when(ignt_q.io.count === UInt(0) || + (ignt_data_done && ignt_q.io.count === UInt(1))) { val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, - Mux(io.ignt().requiresAck(), s_inner_finish, s_idle)) + Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), + s_inner_finish, s_idle)) } - io.inner.finish.ready := Bool(true) } is(s_meta_write) { io.meta.write.valid := Bool(true) - io.inner.finish.ready := Bool(true) when(io.meta.write.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) + state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { - when(ifin_cnt <= UInt(1)) { state := s_idle } + when(ifin_cnt === UInt(0) || + (io.inner.finish.valid && ifin_cnt === UInt(1))) { + state := s_idle } - when(ifin_cnt === UInt(0)) { state := s_idle } } } From 42aa4aa8cab91fad1a5220108f16a47e63483f61 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 17 Mar 2015 22:53:50 -0700 Subject: [PATCH 349/688] Secondary miss param --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b632bbaa..98e122d4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -10,6 +10,8 @@ case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object NPrimaryMisses extends Field[Int] +case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] @@ -171,7 +173,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen val amoAluOperandBits = params(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states - val nSecondaryMisses = 4 + val nSecondaryMisses = params(NSecondaryMisses) } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters From e325399c8781181b9d030d7bb4f8d85c87d0ea54 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 12:49:53 -0700 Subject: [PATCH 350/688] Re-split mem resp tag and data queues --- uncore/src/main/scala/memserdes.scala | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index a5bea3a3..e2d6df85 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -544,11 +544,21 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul io.mem.req_data <> io.cpu.req_data // Have separate queues to allow for different mem implementations - val resp_data_q = Module((new HellaQueue(numEntries)) { new MemResp }) - resp_data_q.io.enq <> io.mem.resp - io.cpu.resp <> resp_data_q.io.deq + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemData }) + resp_data_q.io.enq.valid := io.mem.resp.valid + resp_data_q.io.enq.bits.data := io.mem.resp.bits.data - inc := resp_data_q.io.deq.fire() + val resp_tag_q = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tag_q.io.enq.valid := io.mem.resp.valid + resp_tag_q.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_data_q.io.deq.valid && resp_tag_q.io.deq.valid + io.cpu.resp.bits.data := resp_data_q.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tag_q.io.deq.bits.tag + resp_data_q.io.deq.ready := io.cpu.resp.ready + resp_tag_q.io.deq.ready := io.cpu.resp.ready + + inc := resp_data_q.io.deq.fire() && resp_tag_q.io.deq.fire() dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } From 1ff184bf62ea41370a0ee083ee69257dae3dd86f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 17:55:05 -0700 Subject: [PATCH 351/688] first cut at optimized state transitions --- uncore/src/main/scala/cache.scala | 169 ++++++++++++++--------------- uncore/src/main/scala/uncore.scala | 19 +++- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 98e122d4..85fedbbc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -174,6 +174,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) + val isLastLevelCache = true } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -594,7 +595,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) val state = Reg(init=s_idle) val xact_src = Reg(io.inner.acquire.bits.header.src.clone) @@ -605,15 +606,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts | addPendingBitWhenHasData(io.inner.acquire)) + val present_puts = Reg(init=Bits(0, width = innerDataBeats)) + present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()) + val is_hit = xact_tag_match && + (xact_meta.coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + xact_meta.coh.outer.isValid())) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) - val needs_more_put_data = !pending_puts.andR + val needs_more_put_data = !present_puts.andR val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -652,7 +656,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & - dropPendingBit(io.data.read) + (dropPendingBit(io.data.read) & + dropPendingBitWhenWmaskIsFull(io.inner.acquire) & + dropPendingBitWhenHasData(io.inner.release) & + dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) @@ -683,6 +690,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), pending_coh.outer) + val pending_ofin_on_ognt = io.ognt().makeFinish() val amo_result = xact.data val amoalu = Module(new AMOALU) @@ -714,11 +722,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact_src === io.inner.acquire.bits.header.src) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish).contains(state) && do_allocate && ignt_q.io.enq.ready - //TODO: mix Puts and PutBlocks + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && @@ -727,7 +735,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.client_xact_id === io.iacq().client_xact_id) && xact.conflicts(io.iacq()) && Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_probe, s_outer_acquire, s_outer_grant, + s_inner_probe, s_outer_acquire, s_outer_grant, s_outer_finish, s_data_read, s_data_resp).contains(state) && do_allocate && @@ -737,7 +745,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().addr_block(idxMSB,idxLSB) io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && - (state === s_probe) + (state === s_inner_probe) io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && (state != s_idle) && @@ -746,7 +754,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := Bool(false) + io.outer.acquire.valid := state === s_outer_acquire io.outer.acquire.bits.payload := Mux(do_allocate, xact_meta.coh.outer.makeAcquire( client_xact_id = UInt(trackerId), @@ -756,12 +764,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.bits.header.src := UInt(bankId) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) + io.outer.grant.ready := state === s_outer_grant + io.outer.finish.valid := state === s_outer_finish io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() - io.inner.probe.valid := Bool(false) + io.inner.probe.valid := state === s_inner_probe && pending_probes.orR io.inner.probe.bits.header.src := UInt(bankId) io.inner.probe.bits.header.dst := curr_probe_dst io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) @@ -784,34 +791,34 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_probe + io.inner.release.ready := state === s_inner_probe io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := Bool(false) + io.data.read.valid := state === s_data_read && pending_reads.orR io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := Bool(false) + io.data.write.valid := state === s_data_write && pending_writes.orR io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := Bool(false) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := Bool(false) + io.wb.req.valid := state === s_wb_req io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) io.wb.req.bits.coh := xact_meta.coh io.wb.req.bits.way_en := xact_way_en @@ -824,12 +831,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), addPendingBitWhenHasData(io.inner.acquire), SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + UInt(0), + Mux(io.iacq().isSubBlockType(), + addPendingBitWhenWmaskIsNotFull(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt) pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) @@ -838,10 +847,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := s_meta_read } } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } + is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match @@ -850,31 +856,33 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && _coh.outer.isHit(xact.op_code()) + val _is_hit = _tag_match && + (_coh.outer.isHit(xact.op_code()) || + (Bool(isLastLevelCache) && // LLC has all the permissions + _coh.outer.isValid())) + val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_probes = _tag_match && _coh.inner.requiresProbes(xact) + val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_probes) { + when(_needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(_tag_match, - Mux(_needs_probes, s_probe, Mux(_is_hit, s_data_read, s_outer_acquire)), // Probe, hit or upgrade - Mux(_needs_writeback, s_wb_req, s_outer_acquire)) // Evict ifneedbe + state := Mux(!_tag_match, + Mux(_needs_writeback, s_wb_req, s_outer_acquire), + Mux(_needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } - is(s_wb_req) { - io.wb.req.valid := Bool(true) - when(io.wb.req.ready) { state := s_wb_resp } - } - is(s_wb_resp) { - when(io.wb.resp.valid) { state := s_outer_acquire } - } - is(s_probe) { + is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } + is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_inner_probe) { // Send probes - io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) } @@ -892,21 +900,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(is_hit, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read), - s_outer_acquire) - } - } - is(s_outer_acquire) { - io.outer.acquire.valid := Bool(true) - when(oacq_data_done) { - state := s_outer_grant + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) } } + is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } is(s_outer_grant) { - io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { when(io.ognt().hasData()) { mergeDataOuter(io.ognt().addr_beat, io.ognt().data) @@ -919,51 +920,40 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ofin.header.src := UInt(bankId) state := s_outer_finish }.otherwise { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } } } is(s_outer_finish) { - io.outer.finish.valid := Bool(true) when(io.outer.finish.ready) { - state := Mux(!do_allocate, s_inner_grant, - Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_data_read)) + state := Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write))) } } is(s_data_read) { - io.data.read.valid := pending_reads.orR - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } - } when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) } - when(PopCount(pending_reads) === UInt(0)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) + when(io.data.read.ready) { + when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } } } is(s_data_resp) { when(io.data.resp.valid) { mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - pending_resps := pending_resps & ~UIntToOH(io.data.resp.bits.addr_beat) - when(PopCount(pending_resps) <= UInt(1)) { - state := Mux(pending_writes.orR, - Mux(needs_more_put_data, s_wait_puts, s_data_write), - s_inner_grant) - } + } + when(PopCount(pending_resps) === UInt(0) || + (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { - io.data.write.valid := pending_writes.orR when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } @@ -978,15 +968,14 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { s_inner_finish, s_idle)) } } - is(s_meta_write) { - io.meta.write.valid := Bool(true) + is(s_meta_write) { when(io.meta.write.ready) { state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) } } is(s_inner_finish) { when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + (io.inner.finish.valid && ifin_cnt === UInt(1))) { state := s_idle } } @@ -1031,7 +1020,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1057,12 +1046,13 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { incoming = io.irel(), src = io.inner.release.bits.header.src) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write + val pending_ofin_on_ognt = io.ognt().makeFinish() io.has_acquire_conflict := Bool(false) io.has_acquire_match := Bool(false) io.has_release_match := !io.irel().isVoluntary() && io.irel().conflicts(xact_addr_block) && - (state === s_probe) + (state === s_inner_probe) io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) @@ -1076,7 +1066,6 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.outer.finish.valid := Bool(false) // default io.outer.finish.bits := pending_ofin - val pending_ofin_on_ognt = io.ognt().makeFinish() io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1109,15 +1098,15 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_way_en := io.wb.req.bits.way_en xact_id := io.wb.req.bits.id irel_had_data := Bool(false) - val needs_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_probes) { + val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(needs_probes, s_probe, s_data_read) + state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } } - is(s_probe) { + is(s_inner_probe) { // Send probes io.inner.probe.valid := pending_probes != UInt(0) when(io.inner.probe.ready) { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 14414b76..cfd33567 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -151,12 +151,23 @@ abstract class XactTracker extends CoherenceAgentModule { } def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & + UIntToOH(in.bits.payload.addr_beat) } + def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | + ~UIntToOH(in.bits.payload.addr_beat) + } + + //TODO | with existing wmask_buffer? def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - (Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat)) + Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & + UIntToOH(in.bits.payload.addr_beat) + } + + def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | + ~UIntToOH(in.bits.payload.addr_beat) } } From 19059bf0ebc0e3e2f2c6bfb746ff19a04208e6dd Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:28:03 -0700 Subject: [PATCH 352/688] put data can be used for ignts --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 85fedbbc..7257a699 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -638,7 +638,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_ignt_data := pending_ignt_data | addPendingBitInternal(io.data.resp) | addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) + addPendingBitWhenHasData(io.outer.grant) | + addPendingBitWhenHasData(io.inner.acquire) val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) ignt_q.io.enq.valid := Bool(false) From fb8071c12d90b20803c933f7a5a416855691cc44 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 18:49:32 -0700 Subject: [PATCH 353/688] generous hit detection on PutBlocks --- uncore/src/main/scala/cache.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7257a699..741e168d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -609,10 +609,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val is_hit = xact_tag_match && - (xact_meta.coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - xact_meta.coh.outer.isValid())) + val is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) val do_allocate = xact.allocate() val needs_writeback = !xact_tag_match && do_allocate && (xact_meta.coh.outer.requiresVoluntaryWriteback() || @@ -857,10 +857,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh := io.meta.resp.bits.meta.coh val _coh = io.meta.resp.bits.meta.coh val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = _tag_match && - (_coh.outer.isHit(xact.op_code()) || - (Bool(isLastLevelCache) && // LLC has all the permissions - _coh.outer.isValid())) + val _is_hit = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + _coh.outer.isValid()) + else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || From b92ea608914be20519f57b3d1189c27cd794c85c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 19:32:46 -0700 Subject: [PATCH 354/688] you can 'hit' with putblocks even when the tag doesn't match but you still better writeback --- uncore/src/main/scala/cache.scala | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 741e168d..146250bc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -859,7 +859,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val _tag_match = io.meta.resp.bits.tag_match val _is_hit = (if(isLastLevelCache) (xact.isBuiltInType(Acquire.putBlockType) || - _coh.outer.isValid()) + _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) val _needs_writeback = !_tag_match && do_allocate && @@ -871,17 +871,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_probes := mask_incoherent release_count := PopCount(mask_incoherent) } - state := Mux(!_tag_match, - Mux(_needs_writeback, s_wb_req, s_outer_acquire), + state := Mux(_needs_writeback, s_wb_req, Mux(_needs_inner_probes, s_inner_probe, - Mux(!is_hit, s_outer_acquire, + Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { when(io.wb.resp.valid) { state := s_outer_acquire } } + is(s_wb_resp) { + when(io.wb.resp.valid) { + state := Mux(!is_hit, s_outer_acquire, + Mux(pending_reads.orR, s_data_read, + Mux(!pending_writes.orR, s_inner_grant, + Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + } + } is(s_inner_probe) { // Send probes when(io.inner.probe.ready) { From 002851f836b350b34e90aa2fb6eda44b5b18af7b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 21:11:40 -0700 Subject: [PATCH 355/688] disentangle is_hit logic --- uncore/src/main/scala/cache.scala | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 146250bc..55fc544d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,16 +608,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val present_puts = Reg(init=Bits(0, width = innerDataBeats)) present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - - val is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else (xact_tag_match && xact_meta.coh.outer.isHit(xact.op_code()))) - val do_allocate = xact.allocate() - val needs_writeback = !xact_tag_match && do_allocate && - (xact_meta.coh.outer.requiresVoluntaryWriteback() || - xact_meta.coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_more_put_data = !present_puts.andR + val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) @@ -861,7 +853,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { (xact.isBuiltInType(Acquire.putBlockType) || _tag_match && _coh.outer.isValid()) else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && (_coh.outer.requiresVoluntaryWriteback() || _coh.inner.requiresProbesOnVoluntaryWriteback()) @@ -881,11 +872,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } is(s_wb_resp) { - when(io.wb.resp.valid) { - state := Mux(!is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + when(io.wb.resp.valid) { + val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!_skip_outer_acquire, s_outer_acquire, + Mux(needs_more_put_data, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -907,7 +897,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } when(release_count === UInt(0)) { - state := Mux(!is_hit, s_outer_acquire, + val _skip_outer_acquire = (if(isLastLevelCache) + (xact.isBuiltInType(Acquire.putBlockType) || + xact_meta.coh.outer.isValid()) + else xact_meta.coh.outer.isHit(xact.op_code())) + state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, Mux(needs_more_put_data, s_wait_puts, s_data_write)))) @@ -968,7 +962,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_inner_grant) { when(ignt_q.io.count === UInt(0) || (ignt_data_done && ignt_q.io.count === UInt(1))) { - val meta_dirty = !is_hit || pending_coh_on_ignt != xact_meta.coh + val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), From 004ad11af6fa5636ac046871284347a156a72444 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:14:41 -0700 Subject: [PATCH 356/688] cleanup pending signals --- uncore/src/main/scala/cache.scala | 23 ++++++++++------------- uncore/src/main/scala/uncore.scala | 14 +++++--------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 55fc544d..ed10e3c5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -606,9 +606,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val pending_coh = Reg{ xact_meta.coh.clone } - val present_puts = Reg(init=Bits(0, width = innerDataBeats)) - present_puts := (present_puts | addPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = !present_puts.andR + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) + val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -648,9 +648,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) pending_reads := (pending_reads | - addPendingBitWhenWmaskIsNotFull(io.inner.acquire)) & + addPendingBitWhenGetOrAtomic(io.inner.acquire)) & (dropPendingBit(io.data.read) & - dropPendingBitWhenWmaskIsFull(io.inner.acquire) & dropPendingBitWhenHasData(io.inner.release) & dropPendingBitWhenHasData(io.outer.grant)) val curr_read_beat = PriorityEncoder(pending_reads) @@ -824,14 +823,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact := io.iacq() xact.data := UInt(0) wmask_buffer.foreach { w => w := UInt(0) } - present_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - addPendingBitWhenHasData(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - UInt(0), - Mux(io.iacq().isSubBlockType(), - addPendingBitWhenWmaskIsNotFull(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt) + pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux(io.iacq().isBuiltInType(), + addPendingBitWhenGetOrAtomic(io.inner.acquire), + SInt(-1, width = innerDataBeats)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cfd33567..de55c0e2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -160,14 +160,10 @@ abstract class XactTracker extends CoherenceAgentModule { ~UIntToOH(in.bits.payload.addr_beat) } - //TODO | with existing wmask_buffer? - def addPendingBitWhenWmaskIsNotFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && !in.bits.payload.wmask().andR) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenWmaskIsFull(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.wmask().andR) | - ~UIntToOH(in.bits.payload.addr_beat) + def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + val a = in.bits.payload + Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && + (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & + UIntToOH(a.addr_beat) } } From 3cf033180f2980a34b07016a4197d949e57a9b5d Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 18 Mar 2015 22:41:09 -0700 Subject: [PATCH 357/688] pending read fix --- uncore/src/main/scala/cache.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ed10e3c5..5f656f31 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -826,9 +826,10 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(), - addPendingBitWhenGetOrAtomic(io.inner.acquire), - SInt(-1, width = innerDataBeats)).toUInt + pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || + !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt pending_writes := addPendingBitWhenHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) From 4176edaa34956cc1ce08d108d790aebf267bdd1f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 10:17:51 -0700 Subject: [PATCH 358/688] clean up tracker allocation --- uncore/src/main/scala/broadcast.scala | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 2564140b..64a96cf3 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,28 +48,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires && sdq_rdy + val block_acquires = acquireConflicts.orR || !sdq_rdy + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + tracker.valid := io.inner.acquire.valid && + !block_acquires && + (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -80,17 +76,21 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data - val release_idx = Vec(trackerList.map(_.io.has_release_match)).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles From b7af610569ce10c952519359f63cbb10509c2b43 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 11:29:04 -0700 Subject: [PATCH 359/688] broadcast hub bugfix --- uncore/src/main/scala/broadcast.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 64a96cf3..ce72f625 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -229,9 +229,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) From 8959b2e81a0ae6fb198a37a72ebc25d19af4b1ab Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 26 Mar 2015 13:29:52 -0700 Subject: [PATCH 360/688] TileLinkEnqueuer --- uncore/src/main/scala/tilelink.scala | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 42c17ca4..0aff9aab 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -505,6 +505,29 @@ object HeaderlessTileLinkIOWrapper { } } +class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) + io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) + io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) + io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) + io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths))(p) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { + apply(in, (depth, depth, depth, depth, depth))(p) + } +} + abstract trait HasArbiterTypes { val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId From c941f0a68ec202f96bade0bcf1a48ec8151785d5 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 27 Mar 2015 16:21:29 -0700 Subject: [PATCH 361/688] New virtual memory implementation (Sv39) --- uncore/src/main/scala/memserdes.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e2d6df85..059fbdcf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -7,8 +7,9 @@ import scala.math._ case object PAddrBits extends Field[Int] case object VAddrBits extends Field[Int] case object PgIdxBits extends Field[Int] +case object PgLevels extends Field[Int] +case object PgLevelBits extends Field[Int] case object ASIdBits extends Field[Int] -case object PermBits extends Field[Int] case object PPNBits extends Field[Int] case object VPNBits extends Field[Int] From ced627f00a94ac772fc3bfa8e19ddecfae203769 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 19 Mar 2015 01:11:34 -0700 Subject: [PATCH 362/688] slight mod to pending_puts cleaner state transition logic --- uncore/src/main/scala/cache.scala | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 5f656f31..a705f618 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -608,7 +608,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_meta.coh.clone } val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val needs_more_put_data = pending_puts.orR val do_allocate = xact.allocate() val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) @@ -816,6 +815,12 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) + when(io.data.resp.valid) { + mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + } + + def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) + def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) switch (state) { is(s_idle) { when(io.inner.acquire.valid) { @@ -865,7 +870,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { Mux(!_is_hit, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) } } is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } @@ -873,7 +878,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.wb.resp.valid) { val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + Mux(pending_puts.orR, s_wait_puts, s_data_write)) } } is(s_inner_probe) { @@ -902,7 +907,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { state := Mux(!_skip_outer_acquire, s_outer_acquire, Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)))) + Mux(pending_puts.orR, s_wait_puts, s_data_write)))) } } is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } @@ -921,7 +926,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { }.otherwise { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } } @@ -930,36 +935,32 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(io.outer.finish.ready) { state := Mux(pending_reads.orR, s_data_read, Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write))) + Mux(pending_puts.orR, s_wait_puts, s_data_write))) } } is(s_data_read) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(io.data.read.ready) { - when(PopCount(pending_reads) <= UInt(1)) { state := s_data_resp } + when(doneNextCycleHot(io.data.read.ready, pending_reads)) { + state := s_data_resp } } is(s_data_resp) { - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) - } - when(PopCount(pending_resps) === UInt(0) || - (io.data.resp.valid && PopCount(pending_resps) === UInt(1))) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(needs_more_put_data, s_wait_puts, s_data_write)) + when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { + state := Mux(!pending_writes.orR, s_inner_grant, + Mux(pending_puts.orR, s_wait_puts, s_data_write)) + } + } + is(s_wait_puts) { + when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { + state := s_data_write } } - is(s_wait_puts) { when(!needs_more_put_data) { state := s_data_write } } is(s_data_write) { when(io.data.write.ready) { when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } } } is(s_inner_grant) { - when(ignt_q.io.count === UInt(0) || - (ignt_data_done && ignt_q.io.count === UInt(1))) { + when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh when(meta_dirty) { pending_coh := pending_coh_on_ignt } state := Mux(meta_dirty, s_meta_write, @@ -973,8 +974,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { } } is(s_inner_finish) { - when(ifin_cnt === UInt(0) || - (io.inner.finish.valid && ifin_cnt === UInt(1))) { + when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { state := s_idle } } From 9708d25dfff7b4a0a6381be398f00dad608a9436 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 24 Mar 2015 02:06:53 -0700 Subject: [PATCH 363/688] Restructure L2 state machine and utilize HeaderlessTileLinkIO --- uncore/src/main/scala/broadcast.scala | 43 +- uncore/src/main/scala/cache.scala | 709 ++++++++++++-------------- uncore/src/main/scala/htif.scala | 44 +- uncore/src/main/scala/memserdes.scala | 28 +- uncore/src/main/scala/metadata.scala | 4 +- uncore/src/main/scala/network.scala | 5 +- uncore/src/main/scala/tilelink.scala | 393 +++++++++++--- uncore/src/main/scala/uncore.scala | 93 ++-- uncore/src/main/scala/util.scala | 26 +- 9 files changed, 780 insertions(+), 565 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index ce72f625..98c8f0af 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -52,20 +52,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits - val acquire_idx = Mux(acquireMatches.orR, + val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) val block_acquires = acquireConflicts.orR || !sdq_rdy - io.inner.acquire.ready := acquireReadys.orR && !block_acquires + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := - DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && - !block_acquires && - (acquire_idx === UInt(i)) + tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } // Queue to store impending Voluntary Release data @@ -94,24 +91,24 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.payload.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.payload.addr_beat + io.inner.grant.bits.payload.data := io.outer.grant.bits.data + io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new UncachedTileLinkIOArbiterThatPassesId(trackerList.size), + val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } // Get the pending data out of the store data queue - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.payload.data) + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.payload.hasData() && + io.outer.acquire.bits.hasData() && outer_data_ptr.loc === inStoreQueue - io.outer.acquire.bits.payload.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( + io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer <> outer_arb.io.out @@ -147,7 +144,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.outer.acquire.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -159,11 +155,12 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? - io.outer.acquire.bits.payload := Bundle(PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)))(outerTLParams) + io.outer.acquire.bits := Bundle( + PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) when(collect_irel_data) { io.inner.release.ready := Bool(true) @@ -271,7 +268,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact addr_block = xact.addr_block))(outerTLParams) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits.payload := outer_read //default + io.outer.acquire.bits := outer_read //default io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -346,7 +343,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact when(io.inner.release.valid) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits.payload := outer_write_rel + io.outer.acquire.bits := outer_write_rel when(io.outer.acquire.ready) { when(oacq_data_done) { pending_ognt_ack := Bool(true) @@ -368,7 +365,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) - io.outer.acquire.bits.payload := outer_write_acq + io.outer.acquire.bits := outer_write_acq when(oacq_data_done) { pending_ognt_ack := Bool(true) state := Mux(pending_outer_read, s_mem_read, s_mem_resp) @@ -376,7 +373,7 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact } is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := !pending_ognt_ack - io.outer.acquire.bits.payload := outer_read + io.outer.acquire.bits := outer_read when(io.outer.acquire.fire()) { state := s_mem_resp } } is(s_mem_resp) { // Wait to forward grants from outer memory diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a705f618..05588203 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import scala.reflect.ClassTag case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -329,22 +330,13 @@ class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent require(isPow2(nSets)) require(isPow2(nWays)) + val meta = Module(new L2MetadataArray) // TODO: add delay knob + val data = Module(new L2DataArray(1)) val tshrfile = Module(new TSHRFile(bankId)) - - //TODO: Expose queue depths and data array pipeline cycles as parameters? - tshrfile.io.inner.acquire <> io.inner.acquire - tshrfile.io.inner.probe <> io.inner.probe - tshrfile.io.inner.release <> Queue(io.inner.release) - tshrfile.io.inner.grant <> io.inner.grant - tshrfile.io.inner.finish <> io.inner.finish - + tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent - - val meta = Module(new L2MetadataArray) tshrfile.io.meta <> meta.io - - val data = Module(new L2DataArray(1)) tshrfile.io.data <> data.io } @@ -364,6 +356,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule Module(new L2AcquireTracker(id, bankId)) } + // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) @@ -373,21 +366,15 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) - val alloc_arb = Module(new Arbiter(Bool(), trackerList.size)) - alloc_arb.io.out.ready := Bool(true) - trackerAcquireIOs.zip(alloc_arb.io.in).foreach { - case(tracker, arb) => arb.valid := tracker.ready - } - val alloc_idx = Vec(alloc_arb.io.in.map(_.ready)).lastIndexWhere{b: Bool => b} - - val acquireMatchList = trackerList.map(_.io.has_acquire_match) - val any_acquire_matches = acquireMatchList.reduce(_||_) - val match_idx = Vec(acquireMatchList).indexWhere{b: Bool => b} - - val acquire_idx = Mux(any_acquire_matches, match_idx, alloc_idx) - val block_acquires = trackerList.map(_.io.has_acquire_conflict).reduce(_||_) - io.inner.acquire.ready := trackerAcquireIOs.map(_.ready).reduce(_||_) && !block_acquires + val block_acquires = acquireConflicts.orR + io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits @@ -395,15 +382,18 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule } // Wire releases from clients - val release_idx = Vec(trackerList.map(_.io.has_release_match) :+ - wb.io.has_release_match).indexWhere{b: Bool => b} val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - trackerReleaseIOs.zipWithIndex.foreach { + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) + trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.release.bits tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } - io.inner.release.ready := Vec(trackerReleaseIOs.map(_.ready)).read(release_idx) + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) @@ -412,7 +402,7 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new TileLinkIOArbiterThatPassesId(outerList.size))(outerTLParams) + val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out @@ -483,10 +473,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack val xact_src = Reg(io.inner.release.bits.header.src.clone) val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } + val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val coh = xact_meta.coh + val coh = xact_old_meta.coh val collect_irel_data = Reg(init=Bool(false)) val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) @@ -501,7 +491,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) io.outer.grant.ready := Bool(false) - io.outer.finish.valid := Bool(false) io.inner.acquire.ready := Bool(false) io.inner.probe.valid := Bool(false) io.inner.release.ready := Bool(false) @@ -529,8 +518,8 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_meta.coh.inner.onRelease(xact, xact_src) - io.meta.write.bits.data.coh.outer := xact_meta.coh.outer.onHit(M_XWR) // WB is a write + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) when(collect_irel_data) { @@ -561,7 +550,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_meta_resp) { when(io.meta.resp.valid) { xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta + xact_old_meta := io.meta.resp.bits.meta xact_way_en := io.meta.resp.bits.way_en state := Mux(io.meta.resp.bits.tag_match, Mux(xact.hasData(), s_data_write, s_meta_write), @@ -595,101 +584,73 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2XactTrackerIO - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_outer_grant :: s_outer_finish :: s_data_read :: s_data_resp :: s_wait_puts :: s_data_write :: s_inner_grant :: s_meta_write :: s_inner_finish :: Nil = Enum(UInt(), 16) + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) + // State holding transaction metadata val xact_src = Reg(io.inner.acquire.bits.header.src.clone) val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(UInt(width = innerDataBits)) } - val wmask_buffer = Vec.fill(innerDataBeats){ Reg(Bits(width = innerDataBits/8)) } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } - val xact_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } - val pending_coh = Reg{ xact_meta.coh.clone } - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - pending_puts := (pending_puts & dropPendingBitWhenHasData(io.inner.acquire)) - val do_allocate = xact.allocate() + val xact_old_meta = Reg{ new L2Metadata } + val pending_coh = Reg{ xact_old_meta.coh.clone } - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.meta.resp.bits.meta.coh.inner.full() - val probe_self = xact.requiresSelfProbe() - val mask_self = Mux(probe_self, - full_sharers | UInt(UInt(1) << xact_src, width = nCoherentClients), - full_sharers & ~UInt(UInt(1) << xact_src, width = nCoherentClients)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire, xact.addr_beat) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val pending_ofin = Reg{ io.outer.finish.bits.clone } - - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) - pending_ignt_data := pending_ignt_data | - addPendingBitInternal(io.data.resp) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant) | - addPendingBitWhenHasData(io.inner.acquire) + // Secondary miss queue val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - ignt_q.io.enq.valid := Bool(false) ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - ignt_q.io.deq.ready := ignt_data_done - - val ifin_cnt = Reg(init = UInt(0, width = log2Up(nSecondaryMisses+1))) - when(ignt_data_done) { - ifin_cnt := Mux(io.inner.finish.fire(), - Mux(io.ignt().requiresAck(), ifin_cnt, ifin_cnt - UInt(1)), - Mux(io.ignt().requiresAck(), ifin_cnt + UInt(1), ifin_cnt)) - } .elsewhen(io.inner.finish.fire()) { ifin_cnt := ifin_cnt - UInt(1) } + // TODO add ignt.dst <- iacq.src + // State holding progress made on processing this transaction + val iacq_data_done = + connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = + connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = + connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = + connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - pending_reads := (pending_reads | - addPendingBitWhenGetOrAtomic(io.inner.acquire)) & - (dropPendingBit(io.data.read) & - dropPendingBitWhenHasData(io.inner.release) & - dropPendingBitWhenHasData(io.outer.grant)) - val curr_read_beat = PriorityEncoder(pending_reads) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - pending_writes := (pending_writes | - addPendingBitWhenHasData(io.inner.acquire) | - addPendingBitWhenHasData(io.inner.release) | - addPendingBitWhenHasData(io.outer.grant)) & - dropPendingBit(io.data.write) - val curr_write_beat = PriorityEncoder(pending_writes) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - pending_resps := (pending_resps | - addPendingBitInternal(io.data.read)) & - dropPendingBitInternal(io.data.resp) + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_meta_write = Reg{ Bool() } - val pending_coh_on_hit = HierarchicalMetadata( - io.meta.resp.bits.meta.coh.inner, - io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_icoh_on_irel = pending_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) - val pending_ocoh_on_irel = pending_coh.outer.onHit(M_XWR) // WB is a write - val pending_coh_on_ognt = HierarchicalMetadata( - ManagerMetadata.onReset, - pending_coh.outer.onGrant(io.ognt(), xact.op_code())) - val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), - pending_coh.outer) - val pending_ofin_on_ognt = io.ognt().makeFinish() + val all_pending_done = + !(pending_reads.orR || + pending_writes.orR || + pending_resps.orR || + pending_puts.orR || + pending_ognts || + ignt_q.io.count > UInt(0) || + //pending_meta_write || // Has own state: s_meta_write + pending_ifins) - val amo_result = xact.data + // Provide a single ALU per tracker to merge Puts and AMOs with data being + // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU) amoalu.io.addr := xact.addr() amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() - amoalu.io.lhs := io.data.resp.bits.data //default - amoalu.io.rhs := data_buffer.head // default + amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData + val amo_result = xact.data // Reuse xact buffer space to store AMO result + + // Utility functions for updating the data and metadata that will be kept in + // the cache or granted to the original requestor after this transaction: + + def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { + when(flag && pending_coh != next) { + pending_meta_write := Bool(true) + pending_coh := next + } + } def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data @@ -704,104 +665,182 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { wmask_buffer(beat) := SInt(-1) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } - val mergeDataInternal = mergeData(rowBits) _ - val mergeDataInner = mergeData(innerDataBits) _ - val mergeDataOuter = mergeData(outerDataBits) _ + def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } + } + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { + when(in.fire() && in.bits.payload.hasData()) { + mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + } + } + def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + // Actual transaction processing logic begins here: + // + // First, take care of accpeting new requires or secondary misses + // For now, the only allowed secondary miss types are Gets-under-Get + // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - (xact_src === io.inner.acquire.bits.header.src) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish).contains(state) && - do_allocate && - ignt_q.io.enq.ready + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready + // This logic also allows the tail beats of a PutBlock to be merged in val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - (xact_src === io.inner.acquire.bits.header.src) && - (xact.client_xact_id === io.iacq().client_xact_id) && - xact.conflicts(io.iacq()) && - Vec(s_meta_read, s_meta_resp, s_wb_req, s_wb_resp, - s_inner_probe, s_outer_acquire, s_outer_grant, - s_outer_finish, s_data_read, - s_data_resp).contains(state) && - do_allocate && - ignt_q.io.enq.ready - - val in_same_set = xact.addr_block(idxMSB,idxLSB) === - io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) - io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get - io.has_acquire_conflict := (xact.conflicts(io.iacq()) || in_same_set) && - (state != s_idle) && - !io.has_acquire_match - - // If we're allocating in this cache, we can use the current metadata - // to make an appropriate custom Acquire, otherwise we copy over the - // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := state === s_outer_acquire - io.outer.acquire.bits.payload := Mux(do_allocate, - xact_meta.coh.outer.makeAcquire( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - op_code = xact.op_code()), - Bundle(Acquire(xact))(outerTLParams)) - io.outer.acquire.bits.header.src := UInt(bankId) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := state === s_outer_grant - io.outer.finish.valid := state === s_outer_finish - io.outer.finish.bits := pending_ofin - - io.inner.probe.valid := state === s_inner_probe && pending_probes.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) - - io.inner.grant.valid := state === s_inner_grant && - ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + xact_src === io.inner.acquire.bits.header.src && //TODO + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready io.inner.acquire.ready := state === s_idle || can_merge_iacq_put || can_merge_iacq_get - io.inner.release.ready := state === s_inner_probe - io.inner.finish.ready := Vec(s_inner_finish, s_meta_write, s_inner_grant, - s_data_write, s_wait_puts, s_data_resp).contains(state) - io.data.read.valid := state === s_data_read && pending_reads.orR + // Track whether any beats are missing from a PutBlock + pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits.header.src := UInt(bankId) + io.inner.probe.bits.header.dst := curr_probe_dst + io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + io.inner.release.ready := state === s_inner_probe + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease( // Drop sharer + incoming = io.irel(), + src = io.inner.release.bits.header.src), + Mux(io.irel().hasData(), // Dirty writeback + pending_coh.outer.onHit(M_XWR), + pending_coh.outer)) + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + mergeDataInner(io.inner.release) + + // The following subtrascation handles misses or coherence permission + // upgrades by initiating a new transaction in the outer memory: + // + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := state === s_outer_acquire + io.outer.acquire.bits := Mux( + xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( + client_xact_id = UInt(0), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + + // Probes from the outer memory system are handled in the WritebackUnit + io.outer.probe.ready := Bool(false) + io.outer.release.valid := Bool(false) + + io.outer.grant.ready := state === s_busy + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.outer.grant.bits, xact.op_code())) + updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) + mergeDataOuter(io.outer.grant) + + // Going back to the original inner transaction, we can issue a Grant as + // soon as the data is released, granted, put, or read from the cache + pending_ignt_data := pending_ignt_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) | + addPendingBitInternal(io.data.resp) + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.deq.ready := ignt_data_done + // Make the Grant message using the data stored in the secondary miss queue + io.inner.grant.valid := state === s_busy && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || + pending_ignt_data(ignt_data_idx)) + io.inner.grant.bits.header.src := UInt(bankId) + io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src + io.inner.grant.bits.payload := pending_coh.inner.makeGrant( + acq = xact, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) + // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of + // multiple client transaction ids from merged secondary misses + io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant( + outgoing = io.ignt(), + dst = io.inner.grant.bits.header.dst), + pending_coh.outer) + updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) + + // We must wait for as many Finishes as we sent Grants + io.inner.finish.ready := state === s_busy + + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)) | + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && + pending_reads.orR && + !pending_ognts io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := state === s_data_write && pending_writes.orR + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + mergeDataInternal(io.data.resp) + + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && + pending_writes.orR && + !pending_ognts && + !pending_reads(curr_write_beat) && + !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -810,188 +849,106 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_meta.tag, xact.addr_block(idxMSB,idxLSB)) - io.wb.req.bits.coh := xact_meta.coh + io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en io.wb.req.bits.id := UInt(trackerId) - when(io.data.resp.valid) { - mergeDataInternal(io.data.resp.bits.addr_beat, io.data.resp.bits.data) + // Handling of secondary misses (Gets and Puts only for now) + when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) } - def doneNextCycleHot(rdy: Bool, pending: UInt) = !pending.orR || rdy && PopCount(pending) === UInt(1) - def doneNextCycleCounter(rdy: Bool, pending: UInt) = pending === UInt(0) || rdy && pending === UInt(1) - switch (state) { - is(s_idle) { - when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src - xact := io.iacq() - xact.data := UInt(0) - wmask_buffer.foreach { w => w := UInt(0) } - pending_puts := Mux(io.iacq().isBuiltInType(Acquire.putBlockType), - dropPendingBitWhenHasData(io.inner.acquire), - UInt(0)) - pending_reads := Mux(io.iacq().isBuiltInType(Acquire.getBlockType) || - !io.iacq().isBuiltInType(), - SInt(-1, width = innerDataBeats), - addPendingBitWhenGetOrAtomic(io.inner.acquire)).toUInt - pending_writes := addPendingBitWhenHasData(io.inner.acquire) - pending_resps := UInt(0) - pending_ignt_data := UInt(0) - ifin_cnt := UInt(0) - ignt_q.io.enq.valid := Bool(true) - state := s_meta_read - } - } - is(s_meta_read) { when(io.meta.read.ready) { state := s_meta_resp } } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - pending_coh := io.meta.resp.bits.meta.coh - val _coh = io.meta.resp.bits.meta.coh - val _tag_match = io.meta.resp.bits.tag_match - val _is_hit = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - _tag_match && _coh.outer.isValid()) - else (_tag_match && _coh.outer.isHit(xact.op_code()))) - val _needs_writeback = !_tag_match && do_allocate && - (_coh.outer.requiresVoluntaryWriteback() || - _coh.inner.requiresProbesOnVoluntaryWriteback()) - val _needs_inner_probes = _tag_match && _coh.inner.requiresProbes(xact) - when(_is_hit) { pending_coh := pending_coh_on_hit } - when(_needs_inner_probes) { - pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) - } - state := Mux(_needs_writeback, s_wb_req, - Mux(_needs_inner_probes, s_inner_probe, - Mux(!_is_hit, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))))) - } - } - is(s_wb_req) { when(io.wb.req.ready) { state := s_wb_resp } } - is(s_wb_resp) { - when(io.wb.resp.valid) { - val _skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_inner_probe) { - // Send probes - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - when(io.inner.release.valid) { - pending_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - pending_coh.outer := pending_ocoh_on_irel - mergeDataInner(io.irel().addr_beat, io.irel().data) - } - // We don't decrement release_count until we've received all the data beats. - when(!io.irel().hasMultibeatData() || irel_data_done) { - release_count := release_count - UInt(1) - } - } - when(release_count === UInt(0)) { - val _skip_outer_acquire = (if(isLastLevelCache) - (xact.isBuiltInType(Acquire.putBlockType) || - xact_meta.coh.outer.isValid()) - else xact_meta.coh.outer.isHit(xact.op_code())) - state := Mux(!_skip_outer_acquire, s_outer_acquire, - Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)))) - } - } - is(s_outer_acquire) { when(oacq_data_done) { state := s_outer_grant } } - is(s_outer_grant) { - when(io.outer.grant.valid) { - when(io.ognt().hasData()) { - mergeDataOuter(io.ognt().addr_beat, io.ognt().data) - } - when(ognt_data_done) { - pending_coh := pending_coh_on_ognt - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - pending_ofin.header.src := UInt(bankId) - state := s_outer_finish - }.otherwise { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - } - } - is(s_outer_finish) { - when(io.outer.finish.ready) { - state := Mux(pending_reads.orR, s_data_read, - Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write))) - } - } - is(s_data_read) { - when(doneNextCycleHot(io.data.read.ready, pending_reads)) { - state := s_data_resp - } - } - is(s_data_resp) { - when(doneNextCycleHot(io.data.resp.valid, pending_resps)) { - state := Mux(!pending_writes.orR, s_inner_grant, - Mux(pending_puts.orR, s_wait_puts, s_data_write)) - } - } - is(s_wait_puts) { - when(doneNextCycleHot(io.inner.acquire.fire(), pending_puts)) { - state := s_data_write - } - } - is(s_data_write) { - when(io.data.write.ready) { - when(PopCount(pending_writes) <= UInt(1)) { state := s_inner_grant } - } - } - is(s_inner_grant) { - when(doneNextCycleCounter(ignt_data_done, ignt_q.io.count)) { - val meta_dirty = !xact_tag_match || pending_coh_on_ignt != xact_meta.coh - when(meta_dirty) { pending_coh := pending_coh_on_ignt } - state := Mux(meta_dirty, s_meta_write, - Mux(ifin_cnt > UInt(0) || io.ignt().requiresAck(), - s_inner_finish, s_idle)) - } - } - is(s_meta_write) { - when(io.meta.write.ready) { - state := Mux(ifin_cnt > UInt(0), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - when(doneNextCycleCounter(io.inner.finish.valid, ifin_cnt)) { - state := s_idle - } - } + // Defined here because of Chisel default wire demands, used in s_meta_resp + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.acquire.valid) { + xact_src := io.inner.acquire.bits.header.src + xact := io.iacq() + xact.data := UInt(0) + pending_puts := Mux( + io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenBeatHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux( + io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) + pending_resps := UInt(0) + pending_ignt_data := UInt(0) + pending_meta_write := UInt(0) + state := s_meta_read + } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + val coh = io.meta.resp.bits.meta.coh + val tag_match = io.meta.resp.bits.tag_match + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || + tag_match && coh.outer.isValid()) + val needs_writeback = !tag_match && + xact.allocate() && + (coh.outer.requiresVoluntaryWriteback() || + coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) + when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } + pending_coh := Mux(is_hit, pending_coh_on_hit, coh) + when(needs_inner_probes) { + val full_sharers = coh.inner.full() + val mask_self = Mux( + xact.requiresSelfProbe(), + coh.inner.full() | UIntToOH(xact_src), + coh.inner.full() & ~UIntToOH(xact_src)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + pending_iprbs := mask_incoherent + } + state := Mux(needs_writeback, s_wb_req, + Mux(needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, s_busy))) + } + when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } + when(state === s_wb_resp && io.wb.resp.valid) { + // If we're overwriting the whole block in a last level cache we can + // just do it without fetching any data from memory + val skip_outer_acquire = Bool(isLastLevelCache) && + xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + // Tag matches, so if this is the last level cache we can use the data without upgrading permissions + val skip_outer_acquire = + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_outer_acquire && oacq_data_done) { state := s_busy } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && (io.meta.write.ready || !pending_meta_write)) { + wmask_buffer.foreach { w => w := UInt(0) } + state := s_idle } - // Handle Get and Put merging - when(io.inner.acquire.fire()) { - when (io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - } - when(!io.iacq().hasMultibeatData()) { ignt_q.io.enq.valid := Bool(true) } - } + // These IOs are used for routing in the parent + val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_inner_probe) + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match + //TODO: relax from in_same_set to xact.conflicts(io.iacq())? + // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && io.inner.acquire.bits.header.src != xact_src), "AcquireTracker accepted data beat from different network source than initial request.") @@ -1019,7 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_outer_finish :: s_wb_resp :: Nil = Enum(UInt(), 8) + val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) @@ -1027,10 +984,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_way_en = Reg{ Bits(width = nWays) } val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val xact_id = Reg{ UInt() } - val pending_ofin = Reg{ io.outer.finish.bits.clone } val irel_had_data = Reg(init = Bool(false)) - val release_count = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) + val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() @@ -1056,15 +1012,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.acquire.valid := Bool(false) io.outer.probe.ready := Bool(false) io.outer.release.valid := Bool(false) // default - io.outer.release.bits.payload := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.release.bits.header.src := UInt(bankId) + io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_cnt, + data = data_buffer(orel_data_cnt)) io.outer.grant.ready := Bool(false) // default - io.outer.finish.valid := Bool(false) // default - io.outer.finish.bits := pending_ofin io.inner.probe.valid := Bool(false) io.inner.probe.bits.header.src := UInt(bankId) @@ -1100,7 +1053,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_inner_probes) { pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) + irel_cnt := PopCount(mask_incoherent) } state := Mux(needs_inner_probes, s_inner_probe, s_data_read) } @@ -1121,12 +1074,12 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { xact_coh.outer := pending_ocoh_on_irel data_buffer(io.irel().addr_beat) := io.irel().data } - // We don't decrement release_count until we've received all the data beats. + // We don't decrement irel_cnt until we've received all the data beats. when(!io.irel().hasData() || irel_data_done) { - release_count := release_count - UInt(1) + irel_cnt := irel_cnt - UInt(1) } } - when(release_count === UInt(0)) { + when(irel_cnt === UInt(0)) { state := Mux(irel_had_data, // If someone released a dirty block s_outer_release, // write that block back, otherwise Mux(xact_coh.outer.requiresVoluntaryWriteback(), @@ -1152,19 +1105,9 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { is(s_outer_grant) { io.outer.grant.ready := Bool(true) when(io.outer.grant.valid) { - when(io.ognt().requiresAck()) { - pending_ofin.payload := pending_ofin_on_ognt - pending_ofin.header.dst := io.outer.grant.bits.header.src - state := s_outer_finish - }.otherwise { - state := s_wb_resp - } + state := s_wb_resp } } - is(s_outer_finish) { - io.outer.finish.valid := Bool(true) - when(io.outer.finish.ready) { state := s_wb_resp } - } is(s_wb_resp) { io.wb.resp.valid := Bool(true) state := s_idle diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 81d95453..787847b9 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -127,21 +127,11 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) - val mem_acked = Reg(init=Bool(false)) - val mem_gxid = Reg(Bits()) - val mem_gsrc = Reg(UInt()) - val mem_needs_ack = Reg(Bool()) - when (io.mem.grant.valid) { - mem_acked := Bool(true) - mem_gxid := io.mem.grant.bits.payload.manager_xact_id - mem_gsrc := io.mem.grant.bits.header.src - mem_needs_ack := io.mem.grant.bits.payload.requiresAck() - } - io.mem.grant.ready := Bool(true) - - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_mem_finish :: state_tx :: Nil = Enum(UInt(), 9) + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) val state = Reg(init=state_rx) + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) when (state === state_rx && rx_done) { state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, @@ -149,28 +139,18 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, state_tx))) } - - val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || - (state === state_mem_rresp && io.mem.grant.valid), dataBeats) when (state === state_mem_wreq) { when (cnt_done) { state := state_mem_wresp } } when (state === state_mem_rreq) { when(io.mem.acquire.ready) { state := state_mem_rresp } } - when (state === state_mem_wresp) { - when (mem_acked) { - state := state_mem_finish - mem_acked := Bool(false) - } + when (state === state_mem_wresp && io.mem.grant.valid) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << offsetBits-3) } - when (state === state_mem_rresp) { - when (cnt_done) { - state := state_mem_finish - mem_acked := Bool(false) - } - } - when (state === state_mem_finish && io.mem.finish.ready) { + when (state === state_mem_rresp && cnt_done) { state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) pos := pos - UInt(1) addr := addr + UInt(1 << offsetBits-3) @@ -187,8 +167,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val mem_req_data = (0 until n).map { i => val ui = UInt(i, log2Up(n)) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(Cat(io.mem.grant.bits.payload.addr_beat, ui)) := - io.mem.grant.bits.payload.data((i+1)*short_request_bits-1, i*short_request_bits) + packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := + io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) } packet_ram(Cat(cnt, ui)) }.reverse.reduce(_##_) @@ -202,9 +182,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { client_xact_id = UInt(0), data = mem_req_data), GetBlock(addr_block = init_addr)) - io.mem.finish.valid := (state === state_mem_finish) && mem_needs_ack - io.mem.finish.bits.payload.manager_xact_id := mem_gxid - io.mem.finish.bits.header.dst := mem_gsrc + io.mem.grant.ready := Bool(true) val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 059fbdcf..878ce19a 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -207,7 +207,7 @@ class MemDesser(w: Int) extends Module // test rig side //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends Module { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val io = new Bundle { val tl = new TileLinkIO().flip val mem = new MemIO @@ -216,11 +216,10 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val mifAddrBits = params(MIFAddrBits) val mifDataBits = params(MIFDataBits) val mifDataBeats = params(MIFDataBeats) - val tlDataBits = params(TLDataBits) - val tlDataBeats = params(TLDataBeats) val dataBits = tlDataBits*tlDataBeats - require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats) - require(params(TLClientXactIdBits) <= params(MIFTagBits)) + val dstIdBits = params(LNHeaderBits) + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") + require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) io.tl.acquire.ready := Bool(false) io.tl.probe.valid := Bool(false) @@ -231,6 +230,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) io.tl.grant <> gnt_arb.io.out + val dst_off = dstIdBits + tlClientXactIdBits val acq_has_data = io.tl.acquire.bits.payload.hasData() val rel_has_data = io.tl.release.bits.payload.hasData() @@ -253,6 +253,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), client_xact_id = tag_out >> UInt(1), manager_xact_id = UInt(0)) + gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(1).bits.header.src := UInt(0) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -273,7 +275,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data @@ -284,7 +287,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.acquire.bits.payload.client_xact_id, + tag_out := Cat(io.tl.release.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) addr_out := io.tl.acquire.bits.payload.addr_block has_data := acq_has_data @@ -350,7 +354,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { data_from_rel := Bool(true) make_grant_ack := Bool(true) io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.payload.client_xact_id, + val tag = Cat(io.tl.release.bits.header.src, + io.tl.release.bits.payload.client_xact_id, io.tl.release.bits.payload.isVoluntary()) val addr = io.tl.release.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -364,7 +369,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { make_grant_ack := acq_has_data io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.payload.client_xact_id, + val tag = Cat(io.tl.acquire.bits.header.src, + io.tl.acquire.bits.payload.client_xact_id, io.tl.acquire.bits.payload.isBuiltInType()) val addr = io.tl.acquire.bits.payload.addr_block io.mem.req_cmd.bits.tag := tag @@ -421,6 +427,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -454,6 +462,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends Module { manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = io.mem.resp.bits.data) + gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) + gnt_arb.io.in(0).bits.header.src := UInt(0) } } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 8f5d7e90..af5504c8 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -18,6 +18,7 @@ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + def !=(rhs: ClientMetadata): Bool = !this.===(rhs) def isValid(dummy: Int = 0): Bool = co.isValid(this) def isHit(cmd: UInt): Bool = co.isHit(cmd, this) @@ -99,6 +100,7 @@ class ManagerMetadata extends CoherenceMetadata { def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers + def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) def full(dummy: Int = 0) = co.dir.full(this.sharers) def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) @@ -171,7 +173,7 @@ class HierarchicalMetadata extends CoherenceMetadata { val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer - def !=(rhs: HierarchicalMetadata): Bool = !(this === rhs) + def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } object HierarchicalMetadata { diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 161a9a0c..7fdc8e09 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -6,6 +6,7 @@ import Chisel._ case object LNManagers extends Field[Int] case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] class PhysicalHeader(n: Int) extends Bundle { val src = UInt(width = log2Up(n)) @@ -47,8 +48,8 @@ class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Opti abstract class LogicalNetwork extends Module class LogicalHeader extends Bundle { - val src = UInt(width = log2Up(params(LNEndpoints))) - val dst = UInt(width = log2Up(params(LNEndpoints))) + val src = UInt(width = params(LNHeaderBits)) + val dst = UInt(width = params(LNHeaderBits)) } class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 0aff9aab..c2bc2d30 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,7 +3,8 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ // Parameters exposed to the top-level design, set based on // external requirements or design space exploration @@ -11,17 +12,22 @@ import scala.reflect.ClassTag case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] case object TLBlockAddrBits extends Field[Int] -case object TLManagerXactIdBits extends Field[Int] -case object TLClientXactIdBits extends Field[Int] +case object TLMaxClientXacts extends Field[Int] +case object TLMaxClientPorts extends Field[Int] +case object TLMaxManagerXacts extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { val tlBlockAddrBits = params(TLBlockAddrBits) - val tlClientXactIdBits = params(TLClientXactIdBits) - val tlManagerXactIdBits = params(TLManagerXactIdBits) + val tlMaxClientXacts = params(TLMaxClientXacts) + val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxManagerXacts = params(TLMaxManagerXacts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlDataBits = params(TLDataBits) + val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 @@ -41,12 +47,16 @@ abstract trait TileLinkParameters extends UsesParameters { val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLBundle extends Bundle with TileLinkParameters { +} abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel // Used to hook up logical network ports to physical network ports -trait TileLinkChannel extends TLBundle +trait TileLinkChannel extends TLBundle { + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool +} trait ClientToManagerChannel extends TileLinkChannel trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now @@ -97,8 +107,7 @@ class Acquire extends ClientToManagerChannel val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType() && - (a_type === Acquire.putType || a_type === Acquire.putBlockType), + def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) @@ -328,6 +337,8 @@ class Probe extends ManagerToClientChannel val p_type = UInt(width = tlCoh.probeTypeWidth) def is(t: UInt) = p_type === t + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) } object Probe { @@ -383,6 +394,7 @@ class Grant extends ManagerToClientChannel // Helper funcs def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t def is(t: UInt):Bool = g_type === t def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), Grant.typesWithData.contains(g_type), @@ -394,7 +406,7 @@ class Grant extends ManagerToClientChannel def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Bundle(new Finish, { case TLManagerXactIdBits => tlManagerXactIdBits }) + val f = Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts }) f.manager_xact_id := this.manager_xact_id f } @@ -428,7 +440,10 @@ object Grant { } } -class Finish extends ClientToManagerChannel with HasManagerTransactionId +class Finish extends ClientToManagerChannel with HasManagerTransactionId { + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} // Complete IO definitions for two types of TileLink clients class UncachedTileLinkIO extends TLBundle { @@ -455,33 +470,22 @@ class TileLinkIOWrapper extends TLModule { io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object TileLinkIOWrapper { - def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper)(p) - conv.io.in <> utl - conv.io.out - } - def apply(utl: UncachedTileLinkIO): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> utl - conv.io.out - } - def apply(tl: TileLinkIO) = tl -} // This version of TileLinkIO does not contain network headers for packets // that originate in the Clients (i.e. Acquire and Release). These headers // are provided in the top-level that instantiates the clients and network. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. +// Secondly, this version of the interface elides Finish messages, with the +// assumption that a FinishUnit has been coupled to the TileLinkIO port +// to deal with acking received Grants. class HeaderlessUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) - val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip - val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) + val grant = new DecoupledIO(new Grant).flip } class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { - val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } @@ -492,17 +496,231 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant - io.out.finish <> io.in.finish io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } -object HeaderlessTileLinkIOWrapper { +object TileLinkIOWrapper { + def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { + val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { val conv = Module(new HeaderlessTileLinkIOWrapper) conv.io.in <> utl conv.io.out } + def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasClientId = TileLinkChannel with HasClientTransactionId + type LNAcquire = LogicalNetworkIO[Acquire] + type LNRelease = LogicalNetworkIO[Release] + type LNGrant = LogicalNetworkIO[Grant] + + def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( + in: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, beat) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, beat) + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} + } + } + + def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int): Vec[Bool] = { + val id = in.bits.payload.client_xact_id + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { + in.bits match { + case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => + connectDataBeatCounter(in.fire(), p, UInt(0))._2 + case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => + connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 + case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} + } + } + + def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) + } + + def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( + max: Int, + up: DecoupledIO[LogicalNetworkIO[T]], + down: DecoupledIO[LogicalNetworkIO[S]], + inc: T => Bool = (t: T) => Bool(true), + dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) + val do_inc = up_done && inc(up.bits.payload) + val do_dec = down_done && dec(down.bits.payload) + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} + +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) +} + +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) + +class FinishUnit(srcId: Int = 0) extends TLModule + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + val grant_done = Bool(OUTPUT) + val pending_finish = Bool(OUTPUT) + } + + val entries = 1 << tlClientXactIdBits + val g = io.grant.bits.payload + assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) + + val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) + val q = Module(new FinishQueue(entries)) + + q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := io.grant.valid + io.refill.bits := io.grant.bits.payload + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) + + io.ready := q.io.enq.ready + io.grant_done := done + io.pending_finish := q.io.deq.valid +} + +object TileLinkHeaderOverwriter { + def apply[T <: ClientToManagerChannel]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { + val out = in.clone.asDirectionless + out.bits.payload := in.bits.payload + out.bits.header.src := UInt(clientId) + out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) + out.valid := in.valid + in.ready := out.ready + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[LogicalNetworkIO[T]], + clientId: Int, + nBanks: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) + out.bits.header.dst := addrConvert(in.bits.payload.addr_block) + out + } + + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val client = new HeaderlessTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) + val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.network.release <> rel_with_header + io.client.probe <> prb_without_header + io.client.grant <> gnt_without_header +} + +object TileLinkNetworkPort { + def apply[T <: Data]( + client: HeaderlessTileLinkIO, + clientId: Int = 0, + addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { + val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) + port.io.client <> client + port.io.network + } } class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { @@ -529,7 +747,6 @@ object TileLinkEnqueuer { } abstract trait HasArbiterTypes { - val arbN: Int type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId type ClientSourcedWithIdAndData = ClientToManagerChannel with @@ -538,20 +755,19 @@ abstract trait HasArbiterTypes { } // Utility functions for constructing TileLinkIO arbiters -abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule - with HasArbiterTypes { - +trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ + val arbN: Int // These are filled in depending on whether the arbiter mucks with the // client ids and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -561,11 +777,11 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, params(TLDataBeats), Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -575,17 +791,23 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule arb.io.out <> mngr } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag] - (clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } - arb.io.out <> mngr + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } } - def hookupManagerSourceWithId[M <: ManagerSourcedWithId] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { mngr.ready := Bool(false) for (i <- 0 until arbN) { clts(i).valid := Bool(false) @@ -594,24 +816,45 @@ abstract class TileLinkArbiterLike(val arbN: Int) extends TLModule mngr.ready := clts(i).ready } clts(i).bits := mngr.bits - clts(i).bits.payload.client_xact_id := - managerSourcedClientXactId(mngr.bits.payload) + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) } } - def hookupManagerSourceBroadcast[M <: ManagerToClientChannel] - (clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } + + def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.out <> mngr + } } -abstract class UncachedTileLinkIOArbiter(n: Int) - extends TileLinkArbiterLike(n) { +abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new UncachedTileLinkIO}.flip + val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip val out = new UncachedTileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -619,9 +862,9 @@ abstract class UncachedTileLinkIOArbiter(n: Int) hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { +abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(n){new TileLinkIO}.flip + val in = Vec.fill(arbN){new TileLinkIO}.flip val out = new TileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -631,8 +874,32 @@ abstract class TileLinkIOArbiter(n: Int) extends TileLinkArbiterLike(n) { hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip + val out = new HeaderlessUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module + with TileLinkArbiterLike + with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip + val out = new HeaderlessTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + // Appends the port index of the arbiter to the client_xact_id -abstract trait AppendsArbiterId extends HasArbiterTypes { +trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) def managerSourcedClientXactId(in: ManagerSourcedWithId) = @@ -641,14 +908,14 @@ abstract trait AppendsArbiterId extends HasArbiterTypes { } // Uses the client_xact_id as is (assumes it has been set to port index) -abstract trait PassesId extends HasArbiterTypes { +trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } // Overwrites some default client_xact_id with the port idx -abstract trait UsesNewId extends HasArbiterTypes { +trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index de55c0e2..44981459 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,7 +2,8 @@ package uncore import Chisel._ -import scala.reflect.ClassTag +import scala.reflect._ +import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] @@ -82,26 +83,24 @@ trait HasInnerTLIO extends CoherenceAgentBundle { } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new UncachedTileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new TileLinkIO)(outerTLParams) - def oacq(dummy: Int = 0) = outer.acquire.bits.payload - def oprb(dummy: Int = 0) = outer.probe.bits.payload - def orel(dummy: Int = 0) = outer.release.bits.payload - def ognt(dummy: Int = 0) = outer.grant.bits.payload - def ofin(dummy: Int = 0) = outer.finish.bits.payload + val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def oprb(dummy: Int = 0) = outer.probe.bits + def orel(dummy: Int = 0) = outer.release.bits + def ognt(dummy: Int = 0) = outer.grant.bits } class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { def innerTL: TileLinkIO - def outerTL: TileLinkIO + def outerTL: HeaderlessTileLinkIO def incoherent: Vec[Bool] } @@ -131,39 +130,47 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule { - def connectDataBeatCounter[S <: HasTileLinkData : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) +abstract class XactTracker extends CoherenceAgentModule + with HasDataBeatCounters { + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + + def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + addPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } } - def connectOutgoingDataBeatCounter[T <: HasTileLinkData : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - beat: UInt = UInt(0)) = { - connectDataBeatCounter(in.fire(), in.bits.payload, beat) - } - - def connectIncomingDataBeatCounter[T <: HasTileLinkData : ClassTag](in: DecoupledIO[LogicalNetworkIO[T]]) = { - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - } - - def addPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) & - UIntToOH(in.bits.payload.addr_beat) - } - - def dropPendingBitWhenHasData[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) = { - ~Fill(in.bits.payload.tlDataBeats, in.fire() && in.bits.payload.hasData()) | - ~UIntToOH(in.bits.payload.addr_beat) - } - - def addPendingBitWhenGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]) = { + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { val a = in.bits.payload - Fill(a.tlDataBeats, in.fire() && a.isBuiltInType() && - (a.is(Acquire.getType) || a.is(Acquire.getBlockType) || a.is(Acquire.putAtomicType))) & - UIntToOH(a.addr_beat) + val isGetOrAtomic = a.isBuiltInType() && + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + } + + def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { + in.bits match { + case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => + dropPendingBitWhenBeat(in.fire() && p.hasData(), p) + case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => + dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) + case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } + } + } + + def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { + ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 14575aa5..f02cf2c3 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -34,15 +34,15 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: Int) extends Module { +class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen.clone).flip val out = Decoupled(gen.clone) val cnt = UInt(OUTPUT, log2Up(n)) val done = Bool(OUTPUT) } - val narrowWidth = io.in.bits.payload.data.getWidth / n - require(io.in.bits.payload.data.getWidth % narrowWidth == 0) + val narrowWidth = io.in.bits.data.getWidth / n + require(io.in.bits.data.getWidth % narrowWidth == 0) if(n == 1) { io.in <> io.out @@ -51,12 +51,12 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } else { val cnt = Reg(init=UInt(0, width = log2Up(n))) val wrap = cnt === UInt(n-1) - val rbits = Reg(init=io.in.bits) + val rbits = Reg(io.in.bits.clone) val active = Reg(init=Bool(false)) val shifter = Vec.fill(n){Bits(width = narrowWidth)} (0 until n).foreach { - i => shifter(i) := rbits.payload.data((i+1)*narrowWidth-1,i*narrowWidth) + i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } io.done := Bool(false) @@ -65,16 +65,16 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I io.out.valid := active || io.in.valid io.out.bits := io.in.bits when(!active && io.in.valid) { - when(io.in.bits.payload.hasData()) { + when(io.in.bits.hasData()) { cnt := Mux(io.out.ready, UInt(1), UInt(0)) rbits := io.in.bits active := Bool(true) } - io.done := !io.in.bits.payload.hasData() + io.done := !io.in.bits.hasData() } when(active) { io.out.bits := rbits - io.out.bits.payload.data := shifter(cnt) + io.out.bits.data := shifter(cnt) when(io.out.ready) { cnt := cnt + UInt(1) when(wrap) { @@ -86,3 +86,13 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: LogicalNetworkIO[T], n: I } } } + +object FlowThroughSerializer { + def apply[T <: HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { + val fs = Module(new FlowThroughSerializer(in.bits, n)) + fs.io.in.valid := in.valid + fs.io.in.bits := in.bits + in.ready := fs.io.in.ready + fs.io.out + } +} From 3cf1778c928a717e96d863330762dca14e9161a0 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 3 Apr 2015 17:24:44 -0700 Subject: [PATCH 364/688] moved ecc lib to uncore, l2 checks for partial write masks when ecc is enabled --- uncore/src/main/scala/cache.scala | 15 ++- uncore/src/main/scala/ecc.scala | 146 +++++++++++++++++++++++++++++ uncore/src/main/scala/uncore.scala | 6 +- uncore/src/main/scala/util.scala | 8 ++ 4 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 uncore/src/main/scala/ecc.scala diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 05588203..492ecc0e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -15,6 +15,7 @@ case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] +case object ECCCode extends Field[Option[Code]] abstract trait CacheParameters extends UsesParameters { val nSets = params(NSets) @@ -28,6 +29,7 @@ abstract trait CacheParameters extends UsesParameters { val rowBits = params(RowBits) val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) + val code = params(ECCCode).getOrElse(new IdentityCode) } abstract class CacheBundle extends Bundle with CacheParameters @@ -176,6 +178,7 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = params(NSecondaryMisses) val isLastLevelCache = true + val ignoresWriteMask = !params(ECCCode).isEmpty } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters @@ -462,6 +465,12 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) + + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { + val a = in.bits.payload + val isPartial = a.wmask() != Acquire.fullWriteMask + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + } } class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { @@ -805,7 +814,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & dropPendingBitWhenBeatHasData(io.outer.grant)) | - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) io.data.read.valid := state === s_busy && pending_reads.orR && @@ -880,7 +890,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_reads := Mux( io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire)).toUInt + (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala new file mode 100644 index 00000000..b5864b2c --- /dev/null +++ b/uncore/src/main/scala/ecc.scala @@ -0,0 +1,146 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ + +abstract class Decoding +{ + def uncorrected: Bits + def corrected: Bits + def correctable: Bool + def uncorrectable: Bool + def error = correctable || uncorrectable +} + +abstract class Code +{ + def width(w0: Int): Int + def encode(x: Bits): Bits + def decode(x: Bits): Decoding +} + +class IdentityCode extends Code +{ + def width(w0: Int) = w0 + def encode(x: Bits) = x + def decode(y: Bits) = new Decoding { + def uncorrected = y + def corrected = y + def correctable = Bool(false) + def uncorrectable = Bool(false) + } +} + +class ParityCode extends Code +{ + def width(w0: Int) = w0+1 + def encode(x: Bits) = Cat(x.xorR, x) + def decode(y: Bits) = new Decoding { + def uncorrected = y(y.getWidth-2,0) + def corrected = uncorrected + def correctable = Bool(false) + def uncorrectable = y.xorR + } +} + +class SECCode extends Code +{ + def width(k: Int) = { + val m = new Unsigned(k).log2 + 1 + k + m + (if((1 << m) < m+k+1) 1 else 0) + } + def encode(x: Bits) = { + val k = x.getWidth + require(k > 0) + val n = width(k) + + val y = for (i <- 1 to n) yield { + if (isPow2(i)) { + val r = for (j <- 1 to n; if j != i && (j & i) != 0) + yield x(mapping(j)) + r reduce (_^_) + } else + x(mapping(i)) + } + Vec(y).toBits + } + def decode(y: Bits) = new Decoding { + val n = y.getWidth + require(n > 0 && !isPow2(n)) + + val p2 = for (i <- 0 until log2Up(n)) yield 1 << i + val syndrome = p2 map { i => + val r = for (j <- 1 to n; if (j & i) != 0) + yield y(j-1) + r reduce (_^_) + } + val s = Vec(syndrome).toBits + + private def swizzle(z: Bits) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits + def uncorrected = swizzle(y) + def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) + def correctable = s.orR + def uncorrectable = Bool(false) + } + private def mapping(i: Int) = i-1-log2Up(i) +} + +class SECDEDCode extends Code +{ + private val sec = new SECCode + private val par = new ParityCode + + def width(k: Int) = sec.width(k)+1 + def encode(x: Bits) = par.encode(sec.encode(x)) + def decode(x: Bits) = new Decoding { + val secdec = sec.decode(x(x.getWidth-2,0)) + val pardec = par.decode(x) + + def uncorrected = secdec.uncorrected + def corrected = secdec.corrected + def correctable = pardec.uncorrectable + def uncorrectable = !pardec.uncorrectable && secdec.correctable + } +} + +object ErrGen +{ + // generate a 1-bit error with approximate probability 2^-f + def apply(width: Int, f: Int): Bits = { + require(width > 0 && f >= 0 && log2Up(width) + f <= 16) + UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) + } + def apply(x: Bits, f: Int): Bits = x ^ apply(x.getWidth, f) +} + +class SECDEDTest extends Module +{ + val code = new SECDEDCode + val k = 4 + val n = code.width(k) + + val io = new Bundle { + val original = Bits(OUTPUT, k) + val encoded = Bits(OUTPUT, n) + val injected = Bits(OUTPUT, n) + val uncorrected = Bits(OUTPUT, k) + val corrected = Bits(OUTPUT, k) + val correctable = Bool(OUTPUT) + val uncorrectable = Bool(OUTPUT) + } + + val c = Counter(Bool(true), 1 << k) + val numErrors = Counter(c._2, 3)._1 + val e = code.encode(c._1) + val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1)) + val d = code.decode(i) + + io.original := c._1 + io.encoded := e + io.injected := i + io.uncorrected := d.uncorrected + io.corrected := d.corrected + io.correctable := d.correctable + io.uncorrectable := d.uncorrectable +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 44981459..62a53129 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -132,8 +132,10 @@ class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflict abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { - def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) - def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = + Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = + ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { in.bits match { diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index f02cf2c3..65c5d6cd 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -5,6 +5,14 @@ package uncore import Chisel._ import scala.math._ +class Unsigned(x: Int) { + require(x >= 0) + def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } + def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } + def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 + def nextPow2: Int = if (x == 0) 1 else 1 << clog2 +} + object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) From 90f800d87dd1d05b3837f710d9dd5d131747149e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 15:57:06 -0700 Subject: [PATCH 365/688] Grant bugfixes and more comments --- uncore/src/main/scala/cache.scala | 25 +++++++++++++++++-------- uncore/src/main/scala/tilelink.scala | 3 ++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 492ecc0e..e399eb98 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -577,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } } is(s_inner_grant) { - io.inner.grant.valid := Bool(true) + io.inner.grant.valid := !collect_irel_data when(io.inner.grant.ready) { state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) } @@ -587,6 +587,10 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack when(io.inner.finish.valid) { state := s_idle } } } + + // Checks for illegal behavior + assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), + "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") } @@ -748,8 +752,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) - // The following subtrascation handles misses or coherence permission - // upgrades by initiating a new transaction in the outer memory: + // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the @@ -804,12 +807,17 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_coh.inner.onGrant( outgoing = io.ignt(), dst = io.inner.grant.bits.header.dst), - pending_coh.outer) + Mux(ognt_data_done, + pending_coh_on_ognt.outer, + pending_coh.outer)) updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) // We must wait for as many Finishes as we sent Grants io.inner.finish.ready := state === s_busy + // We read from the the cache at this level if data wasn't written back or refilled. + // We may merge Gets, requiring further beats to be read. + // If ECC requires a full writemask, we'll read out data on partial writes as well. pending_reads := (pending_reads & dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & @@ -829,6 +837,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { addPendingBitInternal(io.data.read) mergeDataInternal(io.data.resp) + // We write data to the cache at this level if it was Put here with allocate flag, + // written back dirty, or refilled from outer memory. pending_writes := (pending_writes & dropPendingBit(io.data.write)) | addPendingBitWhenBeatHasData(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | @@ -883,11 +893,11 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) - pending_puts := Mux( + pending_puts := Mux( // Make sure to collect all data from a PutBlock io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenBeatHasData(io.inner.acquire), UInt(0)) - pending_reads := Mux( + pending_reads := Mux( // GetBlocks and custom types read all beats io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), SInt(-1, width = innerDataBeats), (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | @@ -932,8 +942,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { when(state === s_wb_resp && io.wb.resp.valid) { // If we're overwriting the whole block in a last level cache we can // just do it without fetching any data from memory - val skip_outer_acquire = Bool(isLastLevelCache) && - xact.isBuiltInType(Acquire.putBlockType) + val skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c2bc2d30..d639f791 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -583,6 +583,7 @@ trait HasDataBeatCounters { val cnt = Reg(init = UInt(0, width = log2Up(max+1))) val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) + //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") cnt := Mux(do_dec, Mux(do_inc, cnt, cnt - UInt(1)), Mux(do_inc, cnt + UInt(1), cnt)) @@ -632,7 +633,7 @@ class FinishUnit(srcId: Int = 0) extends TLModule val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) val q = Module(new FinishQueue(entries)) - q.io.enq.valid := io.grant.valid && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) q.io.enq.bits.fin := g.makeFinish() q.io.enq.bits.dst := io.grant.bits.header.src From ce3271aef24329a2bde0e5fe8f5b60e73adac564 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 13 Apr 2015 19:00:40 -0700 Subject: [PATCH 366/688] refactor LNClients and LNManagers --- uncore/src/main/scala/broadcast.scala | 10 ++-- uncore/src/main/scala/cache.scala | 52 ++++++++++------- uncore/src/main/scala/memserdes.scala | 6 +- uncore/src/main/scala/network.scala | 2 - uncore/src/main/scala/tilelink.scala | 82 ++++++++++++++++----------- uncore/src/main/scala/uncore.scala | 33 ++--------- 6 files changed, 97 insertions(+), 88 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 98c8f0af..340db34d 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -187,7 +187,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_outer) { io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) when(oacq_data_done) { - state := Mux(xact.requiresAck(), s_grant, s_idle) + state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? } } is(s_grant) { // Forward the Grant.voluntaryAck @@ -221,13 +221,13 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = nCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = nCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e399eb98..b193c5a7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -182,7 +182,21 @@ abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgen } abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { + def doInternalOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in zip ins map { case (a, in) => a <> in } + } + + def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + val idx = in.bits.id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + } +} trait HasL2Id extends Bundle with CoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) @@ -361,8 +375,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors, bankId)) - doOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) - doInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) @@ -409,13 +423,13 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } io.outer <> outer_arb.io.out - // Wire local memories - doOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) - doOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) - doOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) - doOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) - doInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) - doInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) + // Wire local memory arrays + doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doInternalOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doInternalOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doInternalOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInternalInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } @@ -620,19 +634,19 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = - connectTwoWayBeatCounter(nCoherentClients, io.inner.probe, io.inner.release)._1 + connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) val pending_ifins = connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 - val pending_puts = Reg(init=Bits(0, width = innerDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = nCoherentClients)) - val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) - val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) - val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) - val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_meta_write = Reg{ Bool() } val all_pending_done = @@ -1006,8 +1020,8 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(nCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = nCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 878ce19a..942874cf 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -281,7 +281,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { addr_out := io.tl.release.bits.payload.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() tl_done_out := tl_wrap_out tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data } .elsewhen(io.tl.acquire.valid) { @@ -352,7 +352,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := Bool(true) + make_grant_ack := io.tl.release.bits.payload.requiresAck() io.mem.req_data.bits.data := io.tl.release.bits.payload.data val tag = Cat(io.tl.release.bits.header.src, io.tl.release.bits.payload.client_xact_id, @@ -366,7 +366,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { has_data := rel_has_data } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) - make_grant_ack := acq_has_data + make_grant_ack := acq_has_data // i.e. is it a Put io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data io.mem.req_cmd.bits.rw := acq_has_data val tag = Cat(io.tl.acquire.bits.header.src, diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 7fdc8e09..f61cec6c 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -3,8 +3,6 @@ package uncore import Chisel._ -case object LNManagers extends Field[Int] -case object LNClients extends Field[Int] case object LNEndpoints extends Field[Int] case object LNHeaderBits extends Field[Int] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d639f791..1ed95df6 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -11,25 +11,33 @@ import scala.reflect.runtime.universe._ // case object TLId extends Field[String] // Unique name per network case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLBlockAddrBits extends Field[Int] +case object TLNManagers extends Field[Int] +case object TLNClients extends Field[Int] +case object TLNCoherentClients extends Field[Int] +case object TLNIncoherentClients extends Field[Int] case object TLMaxClientXacts extends Field[Int] case object TLMaxClientPorts extends Field[Int] case object TLMaxManagerXacts extends Field[Int] +case object TLBlockAddrBits extends Field[Int] case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] abstract trait TileLinkParameters extends UsesParameters { - val tlBlockAddrBits = params(TLBlockAddrBits) + val tlCoh = params(TLCoherencePolicy) + val tlNManagers = params(TLNManagers) + val tlNClients = params(TLNClients) + val tlNCoherentClients = params(TLNCoherentClients) + val tlNIncoherentClients = params(TLNIncoherentClients) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) + val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) - val tlCoh = params(TLCoherencePolicy) val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) @@ -44,11 +52,11 @@ abstract trait TileLinkParameters extends UsesParameters { val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val tlNetworkDoesNotInterleaveBeats = true val amoAluOperandBits = params(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters { -} +abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters // Directionality of message channel @@ -62,7 +70,7 @@ trait ManagerToClientChannel extends TileLinkChannel trait ClientToClientChannel extends TileLinkChannel // Unused for now // Common signals that are used in multiple channels. -// These traits are useful for type parameterization. +// These traits are useful for type parameterizing bundle wiring functions. // trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -554,10 +562,10 @@ trait HasDataBeatCounters { def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int): Vec[Bool] = { - val id = in.bits.payload.client_xact_id + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && id === UInt(i), in.bits.payload, UInt(0))._2 + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 }) } @@ -615,41 +623,51 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0) extends TLModule +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) val finish = Decoupled(new LogicalNetworkIO(new Finish)) val ready = Bool(OUTPUT) - val grant_done = Bool(OUTPUT) - val pending_finish = Bool(OUTPUT) } - val entries = 1 << tlClientXactIdBits val g = io.grant.bits.payload - assert(g.client_xact_id <= UInt(entries), "No grant beat counter provisioned, only " + entries) - val done = connectIncomingDataBeatCounters(io.grant, entries).reduce(_||_) - val q = Module(new FinishQueue(entries)) + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounter(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits.fin := g.makeFinish() - q.io.enq.bits.dst := io.grant.bits.header.src + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.dst - io.finish.bits.payload := q.io.deq.bits.fin - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := io.grant.valid - io.refill.bits := io.grant.bits.payload - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && (io.refill.ready || !g.hasData()) - - io.ready := q.io.enq.ready - io.grant_done := done - io.pending_finish := q.io.deq.valid + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } } object TileLinkHeaderOverwriter { diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 62a53129..9d1c93b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -16,9 +16,6 @@ trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val nCoherentClients = params(NCoherentClients) - val nIncoherentClients = params(NIncoherentClients) - val nClients = nCoherentClients + nIncoherentClients def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) @@ -35,33 +32,15 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: Data : ClassTag]( - out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) - out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } - } - - def doOutputArbitration[T <: HasTileLinkData : ClassTag, S <: LogicalNetworkIO[T] : ClassTag]( - out: DecoupledIO[S], - ins: Seq[DecoupledIO[S]]) { + def doOutputArbitration[T <: TileLinkChannel : ClassTag]( + out: DecoupledIO[LogicalNetworkIO[T]], + ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( - out.bits.clone, - ins.size, - out.bits.payload.tlDataBeats, - lock _)) + val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) out <> arb.io.out arb.io.in zip ins map { case (a, in) => a <> in } } - def doInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id - outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } - } - def doInputRouting[T <: HasManagerTransactionId]( in: DecoupledIO[LogicalNetworkIO[T]], outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { @@ -74,7 +53,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new TileLinkIO)(innerTLParams).flip - val incoherent = Vec.fill(nCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits.payload def iprb(dummy: Int = 0) = inner.probe.bits.payload def irel(dummy: Int = 0) = inner.release.bits.payload @@ -173,6 +152,6 @@ abstract class XactTracker extends CoherenceAgentModule } def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(nCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) + ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) } } From ba7a8b175207e6712efddd7c4683e052cb8691c3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 17 Apr 2015 16:55:20 -0700 Subject: [PATCH 367/688] TileLink refactor; TileLinkPorts now available. L2Banks no longer have unique ids (suitable for hierarhical P&R). --- uncore/src/main/scala/broadcast.scala | 54 ++-- uncore/src/main/scala/cache.scala | 132 ++++---- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/memserdes.scala | 185 ++++++----- uncore/src/main/scala/metadata.scala | 41 ++- uncore/src/main/scala/tilelink.scala | 427 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 84 ++--- 7 files changed, 460 insertions(+), 465 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 340db34d..339a7540 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -25,16 +25,16 @@ object DataQueueLocation { } } -class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent +class L2BroadcastHub extends ManagerCoherenceAgent with BroadcastHubParameters { val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id, bankId), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id, bankId), {case TLDataBits => internalDataBits})) + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) // Propagate incoherence flags trackerList.map(_.io.incoherent := io.incoherent.toBits) @@ -61,7 +61,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent trackerAcquireIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.acquire.bits - tracker.bits.payload.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.bits.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } @@ -82,7 +82,7 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent case(tracker, i) => tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) tracker.bits := io.inner.release.bits - tracker.bits.payload.data := DataQueueLocation(rel_data_cnt, + tracker.bits.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits } @@ -91,17 +91,17 @@ class L2BroadcastHub(bankId: Int) extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles - io.inner.grant.bits.payload.data := io.outer.grant.bits.data - io.inner.grant.bits.payload.addr_beat := io.outer.grant.bits.addr_beat + io.inner.grant.bits.data := io.outer.grant.bits.data + io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new HeaderlessUncachedTileLinkIOArbiter(trackerList.size), + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) case TLDataBits => internalDataBits }) - outer_arb.io.in zip trackerList map { case(arb, t) => arb <> t.io.outer } + outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) val is_in_sdq = outer_data_ptr.loc === inStoreQueue @@ -124,12 +124,11 @@ class BroadcastXactTracker extends XactTracker { val io = new ManagerXactTrackerIO } -class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } val coh = ManagerMetadata.onReset @@ -150,9 +149,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? io.outer.acquire.bits := Bundle( @@ -175,7 +172,6 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(UInt(0)) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -207,12 +203,11 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends Broa } } -class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXactTracker { +class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } val coh = ManagerMetadata.onReset @@ -225,9 +220,9 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() - val probe_self = io.inner.acquire.bits.payload.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.header.src, width = io.inner.tlNCoherentClients) + val probe_self = io.inner.acquire.bits.requiresSelfProbe() + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits @@ -272,21 +267,17 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_p_id - io.inner.probe.bits.payload := coh.makeProbe(xact) + io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) io.inner.grant.valid := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent io.inner.acquire.ready := Bool(false) io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.iacq().client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && @@ -316,7 +307,6 @@ class BroadcastAcquireTracker(trackerId: Int, bankId: Int) extends BroadcastXact is(s_idle) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b193c5a7..8ae4d00b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -188,7 +188,7 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { ins: Seq[DecoupledIO[T]]) { val arb = Module(new RRArbiter(out.bits.clone, ins.size)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { @@ -342,14 +342,14 @@ class L2SecondaryMissInfo extends TLBundle with HasTileLinkBeatId with HasClientTransactionId -class L2HellaCacheBank(bankId: Int) extends HierarchicalCoherenceAgent +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) val meta = Module(new L2MetadataArray) // TODO: add delay knob val data = Module(new L2DataArray(1)) - val tshrfile = Module(new TSHRFile(bankId)) + val tshrfile = Module(new TSHRFile) tshrfile.io.inner <> io.inner io.outer <> tshrfile.io.outer io.incoherent <> tshrfile.io.incoherent @@ -362,24 +362,21 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile(bankId: Int) extends L2HellaCacheModule +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map { id => - Module(new L2VoluntaryReleaseTracker(id, bankId)) - } ++ (nReleaseTransactors until nTransactors).map { id => - Module(new L2AcquireTracker(id, bankId)) - } + val trackerList = (0 until nReleaseTransactors).map(id => Module(new L2VoluntaryReleaseTracker(id))) ++ + (nReleaseTransactors until nTransactors).map(id => Module(new L2AcquireTracker(id))) // WritebackUnit evicts data from L2, including invalidating L1s - val wb = Module(new L2WritebackUnit(nTransactors, bankId)) + val wb = Module(new L2WritebackUnit(nTransactors)) doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.incoherent) :+ wb.io.incoherent).map( _ := io.incoherent.toBits) + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent.toBits } // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) @@ -419,8 +416,8 @@ class TSHRFile(bankId: Int) extends L2HellaCacheModule // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new HeaderlessTileLinkIOArbiter(outerList.size))(outerTLParams) - outerList zip outer_arb.io.in map { case(out, arb) => out <> arb } + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size))(outerTLParams) + outer_arb.io.in <> outerList io.outer <> outer_arb.io.out // Wire local memory arrays @@ -480,21 +477,20 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) - def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isPartial = a.wmask() != Acquire.fullWriteMask - addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), in.bits.payload) + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } } -class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact_src = Reg(io.inner.release.bits.header.src.clone) - val xact = Reg(Bundle(new Release, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) val xact_tag_match = Reg{ Bool() } val xact_old_meta = Reg{ new L2Metadata } val xact_way_en = Reg{ Bits(width = nWays) } @@ -520,9 +516,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.inner.grant.valid := Bool(false) io.inner.finish.ready := Bool(false) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src - io.inner.grant.bits.payload := coh.inner.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) io.data.read.valid := Bool(false) io.data.write.valid := Bool(false) @@ -541,7 +535,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact, xact_src) + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write io.wb.req.valid := Bool(false) @@ -558,7 +552,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack is(s_idle) { io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { - xact_src := io.inner.release.bits.header.src xact := io.irel() data_buffer(io.irel().addr_beat) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() @@ -608,15 +601,14 @@ class L2VoluntaryReleaseTracker(trackerId: Int, bankId: Int) extends L2XactTrack } -class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) // State holding transaction metadata - val xact_src = Reg(io.inner.acquire.bits.header.src.clone) - val xact = Reg(Bundle(new Acquire, { case TLId => params(InnerTLId) })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } val xact_tag_match = Reg{ Bool() } @@ -631,16 +623,23 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // TODO add ignt.dst <- iacq.src // State holding progress made on processing this transaction - val iacq_data_done = - connectIncomingDataBeatCounter(io.inner.acquire) - val pending_irels = - connectTwoWayBeatCounter(io.inner.tlNCoherentClients, io.inner.probe, io.inner.release)._1 + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = connectTwoWayBeatCounter( + max = io.inner.tlNCoherentClients, + up = io.inner.probe, + down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = - connectHeaderlessTwoWayBeatCounter(1, io.outer.acquire, io.outer.grant, xact.addr_beat) - val (ignt_data_idx, ignt_data_done) = - connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) - val pending_ifins = - connectTwoWayBeatCounter(nSecondaryMisses, io.inner.grant, io.inner.finish, (g: Grant) => g.requiresAck())._1 + connectTwoWayBeatCounter( + max = 1, + up = io.outer.acquire, + down = io.outer.grant, + beat = xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = connectTwoWayBeatCounter( + max = nSecondaryMisses, + up = io.inner.grant, + down = io.inner.finish, + track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -695,9 +694,9 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } - def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[LogicalNetworkIO[T]]) { - when(in.fire() && in.bits.payload.hasData()) { - mergeData(innerDataBits)(in.bits.payload.addr_beat, in.bits.payload.data) + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) } } def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { @@ -713,7 +712,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // and Puts-under-Put from the same client val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && io.iacq().isBuiltInType(Acquire.getType)) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -728,7 +727,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { io.iacq().isBuiltInType(Acquire.putType)) || (xact.isBuiltInType(Acquire.putBlockType) && io.iacq().isBuiltInType(Acquire.putBlockType))) && - xact_src === io.inner.acquire.bits.header.src && //TODO + xact.client_id === io.iacq().client_id && //TODO remove xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && @@ -749,17 +748,13 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := pending_coh.inner.makeProbe(xact) + io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data io.inner.release.ready := state === s_inner_probe val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease( // Drop sharer - incoming = io.irel(), - src = io.inner.release.bits.header.src), + pending_coh.inner.onRelease(io.irel()), // Drop sharer Mux(io.irel().hasData(), // Dirty writeback pending_coh.outer.onHit(M_XWR), pending_coh.outer)) @@ -804,23 +799,18 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) - io.inner.grant.bits.header.src := UInt(bankId) - io.inner.grant.bits.header.dst := xact_src // TODO: ignt_q.io.deq.bits.src - io.inner.grant.bits.payload := pending_coh.inner.makeGrant( - acq = xact, - manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - // TODO: improve the ManagerMetadata.makeGrant to deal with possibility of - // multiple client transaction ids from merged secondary misses - io.ignt().client_xact_id := ignt_q.io.deq.bits.client_xact_id + io.inner.grant.bits := pending_coh.inner.makeGrant( + dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src + acq = xact, + client_xact_id = ignt_q.io.deq.bits.client_xact_id, + manager_xact_id = UInt(trackerId), + addr_beat = ignt_data_idx, + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant( - outgoing = io.ignt(), - dst = io.inner.grant.bits.header.dst), + pending_coh.inner.onGrant(io.ignt()), Mux(ognt_data_done, pending_coh_on_ognt.outer, pending_coh.outer)) @@ -904,7 +894,6 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { - xact_src := io.inner.acquire.bits.header.src xact := io.iacq() xact.data := UInt(0) pending_puts := Mux( // Make sure to collect all data from a PutBlock @@ -943,8 +932,8 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { val full_sharers = coh.inner.full() val mask_self = Mux( xact.requiresSelfProbe(), - coh.inner.full() | UIntToOH(xact_src), - coh.inner.full() & ~UIntToOH(xact_src)) + coh.inner.full() | UIntToOH(xact.client_id), + coh.inner.full() & ~UIntToOH(xact.client_id)) val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } @@ -984,7 +973,7 @@ class L2AcquireTracker(trackerId: Int, bankId: Int) extends L2XactTracker { // Checks for illegal behavior assert(!(state != s_idle && io.inner.acquire.fire() && - io.inner.acquire.bits.header.src != xact_src), + io.inner.acquire.bits.client_id != xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") } @@ -1007,7 +996,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { val data = new L2DataRWIO } -class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { +class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) @@ -1031,9 +1020,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) val resp_data_done = connectInternalDataBeatCounter(io.data.resp) - val pending_icoh_on_irel = xact_coh.inner.onRelease( - incoming = io.irel(), - src = io.inner.release.bits.header.src) + val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write val pending_ofin_on_ognt = io.ognt().makeFinish() @@ -1054,10 +1041,7 @@ class L2WritebackUnit(trackerId: Int, bankId: Int) extends L2XactTracker { io.outer.grant.ready := Bool(false) // default io.inner.probe.valid := Bool(false) - io.inner.probe.bits.header.src := UInt(bankId) - io.inner.probe.bits.header.dst := curr_probe_dst - io.inner.probe.bits.payload := - xact_coh.inner.makeProbeForVoluntaryWriteback(xact_addr_block) + io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) io.inner.grant.valid := Bool(false) io.inner.acquire.ready := Bool(false) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 787847b9..8a79cda1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -60,7 +60,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec.fill(nCores){new HTIFIO}.flip - val mem = new HeaderlessUncachedTileLinkIO + val mem = new ClientUncachedTileLinkIO val scr = new SCRIO } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 942874cf..7497e2e4 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -18,16 +18,26 @@ case object MIFDataBits extends Field[Int] case object MIFTagBits extends Field[Int] case object MIFDataBeats extends Field[Int] -trait HasMemData extends Bundle { - val data = Bits(width = params(MIFDataBits)) +trait MIFParameters extends UsesParameters { + val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) +} + +abstract class MIFBundle extends Bundle with MIFParameters +abstract class MIFModule extends Module with MIFParameters + +trait HasMemData extends MIFBundle { + val data = Bits(width = mifDataBits) } -trait HasMemAddr extends Bundle { - val addr = UInt(width = params(MIFAddrBits)) +trait HasMemAddr extends MIFBundle { + val addr = UInt(width = mifAddrBits) } -trait HasMemTag extends Bundle { - val tag = UInt(width = params(MIFTagBits)) +trait HasMemTag extends MIFBundle { + val tag = UInt(width = mifTagBits) } class MemReqCmd extends HasMemAddr with HasMemTag { @@ -56,7 +66,7 @@ class MemSerializedIO(w: Int) extends Bundle val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends Module +class MemSerdes(w: Int) extends MIFModule { val io = new Bundle { val wide = new MemIO().flip @@ -65,7 +75,6 @@ class MemSerdes(w: Int) extends Module val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) val out_buf = Reg(Bits()) val in_buf = Reg(Bits()) @@ -73,7 +82,7 @@ class MemSerdes(w: Int) extends Module val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_send_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) @@ -109,12 +118,12 @@ class MemSerdes(w: Int) extends Module } when (state === s_write_data && ddone) { data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(dbeats-1), s_idle, s_write_idle) + state := Mux(data_send_cnt === UInt(mifDataBeats-1), s_idle, s_write_idle) send_cnt := UInt(0) } val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val resp_val = Reg(init=Bool(false)) resp_val := Bool(false) @@ -143,11 +152,11 @@ class MemDesser(w: Int) extends Module // test rig side val abits = io.wide.req_cmd.bits.toBits.getWidth val dbits = io.wide.req_data.bits.toBits.getWidth val rbits = io.wide.resp.bits.getWidth - val dbeats = params(MIFDataBeats) + val mifDataBeats = params(MIFDataBeats) require(dbits >= abits && rbits >= dbits) val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(dbeats))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) @@ -175,13 +184,13 @@ class MemDesser(w: Int) extends Module // test rig side } when (state === s_data && io.wide.req_data.ready) { state := s_data_recv - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } data_recv_cnt := data_recv_cnt + UInt(1) } when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(dbeats-1)) { + when (data_recv_cnt === UInt(mifDataBeats-1)) { state := s_cmd_recv } recv_cnt := UInt(0) @@ -195,7 +204,7 @@ class MemDesser(w: Int) extends Module // test rig side io.wide.req_data.valid := state === s_data io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - val dataq = Module(new Queue(new MemResp, dbeats)) + val dataq = Module(new Queue(new MemResp, mifDataBeats)) dataq.io.enq <> io.wide.resp dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) @@ -203,19 +212,12 @@ class MemDesser(w: Int) extends Module // test rig side io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) } -//Adapter between a TileLinkIO and a UncachedTileLinkIO, merges voluntary - - //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemIO } - val mifTagBits = params(MIFTagBits) - val mifAddrBits = params(MIFAddrBits) - val mifDataBits = params(MIFDataBits) - val mifDataBeats = params(MIFDataBeats) val dataBits = tlDataBits*tlDataBeats val dstIdBits = params(LNHeaderBits) require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") @@ -227,12 +229,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.finish.ready := Bool(true) io.mem.resp.ready := Bool(false) - val gnt_arb = Module(new Arbiter(new LogicalNetworkIO(new Grant), 2)) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.payload.hasData() - val rel_has_data = io.tl.release.bits.payload.hasData() + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() // Decompose outgoing TL Acquires into MemIO cmd and data val active_out = Reg(init=Bool(false)) @@ -248,13 +250,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { val make_grant_ack = Reg(init=Bool(false)) gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits.payload := Grant( - is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> UInt(1), - manager_xact_id = UInt(0)) - gnt_arb.io.in(1).bits.header.dst := (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(1).bits.header.src := UInt(0) + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) @@ -263,7 +264,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { mem_data_q.io.enq.valid := Bool(false) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.payload.data.clone) } + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.data.clone) } val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) @@ -275,27 +276,27 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - addr_out := io.tl.release.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + addr_out := io.tl.release.bits.addr_block has_data := rel_has_data data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() + make_grant_ack := io.tl.release.bits.requiresAck() tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.release.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.release.bits.data } .elsewhen(io.tl.acquire.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - addr_out := io.tl.acquire.bits.payload.addr_block + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + addr_out := io.tl.acquire.bits.addr_block has_data := acq_has_data data_from_rel := Bool(false) make_grant_ack := acq_has_data tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.payload.data + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data } } when(active_out) { @@ -306,8 +307,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.tl.acquire.ready := Bool(true) when(io.tl.acquire.valid) { tl_buf_out(tl_cnt_out) := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) } } when(!mif_done_out) { @@ -338,8 +339,8 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { io.mem.req_cmd.bits.tag := tag_out io.mem.req_cmd.bits.addr := addr_out io.mem.req_data.bits.data := Mux(data_from_rel, - io.tl.release.bits.payload.data, - io.tl.acquire.bits.payload.data) + io.tl.release.bits.data, + io.tl.acquire.bits.data) when(!active_out){ io.tl.release.ready := io.mem.req_data.ready io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid @@ -352,12 +353,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.payload.requiresAck() - io.mem.req_data.bits.data := io.tl.release.bits.payload.data - val tag = Cat(io.tl.release.bits.header.src, - io.tl.release.bits.payload.client_xact_id, - io.tl.release.bits.payload.isVoluntary()) - val addr = io.tl.release.bits.payload.addr_block + make_grant_ack := io.tl.release.bits.requiresAck() + io.mem.req_data.bits.data := io.tl.release.bits.data + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := rel_has_data @@ -367,12 +368,12 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) make_grant_ack := acq_has_data // i.e. is it a Put - io.mem.req_data.bits.data := io.tl.acquire.bits.payload.data + io.mem.req_data.bits.data := io.tl.acquire.bits.data io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.header.src, - io.tl.acquire.bits.payload.client_xact_id, - io.tl.acquire.bits.payload.isBuiltInType()) - val addr = io.tl.acquire.bits.payload.addr_block + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.addr_block io.mem.req_cmd.bits.tag := tag io.mem.req_cmd.bits.addr := addr io.mem.req_cmd.bits.rw := acq_has_data @@ -407,28 +408,25 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { // Aggregate incoming MemIO responses into TL Grants val active_in = Reg(init=Bool(false)) - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.payload.hasMultibeatData(), tlDataBeats) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val tag_in = Reg(UInt(width = mifTagBits)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.payload.data.clone } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = tag_in(0), - g_type = Mux(tag_in(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) when(!active_in) { io.mem.resp.ready := Bool(true) @@ -453,17 +451,14 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule { } else { // Don't generate all the uneeded data buffers and flow resp gnt_arb.io.in(0).valid := io.mem.resp.valid io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits.payload := Grant( - is_builtin_type = io.mem.resp.bits.tag(0), - g_type = Mux(io.mem.resp.bits.tag(0), - Grant.getDataBlockType, - UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) - gnt_arb.io.in(0).bits.header.dst := (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)) - gnt_arb.io.in(0).bits.header.src := UInt(0) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) } } @@ -522,28 +517,28 @@ object HellaQueue } } -class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Module { +class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip val mem = new MemPipeIO } - val numEntries = numRequests * refillCycles + val numEntries = numRequests * mifDataBeats val size = log2Down(numEntries) + 1 val inc = Bool() val dec = Bool() val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(refillCycles) + val watermark = count >= UInt(mifDataBeats) when (inc && !dec) { count := count + UInt(1) } when (!inc && dec) { - count := count - UInt(refillCycles) + count := count - UInt(mifDataBeats) } when (inc && dec) { - count := count - UInt(refillCycles-1) + count := count - UInt(mifDataBeats-1) } val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark @@ -573,17 +568,17 @@ class MemPipeIOMemIOConverter(numRequests: Int, refillCycles: Int) extends Modul dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw } -class MemPipeIOTileLinkIOConverter(outstanding: Int, refillCycles: Int) extends Module { +class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { val io = new Bundle { - val tl = new TileLinkIO().flip + val tl = new ManagerTileLinkIO val mem = new MemPipeIO } val a = Module(new MemIOTileLinkIOConverter(1)) - val b = Module(new MemPipeIOMemIOConverter(outstanding, refillCycles)) + val b = Module(new MemPipeIOMemIOConverter(outstanding)) a.io.tl <> io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, refillCycles, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) a.io.mem.resp <> b.io.cpu.resp b.io.mem <> io.mem } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index af5504c8..4e8082ee 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -108,17 +108,18 @@ class ManagerMetadata extends CoherenceMetadata { def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(cmd: UInt, addr_block: UInt): Probe = - Bundle(Probe(co.getProbeType(cmd, this), addr_block), { case TLId => id }) + def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - def makeProbe(acq: Acquire): Probe = - Bundle(Probe(co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) - def makeProbeForVoluntaryWriteback(addr_block: UInt): Probe = - makeProbe(M_FLUSH, addr_block) + def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = + makeProbe(dst, M_FLUSH, addr_block) - def makeGrant(rel: Release, manager_xact_id: UInt): Grant = { + def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( + dst = rel.client_id, is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, @@ -126,11 +127,12 @@ class ManagerMetadata extends CoherenceMetadata { } def makeGrant( - acq: Acquire, + acq: AcquireFromSrc, manager_xact_id: UInt, addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + data: UInt = UInt(0)): GrantToDst = { Bundle(Grant( + dst = acq.client_id, is_builtin_type = acq.isBuiltInType(), g_type = Mux(acq.isBuiltInType(), acq.getBuiltInGrantType(), @@ -141,11 +143,24 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } - def onRelease(incoming: Release, src: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnRelease(incoming, src, this), { case TLId => id }) + def makeGrant( + dst: UInt, + acq: AcquireFromSrc, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt, + data: UInt): GrantToDst = { + val g = makeGrant(acq, manager_xact_id, addr_beat, data) + g.client_id := dst + g.client_xact_id := client_xact_id + g + } + + def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) - def onGrant(outgoing: Grant, dst: UInt): ManagerMetadata = - Bundle(co.managerMetadataOnGrant(outgoing, dst, this), { case TLId => id }) + def onGrant(outgoing: GrantToDst): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } object ManagerMetadata { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1ed95df6..45cdb454 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -23,12 +23,14 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] case object TLNetworkIsOrderedP2P extends Field[Boolean] -abstract trait TileLinkParameters extends UsesParameters { +trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) val tlNCoherentClients = params(TLNCoherentClients) val tlNIncoherentClients = params(TLNIncoherentClients) + val tlClientIdBits = log2Up(tlNClients) + val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) val tlMaxClientPorts = params(TLMaxClientPorts) val tlMaxManagerXacts = params(TLMaxManagerXacts) @@ -98,6 +100,10 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +trait HasClientId extends TLBundle { + val client_id = UInt(width = tlClientIdBits) +} + // Actual TileLink channel bundle definitions class Acquire extends ClientToManagerChannel @@ -350,12 +356,19 @@ class Probe extends ManagerToClientChannel } object Probe { - def apply(p_type: UInt, addr_block: UInt) = { + def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe prb.p_type := p_type prb.addr_block := addr_block prb } + def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { + val prb = new ProbeToDst + prb.client_id := dst + prb.p_type := p_type + prb.addr_block := addr_block + prb + } } class Release extends ClientToManagerChannel @@ -435,8 +448,8 @@ object Grant { g_type: UInt, client_xact_id: UInt, manager_xact_id: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Grant = { + addr_beat: UInt, + data: UInt): Grant = { val gnt = new Grant gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type @@ -446,6 +459,25 @@ object Grant { gnt.data := data gnt } + + def apply( + dst: UInt, + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = { + val gnt = new GrantToDst + gnt.client_id := dst + gnt.is_builtin_type := is_builtin_type + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat + gnt.data := data + gnt + } } class Finish extends ClientToManagerChannel with HasManagerTransactionId { @@ -453,7 +485,14 @@ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// Complete IO definitions for two types of TileLink clients +// These subtypes include a field for the source or destination ClientId +class AcquireFromSrc extends Acquire with HasClientId +class ProbeToDst extends Probe with HasClientId +class ReleaseFromSrc extends Release with HasClientId +class GrantToDst extends Grant with HasClientId + +// Complete IO definitions for two types of TileLink clients, including +// networking headers class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip @@ -479,28 +518,36 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } -// This version of TileLinkIO does not contain network headers for packets -// that originate in the Clients (i.e. Acquire and Release). These headers -// are provided in the top-level that instantiates the clients and network. +// This version of TileLinkIO does not contain network headers. The headers +// are provided in the top-level that instantiates the clients and network, +// probably using a TileLinkClientPort module. // By eliding the header subbundles within the clients we can enable // hierarchical P&R while minimizing unconnected port errors in GDS. // Secondly, this version of the interface elides Finish messages, with the // assumption that a FinishUnit has been coupled to the TileLinkIO port // to deal with acking received Grants. -class HeaderlessUncachedTileLinkIO extends TLBundle { +class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } -class HeaderlessTileLinkIO extends HeaderlessUncachedTileLinkIO { +class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } -class HeaderlessTileLinkIOWrapper extends TLModule { +class ManagerTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new AcquireFromSrc).flip + val grant = new DecoupledIO(new GrantToDst) + val finish = new DecoupledIO(new Finish).flip + val probe = new DecoupledIO(new ProbeToDst) + val release = new DecoupledIO(new ReleaseFromSrc).flip +} + +class ClientTileLinkIOWrapper extends TLModule { val io = new Bundle { - val in = new HeaderlessUncachedTileLinkIO().flip - val out = new HeaderlessTileLinkIO + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO } io.out.acquire <> io.in.acquire io.out.grant <> io.in.grant @@ -509,17 +556,17 @@ class HeaderlessTileLinkIOWrapper extends TLModule { } object TileLinkIOWrapper { - def apply(utl: HeaderlessUncachedTileLinkIO, p: Parameters): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper)(p) + def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper)(p) conv.io.in <> utl conv.io.out } - def apply(utl: HeaderlessUncachedTileLinkIO): HeaderlessTileLinkIO = { - val conv = Module(new HeaderlessTileLinkIOWrapper) + def apply(utl: ClientUncachedTileLinkIO): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) conv.io.in <> utl conv.io.out } - def apply(tl: HeaderlessTileLinkIO): HeaderlessTileLinkIO = tl + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { val conv = Module(new TileLinkIOWrapper)(p) conv.io.in <> utl @@ -533,89 +580,6 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - type HasClientId = TileLinkChannel with HasClientTransactionId - type LNAcquire = LogicalNetworkIO[Acquire] - type LNRelease = LogicalNetworkIO[Release] - type LNGrant = LogicalNetworkIO[Grant] - - def connectDataBeatCounter[S <: TileLinkChannel : ClassTag](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - def connectOutgoingDataBeatCounter[T <: Data : TypeTag]( - in: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, beat) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, beat) - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); (UInt(0), Bool(false))} - } - } - - def connectIncomingDataBeatCounters[T <: HasClientId : ClassTag]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - def connectIncomingDataBeatCounter[T <: Data : TypeTag](in: DecoupledIO[T]): Bool = { - in.bits match { - case p: TileLinkChannel if typeTag[T].tpe <:< typeTag[TileLinkChannel].tpe => - connectDataBeatCounter(in.fire(), p, UInt(0))._2 - case ln: LNAcquire if typeTag[T].tpe =:= typeTag[LNAcquire].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNRelease if typeTag[T].tpe =:= typeTag[LNRelease].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case ln: LNGrant if typeTag[T].tpe =:= typeTag[LNGrant].tpe => - connectDataBeatCounter(in.fire(), ln.payload, UInt(0))._2 - case _ => { require(false, "Don't know how to connect a beat counter to " + typeTag[T].tpe); Bool(false)} - } - } - - def connectHeaderlessTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[T], - down: DecoupledIO[S], - beat: UInt): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, do_inc) = connectDataBeatCounter(up.fire(), up.bits, beat) - val (down_idx, do_dec) = connectDataBeatCounter(down.fire(), down.bits, beat) - //Module.assert(!(do_dec && cnt === UInt(0)), "Decrementing 2way beat counter before ever incrementing") - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, do_inc, down_idx, do_dec) - } - - def connectTwoWayBeatCounter[ T <: TileLinkChannel : ClassTag, S <: TileLinkChannel : ClassTag]( - max: Int, - up: DecoupledIO[LogicalNetworkIO[T]], - down: DecoupledIO[LogicalNetworkIO[S]], - inc: T => Bool = (t: T) => Bool(true), - dec: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) - val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits.payload, UInt(0)) - val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits.payload, UInt(0)) - val do_inc = up_done && inc(up.bits.payload) - val do_dec = down_done && dec(down.bits.payload) - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) - (cnt > UInt(0), up_idx, up_done, down_idx, down_done) - } -} - class FinishQueueEntry extends TLBundle { val fin = new Finish val dst = UInt(width = log2Up(params(LNEndpoints))) @@ -623,8 +587,7 @@ class FinishQueueEntry extends TLBundle { class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) -class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule - with HasDataBeatCounters { +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) @@ -645,12 +608,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule // a multibeat Grant. But Grants from multiple managers or transactions may // get interleaved, so we could need a counter for each. val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounter(io.grant) + connectIncomingDataBeatCounterWithHeader(io.grant) } else { val entries = 1 << tlClientXactIdBits def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCounters(io.grant, entries, getId).reduce(_||_) + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) } val q = Module(new FinishQueue(outstanding)) q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) @@ -670,30 +633,7 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule } } -object TileLinkHeaderOverwriter { - def apply[T <: ClientToManagerChannel]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - passThrough: Boolean): DecoupledIO[LogicalNetworkIO[T]] = { - val out = in.clone.asDirectionless - out.bits.payload := in.bits.payload - out.bits.header.src := UInt(clientId) - out.bits.header.dst := (if(passThrough) in.bits.header.dst else UInt(0)) - out.valid := in.valid - in.ready := out.ready - out - } - - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[LogicalNetworkIO[T]], - clientId: Int, - nBanks: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out: DecoupledIO[LogicalNetworkIO[T]] = apply(in, clientId, false) - out.bits.header.dst := addrConvert(in.bits.payload.addr_block) - out - } - +object ClientTileLinkHeaderCreator { def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( in: DecoupledIO[T], clientId: Int, @@ -708,9 +648,24 @@ object TileLinkHeaderOverwriter { } } -class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { - val client = new HeaderlessTileLinkIO().flip + val client = new ClientTileLinkIO().flip val network = new TileLinkIO } @@ -718,8 +673,8 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo finisher.io.grant <> io.network.grant io.network.finish <> finisher.io.finish - val acq_with_header = TileLinkHeaderOverwriter(io.client.acquire, clientId, addrConvert) - val rel_with_header = TileLinkHeaderOverwriter(io.client.release, clientId, addrConvert) + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) val gnt_without_header = finisher.io.refill @@ -731,58 +686,63 @@ class TileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLMo io.client.grant <> gnt_without_header } -object TileLinkNetworkPort { - def apply[T <: Data]( - client: HeaderlessTileLinkIO, - clientId: Int = 0, - addrConvert: UInt => UInt = u => UInt(0))(implicit p: Parameters): TileLinkIO = { - val port = Module(new TileLinkNetworkPort(clientId, addrConvert))(p) - port.io.client <> client - port.io.network +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } -class TileLinkEnqueuer(depths: (Int, Int, Int, Int, Int)) extends Module { +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip val manager = new TileLinkIO } - io.manager.acquire <> (if(depths._1 > 0) Queue(io.client.acquire, depths._1) else io.client.acquire) - io.client.probe <> (if(depths._2 > 0) Queue(io.manager.probe, depths._2) else io.manager.probe) - io.manager.release <> (if(depths._3 > 0) Queue(io.client.release, depths._3) else io.client.release) - io.client.grant <> (if(depths._4 > 0) Queue(io.manager.grant, depths._4) else io.manager.grant) - io.manager.finish <> (if(depths._5 > 0) Queue(io.client.finish, depths._5) else io.client.finish) + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) } object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: (Int, Int, Int, Int, Int))(p: Parameters): TileLinkIO = { + def apply(in: TileLinkIO, depths: TileLinkDepths)(p: Parameters): TileLinkIO = { val t = Module(new TileLinkEnqueuer(depths))(p) t.io.client <> in t.io.manager } def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { - apply(in, (depth, depth, depth, depth, depth))(p) + apply(in, TileLinkDepths(depth, depth, depth, depth, depth))(p) } } -abstract trait HasArbiterTypes { +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends TileLinkParameters { + // Some shorthand type variables type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId - type ClientSourcedWithIdAndData = ClientToManagerChannel with - HasClientTransactionId with - HasTileLinkData -} + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData -// Utility functions for constructing TileLinkIO arbiters -trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ - val arbN: Int - // These are filled in depending on whether the arbiter mucks with the - // client ids and then needs to revert them on the way back + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits def arbIdx(in: ManagerSourcedWithId): UInt - def hookupClientSource[M <: ClientSourcedWithIdAndData : ClassTag]( + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() @@ -796,7 +756,7 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ arb.io.out <> mngr } - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData : ClassTag]( + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() @@ -854,23 +814,20 @@ trait TileLinkArbiterLike extends HasArbiterTypes with TileLinkParameters{ } } - def hookupManagerSourceBroadcast[M <: Data]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { clts.map{ _.valid := mngr.valid } clts.map{ _.bits := mngr.bits } mngr.ready := clts.map(_.ready).reduce(_&&_) } - def hookupFinish[M <: LogicalNetworkIO[Finish] : ClassTag]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) - arb.io.in zip clts map { case (arb, req) => arb <> req } + arb.io.in <> clts arb.io.out <> mngr } } +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip @@ -881,6 +838,7 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with Tile hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { val in = Vec.fill(arbN){new TileLinkIO}.flip @@ -893,31 +851,7 @@ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbi hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) } -class HeaderlessUncachedTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessUncachedTileLinkIO}.flip - val out = new HeaderlessUncachedTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -class HeaderlessTileLinkIOArbiter(val arbN: Int) extends Module - with TileLinkArbiterLike - with AppendsArbiterId { - val io = new Bundle { - val in = Vec.fill(arbN){new HeaderlessTileLinkIO}.flip - val out = new HeaderlessTileLinkIO - } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) -} - -// Appends the port index of the arbiter to the client_xact_id +/** Appends the port index of the arbiter to the client_xact_id */ trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) @@ -926,24 +860,125 @@ trait AppendsArbiterId extends TileLinkArbiterLike { def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } -// Uses the client_xact_id as is (assumes it has been set to port index) +/** Uses the client_xact_id as is (assumes it has been set to port index) */ trait PassesId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Overwrites some default client_xact_id with the port idx +/** Overwrites some default client_xact_id with the port idx */ trait UsesNewId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id } -// Mix-in id generation traits to make concrete arbiter classes +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientUncachedTileLinkIO}.flip + val out = new ClientUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientTileLinkIO}.flip + val out = new ClientTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular TileLinkChannel or pair of channels. + * + * Won't count message types that don't have data. + * Used in XactTrackers and FinishUnit. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing DecoupledIOs */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(in.fire(), in.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param max max number of outstanding ups with no down + * @param up outgoing channel + * @param down incoming channel + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt = UInt(0), + track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) + val do_inc = up_done && track(up.bits) + val do_dec = down_done + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 9d1c93b7..8dda15b7 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -32,19 +32,19 @@ abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { - def doOutputArbitration[T <: TileLinkChannel : ClassTag]( - out: DecoupledIO[LogicalNetworkIO[T]], - ins: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - def lock(o: LogicalNetworkIO[T]) = o.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter( out.bits.clone, ins.size, out.bits.payload.tlDataBeats, lock _)) + def doOutputArbitration[T <: TileLinkChannel]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + def lock(o: T) = o.hasMultibeatData() + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, out.bits.tlDataBeats, lock _)) out <> arb.io.out - arb.io.in zip ins map { case (a, in) => a <> in } + arb.io.in <> ins } def doInputRouting[T <: HasManagerTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - outs: Seq[DecoupledIO[LogicalNetworkIO[T]]]) { - val idx = in.bits.payload.manager_xact_id + in: DecoupledIO[T], + outs: Seq[DecoupledIO[T]]) { + val idx = in.bits.manager_xact_id outs.map(_.bits := in.bits) outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } in.ready := Vec(outs.map(_.ready)).read(idx) @@ -52,23 +52,23 @@ trait HasCoherenceAgentWiringHelpers { } trait HasInnerTLIO extends CoherenceAgentBundle { - val inner = Bundle(new TileLinkIO)(innerTLParams).flip + val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput - def iacq(dummy: Int = 0) = inner.acquire.bits.payload - def iprb(dummy: Int = 0) = inner.probe.bits.payload - def irel(dummy: Int = 0) = inner.release.bits.payload - def ignt(dummy: Int = 0) = inner.grant.bits.payload - def ifin(dummy: Int = 0) = inner.finish.bits.payload + def iacq(dummy: Int = 0) = inner.acquire.bits + def iprb(dummy: Int = 0) = inner.probe.bits + def irel(dummy: Int = 0) = inner.release.bits + def ignt(dummy: Int = 0) = inner.grant.bits + def ifin(dummy: Int = 0) = inner.finish.bits } trait HasUncachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessUncachedTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends CoherenceAgentBundle { - val outer = Bundle(new HeaderlessTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits def orel(dummy: Int = 0) = outer.release.bits @@ -78,8 +78,8 @@ trait HasCachedOuterTLIO extends CoherenceAgentBundle { class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO abstract class CoherenceAgent extends CoherenceAgentModule { - def innerTL: TileLinkIO - def outerTL: HeaderlessTileLinkIO + def innerTL: ManagerTileLinkIO + def outerTL: ClientTileLinkIO def incoherent: Vec[Bool] } @@ -109,49 +109,25 @@ trait HasTrackerConflictIO extends Bundle { class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule - with HasDataBeatCounters { +abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) - def addPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - addPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - addPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[LogicalNetworkIO[Acquire]]): UInt = { - val a = in.bits.payload + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits val isGetOrAtomic = a.isBuiltInType() && - (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) - addPendingBitWhenBeat(in.fire() && isGetOrAtomic, in.bits.payload) + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) } - def dropPendingBitWhenBeatHasData[T <: Data : TypeTag](in: DecoupledIO[T]): UInt = { - in.bits match { - case p: HasBeat if typeTag[T].tpe <:< typeTag[HasBeat].tpe => - dropPendingBitWhenBeat(in.fire() && p.hasData(), p) - case ln: LNAcquire if typeTag[T].tpe <:< typeTag[LNAcquire].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNRelease if typeTag[T].tpe <:< typeTag[LNRelease].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case ln: LNGrant if typeTag[T].tpe <:< typeTag[LNGrant].tpe => - dropPendingBitWhenBeat(in.fire() && ln.payload.hasData(), ln.payload) - case _ => { require(false, "Don't know how track beats of " + typeTag[T].tpe); UInt(0) } - } - } + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def dropPendingBitAtDest(in: DecoupledIO[LogicalNetworkIO[Probe]]): UInt = { - ~Fill(in.bits.payload.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.header.dst) - } + def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = + ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From 6d40a610602e3250cd247cacb5d84722914f01d5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 19 Apr 2015 22:06:44 -0700 Subject: [PATCH 368/688] TileLink scala doc and parameter renaming --- uncore/src/main/scala/broadcast.scala | 8 +- uncore/src/main/scala/cache.scala | 8 +- uncore/src/main/scala/metadata.scala | 10 +- uncore/src/main/scala/tilelink.scala | 500 +++++++++++++++++++------- uncore/src/main/scala/uncore.scala | 10 +- 5 files changed, 381 insertions(+), 155 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 339a7540..b554b231 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -216,13 +216,13 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { Acquire.prefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO - val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCoherentClients)) + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) val curr_p_id = PriorityEncoder(pending_probes) val full_sharers = coh.full() val probe_self = io.inner.acquire.bits.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCoherentClients) + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) val mask_incoherent = mask_self & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 8ae4d00b..75bb9e1f 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -625,7 +625,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) val pending_irels = connectTwoWayBeatCounter( - max = io.inner.tlNCoherentClients, + max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = @@ -641,7 +641,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { down = io.inner.finish, track = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -1009,8 +1009,8 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val xact_id = Reg{ UInt() } val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCoherentClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCoherentClients)) + val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val curr_probe_dst = PriorityEncoder(pending_probes) val full_sharers = io.wb.req.bits.coh.inner.full() val mask_incoherent = full_sharers & ~io.incoherent.toBits diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 4e8082ee..cf52d286 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -9,11 +9,11 @@ abstract class CoherenceMetadata extends Bundle { val id = params(TLId) } -/* The ClientMetadata stores the client-side coherence information, - such as permissions on the data and whether the data is dirty. - Its API can be used to make TileLink messages in response to - memory operations or TileLink Probes. -*/ +/** Stores the client-side coherence information, + * such as permissions on the data and whether the data is dirty. + * Its API can be used to make TileLink messages in response to + * memory operations or [[uncore.Probe]] messages. + */ class ClientMetadata extends CoherenceMetadata { val state = UInt(width = co.clientStateWidth) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 45cdb454..b759d48b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -3,38 +3,50 @@ package uncore import Chisel._ import scala.math.max -import scala.reflect._ -import scala.reflect.runtime.universe._ -// Parameters exposed to the top-level design, set based on -// external requirements or design space exploration -// -case object TLId extends Field[String] // Unique name per network +/** Parameters exposed to the top-level design, set based on + * external requirements or design space exploration + */ +/** Unique name per TileLink network*/ +case object TLId extends Field[String] +/** Coherency policy used to define custom mesage types */ case object TLCoherencePolicy extends Field[CoherencePolicy] -case object TLNManagers extends Field[Int] +/** Number of manager agents */ +case object TLNManagers extends Field[Int] +/** Number of client agents */ case object TLNClients extends Field[Int] -case object TLNCoherentClients extends Field[Int] -case object TLNIncoherentClients extends Field[Int] +/** Number of client agents that cache data and use custom [[uncore.Acquire]] types */ +case object TLNCachingClients extends Field[Int] +/** Number of client agents that do not cache data and use built-in [[uncoreAcquire]] types */ +case object TLNCachelessClients extends Field[Int] +/** Maximum number of unique outstanding transactions per client */ case object TLMaxClientXacts extends Field[Int] -case object TLMaxClientPorts extends Field[Int] +/** Maximum number of clients multiplexed onto a single port */ +case object TLMaxClientsPerPort extends Field[Int] +/** Maximum number of unique outstanding transactions per manager */ case object TLMaxManagerXacts extends Field[Int] +/** Width of cache block addresses */ case object TLBlockAddrBits extends Field[Int] +/** Width of data beats */ case object TLDataBits extends Field[Int] +/** Number of data beats per cache block */ case object TLDataBeats extends Field[Int] +/** Whether the underlying physical network preserved point-to-point ordering of messages */ case object TLNetworkIsOrderedP2P extends Field[Boolean] +/** Utility trait for building Modules and Bundles that use TileLink parameters */ trait TileLinkParameters extends UsesParameters { val tlCoh = params(TLCoherencePolicy) val tlNManagers = params(TLNManagers) val tlNClients = params(TLNClients) - val tlNCoherentClients = params(TLNCoherentClients) - val tlNIncoherentClients = params(TLNIncoherentClients) + val tlNCachingClients = params(TLNCachingClients) + val tlNCachelessClients = params(TLNCachelessClients) val tlClientIdBits = log2Up(tlNClients) val tlManagerIdBits = log2Up(tlNManagers) val tlMaxClientXacts = params(TLMaxClientXacts) - val tlMaxClientPorts = params(TLMaxClientPorts) + val tlMaxClientsPerPort = params(TLMaxClientsPerPort) val tlMaxManagerXacts = params(TLMaxManagerXacts) - val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientPorts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) val tlBlockAddrBits = params(TLBlockAddrBits) val tlDataBits = params(TLDataBits) @@ -61,19 +73,23 @@ trait TileLinkParameters extends UsesParameters { abstract class TLBundle extends Bundle with TileLinkParameters abstract class TLModule extends Module with TileLinkParameters -// Directionality of message channel -// Used to hook up logical network ports to physical network ports +/** Base trait for all TileLink channels */ trait TileLinkChannel extends TLBundle { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToManagerChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ManagerToClientChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ trait ClientToClientChannel extends TileLinkChannel // Unused for now -// Common signals that are used in multiple channels. -// These traits are useful for type parameterizing bundle wiring functions. -// +/** Common signals that are used in multiple channels. + * These traits are useful for type parameterizing bundle wiring functions. + */ + +/** Address of a cache block. */ trait HasCacheBlockAddress extends TLBundle { val addr_block = UInt(width = tlBlockAddrBits) @@ -81,18 +97,22 @@ trait HasCacheBlockAddress extends TLBundle { def conflicts(addr: UInt) = this.addr_block === addr } +/** Sub-block address or beat id of multi-beat data */ trait HasTileLinkBeatId extends TLBundle { val addr_beat = UInt(width = tlBeatAddrBits) } +/* Client-side transaction id. Usually Miss Status Handling Register File index */ trait HasClientTransactionId extends TLBundle { val client_xact_id = Bits(width = tlClientXactIdBits) } +/** Manager-side transaction id. Usually Transaction Status Handling Register File index. */ trait HasManagerTransactionId extends TLBundle { val manager_xact_id = Bits(width = tlManagerXactIdBits) } +/** A single beat of cache block data */ trait HasTileLinkData extends HasTileLinkBeatId { val data = UInt(width = tlDataBits) @@ -100,61 +120,90 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +/** The id of a client source or destination. Used in managers. */ trait HasClientId extends TLBundle { val client_id = UInt(width = tlClientIdBits) } -// Actual TileLink channel bundle definitions +/** TileLink channel bundle definitions */ +/** The Acquire channel is used to intiate coherence protocol transactions in + * order to gain access to a cache block's data with certain permissions + * enabled. Messages sent over this channel may be custom types defined by + * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types + * used for uncached data accesses. Acquires may contain data for Put or + * PutAtomic built-in types. After sending an Acquire, clients must + * wait for a manager to send them a [[uncore.Grant]] message in response. + */ class Acquire extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { - // Actual bundle fields + // Actual bundle fields: val is_builtin_type = Bool() val a_type = UInt(width = tlAcquireTypeBits) val union = Bits(width = tlAcquireUnionBits) - // Utility funcs for accessing subblock union + // Utility funcs for accessing subblock union: val opCodeOff = 1 val opSizeOff = tlMemoryOpcodeBits + opCodeOff val addrByteOff = tlMemoryOperandSizeBits + opSizeOff val addrByteMSB = tlByteAddrBits + addrByteOff + /** Hint whether to allocate the block in any interveneing caches */ def allocate(dummy: Int = 0) = union(0) - def op_code(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), + /** Op code for [[uncore.PutAtomic]] operations */ + def op_code(dummy: Int = 0) = Mux( + isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), M_XWR, union(opSizeOff-1, opCodeOff)) + /** Operand size for [[uncore.PutAtomic]] */ def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + /** Bit offset of [[uncore.PutAtomic]] operand */ def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0) = Mux(isBuiltInType(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) + /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) - + /** Complete physical address for block, beat or operand */ def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) - // Other helper funcs + // Other helper functions: + /** Message type equality */ def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? + /** Is this message a built-in or custom type */ def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + /** Is this message a particular built-in type */ def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t + /** Does this message refer to subblock operands using info in the Acquire.union subbundle */ def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + /** Is this message a built-in prefetch message */ def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) - // Assumes no custom types have data + /** Does this message contain data? Assumes that no custom message types have data. */ def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + /** Does this message contain multiple beats of data? Assumes that no custom message types have data. */ def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) + /** Does this message require the manager to probe the client the very client that sent it? + * Needed if multiple caches are attached to the same port. + */ def requiresSelfProbe(dummy: Int = 0) = Bool(false) + /** Mapping between each built-in Acquire type (defined in companion object) + * and a built-in Grant type. + */ def getBuiltInGrantType(dummy: Int = 0): UInt = { MuxLookup(this.a_type, Grant.putAckType, Array( Acquire.getType -> Grant.getDataBeatType, @@ -166,15 +215,33 @@ class Acquire extends ClientToManagerChannel } } +/** [[uncore.Acquire]] with an extra field stating its source id */ +class AcquireFromSrc extends Acquire with HasClientId + +/** Contains definitions of the the built-in Acquire types and a factory + * for [[uncore.Acquire]] + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeAcquire]] for custom cached Acquires and + * [[uncore.Get]], [[uncore.Put]], etc. for built-in uncached Acquires. + * + * @param is_builtin_type built-in or custom type message? + * @param a_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being put outwards + * @param union additional fields used for uncached types + */ object Acquire { val nBuiltInTypes = 5 //TODO: Use Enum - def getType = UInt("b000") - def getBlockType = UInt("b001") - def putType = UInt("b010") - def putBlockType = UInt("b011") - def putAtomicType = UInt("b100") - def prefetchType = UInt("b101") + def getType = UInt("b000") // Get a single beat of data + def getBlockType = UInt("b001") // Get a whole block of data + def putType = UInt("b010") // Put a single beat of data + def putBlockType = UInt("b011") // Put a whole block of data + def putAtomicType = UInt("b100") // Perform an atomic memory op + def prefetchType = UInt("b101") // Prefetch a whole block of data def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -208,7 +275,16 @@ object Acquire { } } -// Asks for a single TileLink beat of data +/** Get a single beat of data from the outer memory hierarchy + * + * The client can hint whether he block containing this beat should be + * allocated in the intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param alloc hint whether the block should be allocated in intervening caches + */ object Get { def apply( client_xact_id: UInt, @@ -225,7 +301,15 @@ object Get { } } -// Asks for an entire cache block of data +/** Get a whole cache block of data from the outer memory hierarchy + * + * The client can hint whether the block should be allocated in the + * intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param alloc hint whether the block should be allocated in intervening caches + */ object GetBlock { def apply( client_xact_id: UInt = UInt(0), @@ -240,8 +324,12 @@ object GetBlock { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with read permissions +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with read permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ object GetPrefetch { def apply( client_xact_id: UInt, @@ -256,7 +344,16 @@ object GetPrefetch { } } -// Writes up to a single TileLink beat of data, using mask +/** Put a single beat of data into the outer memory hierarchy + * + * The block will be allocated in the next-outermost level of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + */ object Put { def apply( client_xact_id: UInt, @@ -275,7 +372,19 @@ object Put { } } -// Writes an entire cache block of data +/** Put a whole cache block of data into the outer memory hierarchy + * + * If the write mask is not full, the block will be allocated in the + * next-outermost level of the hierarchy. If the write mask is full, the + * client can hint whether the block should be allocated or not. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat of several) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + * @param alloc hint whether the block should be allocated in intervening caches + */ object PutBlock { def apply( client_xact_id: UInt, @@ -309,7 +418,36 @@ object PutBlock { } } -// Performs an atomic operation in the outer memory +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with write permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + +/** Perform an atomic memory operation in the next-outermost level of the memory hierarchy + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (within which beat) + * @param addr_byte sub-block address (which byte) + * @param atomic_opcode {swap, add, xor, and, min, max, minu, maxu} from [[uncore.MemoryOpConstants]] + * @param operand_size {byte, half, word, double} from [[uncore.MemoryOpConstants]] + * @param data source operand data + */ object PutAtomic { def apply( client_xact_id: UInt, @@ -330,22 +468,11 @@ object PutAtomic { } } -// Prefetch a cache block into the next level of the memory hierarchy -// with write permissions -object PutPrefetch { - def apply( - client_xact_id: UInt, - addr_block: UInt): Acquire = { - Acquire( - is_builtin_type = Bool(true), - a_type = Acquire.prefetchType, - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = UInt(0), - union = Cat(M_XWR, Bool(true))) - } -} - +/** The Probe channel is used to force clients to release data or cede permissions + * on a cache block. Clients respond to Probes with [[uncore.Release]] messages. + * The available types of Probes are customized by a particular + * [[uncore.CoherencePolicy]]. + */ class Probe extends ManagerToClientChannel with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) @@ -355,6 +482,18 @@ class Probe extends ManagerToClientChannel def hasMultibeatData(dummy: Int = 0) = Bool(false) } +/** [[uncore.Probe]] with an extra field stating its destination id */ +class ProbeToDst extends Probe with HasClientId + +/** Contains factories for [[uncore.Probe]] and [[uncore.ProbeToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeProbe]] instead. + * + * @param dst id of client to which probe should be sent + * @param p_type custom probe type + * @param addr_block address of the cache block + */ object Probe { def apply(p_type: UInt, addr_block: UInt): Probe = { val prb = new Probe @@ -371,6 +510,13 @@ object Probe { } } +/** The Release channel is used to release data or permission back to the manager + * in response to [[uncore.Probe]] messages. It can also be used to voluntarily + * write back data, for example in the event that dirty data must be evicted on + * a cache miss. The available types of Release messages are always customized by + * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be + * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grants]]. + */ class Release extends ClientToManagerChannel with HasCacheBlockAddress with HasClientTransactionId @@ -387,6 +533,21 @@ class Release extends ClientToManagerChannel def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } +/** [[uncore.Release]] with an extra field stating its source id */ +class ReleaseFromSrc extends Release with HasClientId + +/** Contains a [[uncore.Release]] factory + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeRelease]] instead. + * + * @param voluntary is this a voluntary writeback + * @param r_type type enum defined by coherence protocol + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat beat id of the data + * @param data data being written back + */ object Release { def apply( voluntary: Bool, @@ -406,6 +567,13 @@ object Release { } } +/** The Grant channel is used to refill data or grant permissions requested of the + * manager agent via an [[uncore.Acquire]] message. It is also used to acknowledge + * the receipt of voluntary writeback from clients in the form of [[uncore.Release]] + * messages. There are built-in Grant messages used for Gets and Puts, and + * coherence policies may also define custom Grant types. Grants may contain data + * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. + */ class Grant extends ManagerToClientChannel with HasTileLinkData with HasClientTransactionId @@ -433,13 +601,30 @@ class Grant extends ManagerToClientChannel } } +/** [[uncore.Grant]] with an extra field stating its destination */ +class GrantToDst extends Grant with HasClientId + +/** Contains definitions of the the built-in grant types and factories + * for [[uncore.Grant]] and [[uncore.GrantToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeGrant]] instead. + * + * @param dst id of client to which grant should be sent + * @param is_builtin_type built-in or custom type message? + * @param g_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ object Grant { val nBuiltInTypes = 5 - def voluntaryAckType = UInt("b000") - def putAckType = UInt("b001") - def prefetchAckType = UInt("b011") - def getDataBeatType = UInt("b100") - def getDataBlockType = UInt("b101") + def voluntaryAckType = UInt("b000") // For acking Releases + def prefetchAckType = UInt("b001") // For acking any kind of Prefetch + def putAckType = UInt("b011") // For acking any kind of non-prfetch Put + def getDataBeatType = UInt("b100") // Supplying a single beat of Get + def getDataBlockType = UInt("b101") // Supplying all beats of a GetBlock def typesWithData = Vec(getDataBlockType, getDataBeatType) def typesWithMultibeatData= Vec(getDataBlockType) @@ -480,62 +665,69 @@ object Grant { } } +/** The Finish channel is used to provide a global ordering of transactions + * in networks that do not guarantee point-to-point ordering of messages. + * A Finsish message is sent as acknowledgement of receipt of a [[uncore.Grant]]. + * When a Finish message is received, a manager knows it is safe to begin + * processing other transactions that touch the same cache block. + */ class Finish extends ClientToManagerChannel with HasManagerTransactionId { def hasData(dummy: Int = 0) = Bool(false) def hasMultibeatData(dummy: Int = 0) = Bool(false) } -// These subtypes include a field for the source or destination ClientId -class AcquireFromSrc extends Acquire with HasClientId -class ProbeToDst extends Probe with HasClientId -class ReleaseFromSrc extends Release with HasClientId -class GrantToDst extends Grant with HasClientId - -// Complete IO definitions for two types of TileLink clients, including -// networking headers +/** Complete IO definition for incoherent TileLink, including networking headers */ class UncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } +/** Complete IO definition for coherent TileLink, including networking headers */ class TileLinkIO extends UncachedTileLinkIO { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } -// Converts UncachedTileLinkIO to regular TileLinkIO by pinning -// probe.ready and release.valid low -class TileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -// This version of TileLinkIO does not contain network headers. The headers -// are provided in the top-level that instantiates the clients and network, -// probably using a TileLinkClientPort module. -// By eliding the header subbundles within the clients we can enable -// hierarchical P&R while minimizing unconnected port errors in GDS. -// Secondly, this version of the interface elides Finish messages, with the -// assumption that a FinishUnit has been coupled to the TileLinkIO port -// to deal with acking received Grants. +/** This version of UncachedTileLinkIO does not contain network headers. + * It is intended for use within client agents. + * + * Headers are provided in the top-level that instantiates the clients and network, + * probably using a [[uncore.ClientTileLinkPort]] module. + * By eliding the header subbundles within the clients we can enable + * hierarchical P-and-R while minimizing unconnected port errors in GDS. + * + * Secondly, this version of the interface elides [[uncore.Finish]] messages, with the + * assumption that a [[uncore.FinishUnit]] has been coupled to the TileLinkIO port + * to deal with acking received [[uncore.Grants]]. + */ class ClientUncachedTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } +/** This version of TileLinkIO does not contain network headers. + * It is intended for use within client agents. + */ class ClientTileLinkIO extends ClientUncachedTileLinkIO { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } +/** This version of TileLinkIO does not contain network headers, but + * every channel does include an extra client_id subbundle. + * It is intended for use within Management agents. + * + * Managers need to track where [[uncore.Acquire]] and [[uncore.Release]] messages + * originated so that they can send a [[uncore.Grant]] to the right place. + * Similarly they must be able to issues Probes to particular clients. + * However, we'd still prefer to have [[uncore.ManagerTileLinkPort]] fill in + * the header.src to enable hierarchical p-and-r of the managers. Additionally, + * coherent clients might be mapped to random network port ids, and we'll leave it to the + * [[uncore.ManagerTileLinkPort]] to apply the correct mapping. Managers do need to + * see Finished so they know when to allow new transactions on a cache + * block to proceed. + */ class ManagerTileLinkIO extends TLBundle { val acquire = new DecoupledIO(new AcquireFromSrc).flip val grant = new DecoupledIO(new GrantToDst) @@ -544,17 +736,7 @@ class ManagerTileLinkIO extends TLBundle { val release = new DecoupledIO(new ReleaseFromSrc).flip } -class ClientTileLinkIOWrapper extends TLModule { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ object TileLinkIOWrapper { def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { val conv = Module(new ClientTileLinkIOWrapper)(p) @@ -580,13 +762,32 @@ object TileLinkIOWrapper { def apply(tl: TileLinkIO): TileLinkIO = tl } -class FinishQueueEntry extends TLBundle { - val fin = new Finish - val dst = UInt(width = log2Up(params(LNEndpoints))) +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) } -class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +class ClientTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip @@ -633,36 +834,23 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with Has } } -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress : ClassTag]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) } -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId : ClassTag]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probes]]. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { val io = new Bundle { val client = new ClientTileLinkIO().flip @@ -686,6 +874,31 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extend io.client.grant <> gnt_without_header } +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { val io = new Bundle { val manager = new ManagerTileLinkIO().flip @@ -700,8 +913,25 @@ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extend io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** Struct for describing per-channel queue depths */ case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { val io = new Bundle { val client = new TileLinkIO().flip @@ -905,10 +1135,10 @@ class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiter } /** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular TileLinkChannel or pair of channels. + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. * * Won't count message types that don't have data. - * Used in XactTrackers and FinishUnit. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. */ trait HasDataBeatCounters { type HasBeat = TileLinkChannel with HasTileLinkBeatId @@ -926,7 +1156,7 @@ trait HasDataBeatCounters { (cnt, done) } - /** Counter for beats on outgoing DecoupledIOs */ + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = connectDataBeatCounter(in.fire(), in.bits, beat) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8dda15b7..d7573aec 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,15 +2,10 @@ package uncore import Chisel._ -import scala.reflect._ -import scala.reflect.runtime.universe._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object NIncoherentClients extends Field[Int] -case object NCoherentClients extends Field[Int] -case object L2CoherencePolicy extends Field[CoherencePolicy] trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 @@ -28,6 +23,7 @@ trait CoherenceAgentParameters extends UsesParameters { val innerByteAddrBits = log2Up(innerDataBits/8) require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } + abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters @@ -53,7 +49,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) - val incoherent = Vec.fill(inner.tlNCoherentClients){Bool()}.asInput + val incoherent = Vec.fill(inner.tlNCachingClients){Bool()}.asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits def irel(dummy: Int = 0) = inner.release.bits @@ -129,5 +125,5 @@ abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = - ~Fill(in.bits.tlNCoherentClients, in.fire()) | ~UIntToOH(in.bits.client_id) + ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) } From f66a9fd7a6fd8f438c823cae85b24fd7cc4cd0c7 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 10:46:02 -0700 Subject: [PATCH 369/688] simplify ClientMetadata.makeRelease --- uncore/src/main/scala/metadata.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index cf52d286..f91035f4 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -59,13 +59,12 @@ class ClientMetadata extends CoherenceMetadata { def makeRelease( prb: Probe, - client_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Release = { Bundle(Release( voluntary = Bool(false), r_type = co.getReleaseType(prb, this), - client_xact_id = client_xact_id, + client_xact_id = UInt(0), addr_block = prb.addr_block, addr_beat = addr_beat, data = data), { case TLId => id }) From 4c7969b2b37bafa83dac60b1a20c6288110d958b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 20 Apr 2015 16:32:09 -0700 Subject: [PATCH 370/688] Metadata docs and api cleanup --- uncore/src/main/scala/cache.scala | 25 ++-- uncore/src/main/scala/metadata.scala | 173 ++++++++++++++++++++++----- uncore/src/main/scala/tilelink.scala | 7 ++ 3 files changed, 160 insertions(+), 45 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 75bb9e1f..4adb09a3 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -338,10 +338,6 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2SecondaryMissInfo extends TLBundle - with HasTileLinkBeatId - with HasClientTransactionId - class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) @@ -617,10 +613,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val pending_coh = Reg{ xact_old_meta.coh.clone } // Secondary miss queue - val ignt_q = Module(new Queue(new L2SecondaryMissInfo, nSecondaryMisses))(innerTLParams) - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - // TODO add ignt.dst <- iacq.src + val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) @@ -741,6 +734,12 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { can_merge_iacq_put || can_merge_iacq_get + // Enqueue secondary miss information + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + // TODO add ignt.dst <- iacq.src + // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) @@ -792,22 +791,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) - ignt_q.io.enq.valid := iacq_data_done ignt_q.io.deq.ready := ignt_data_done - // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) + // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( - dst = xact.client_id, // TODO: ignt_q.io.deq.bits.src - acq = xact, - client_xact_id = ignt_q.io.deq.bits.client_xact_id, + pri = xact, + sec = ignt_q.io.deq.bits, manager_xact_id = UInt(trackerId), - addr_beat = ignt_data_idx, data = Mux(xact.is(Acquire.putAtomicType), amo_result, data_buffer(ignt_data_idx))) + io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( pending_coh.inner.onGrant(io.ignt()), diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index f91035f4..74e410c6 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -// Classes to represent coherence information in clients and managers +/** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata extends Bundle { val co = params(TLCoherencePolicy) val id = params(TLId) @@ -15,21 +15,35 @@ abstract class CoherenceMetadata extends Bundle { * memory operations or [[uncore.Probe]] messages. */ class ClientMetadata extends CoherenceMetadata { + /** Actual state information stored in this bundle */ val state = UInt(width = co.clientStateWidth) + /** Metadata equality */ def ===(rhs: ClientMetadata): Bool = this.state === rhs.state def !=(rhs: ClientMetadata): Bool = !this.===(rhs) + /** Is the block's data present in this cache */ def isValid(dummy: Int = 0): Bool = co.isValid(this) - def isHit(cmd: UInt): Bool = co.isHit(cmd, this) - def isMiss(cmd: UInt): Bool = !co.isHit(cmd, this) - def requiresAcquireOnSecondaryMiss(first_cmd: UInt, second_cmd: UInt): Bool = - co.requiresAcquireOnSecondaryMiss(first_cmd, second_cmd, this) - def requiresReleaseOnCacheControl(cmd: UInt): Bool = - co.requiresReleaseOnCacheControl(cmd: UInt, this) + /** Does this cache have permissions on this block sufficient to perform op */ + def isHit(op_code: UInt): Bool = co.isHit(op_code, this) + /** Does this cache lack permissions on this block sufficient to perform op */ + def isMiss(op_code: UInt): Bool = !co.isHit(op_code, this) + /** Does a secondary miss on the block require another Acquire message */ + def requiresAcquireOnSecondaryMiss(first_op: UInt, second_op: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_op, second_op, this) + /** Does op require a Release to be made to outer memory */ + def requiresReleaseOnCacheControl(op_code: UInt): Bool = + co.requiresReleaseOnCacheControl(op_code: UInt, this) + /** Does an eviction require a Release to be made to outer memory */ def requiresVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresReleaseOnCacheControl(M_FLUSH, this) + /** Constructs an Acquire message based on this metdata and a memory operation + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ def makeAcquire( client_xact_id: UInt, addr_block: UInt, @@ -43,6 +57,13 @@ class ClientMetadata extends CoherenceMetadata { { case TLId => id }) } + /** Constructs a Release message based on this metadata on an eviction + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being written back + */ def makeVoluntaryWriteback( client_xact_id: UInt, addr_block: UInt, @@ -57,6 +78,12 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Constructs a Release message based on this metadata and a [[uncore.Probe]] + * + * @param the incoming [[uncore.Probe]] + * @param addr_beat sub-block address (which beat) + * @param data data being released + */ def makeRelease( prb: Probe, addr_beat: UInt = UInt(0), @@ -70,16 +97,37 @@ class ClientMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** New metadata after receiving a [[uncore.Grant]] + * + * @param incoming the incoming [[uncore.Grant]] + * @param pending the mem op that triggered this transaction + */ def onGrant(incoming: Grant, pending: UInt): ClientMetadata = Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param incoming the incoming [[uncore.Probe]] + */ def onProbe(incoming: Probe): ClientMetadata = Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) - def onHit(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnHit(cmd, this), { case TLId => id }) - def onCacheControl(cmd: UInt): ClientMetadata = - Bundle(co.clientMetadataOnCacheControl(cmd, this), { case TLId => id }) + + /** New metadata after a op_code hits this block + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onHit(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param op_code a memory operation from [[uncore.MemoryOpConstants]] + */ + def onCacheControl(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(op_code, this), { case TLId => id }) } +/** Factories for ClientMetadata, including on reset */ object ClientMetadata { def apply(state: UInt) = { val meta = new ClientMetadata @@ -89,33 +137,64 @@ object ClientMetadata { def onReset = new ClientMetadata().co.clientMetadataOnReset } -/* The ManagerMetadata stores manager-side information about the status - of a cache block, including whether it has any known sharers. Its - API can be used to create Probe and Grant TileLink messages. -*/ +/** Stores manager-side information about the status + * of a cache block, including whether it has any known sharers. + * + * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. + */ class ManagerMetadata extends CoherenceMetadata { + // Currently no coherence policies assume manager-side state information // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + + /** The directory information for this block */ val sharers = UInt(width = co.dir.width) + /** Metadata equality */ def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) - def full(dummy: Int = 0) = co.dir.full(this.sharers) + /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ + def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) + + /** Does this [[uncore.Acquire]] require [[uncore.Probes]] to be sent */ def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) - def requiresProbes(cmd: UInt): Bool = co.requiresProbes(cmd, this) + /** Does this memory op require [[uncore.Probes]] to be sent */ + def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) + /** Does an eviction require [[uncore.Probes]] to be sent */ def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = co.requiresProbes(M_FLUSH, this) - def makeProbe(dst: UInt, cmd: UInt, addr_block: UInt): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(cmd, this), addr_block), { case TLId => id }) - + /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] + * + * @param dst Destination client id for this Probe + * @param acq Acquire message triggering this Probe + */ def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op + * + * @param dst Destination client id for this Probe + * @param op_code memory operation triggering this Probe + * @param addr_block address of the cache block being probed + */ + def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block), { case TLId => id }) + + /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction + * + * @param dst Destination client id for this Probe + * @param addr_block address of the cache block being probed prior to eviction + */ def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = makeProbe(dst, M_FLUSH, addr_block) + /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] + * + * @param rel Release message being acknowledged by this Grant + * @param manager_xact_id manager's transaction id + */ def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( dst = rel.client_id, @@ -125,6 +204,15 @@ class ManagerMetadata extends CoherenceMetadata { manager_xact_id = manager_xact_id), { case TLId => id }) } + /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] + * + * May contain single or multiple beats of data, or just be a permissions upgrade. + * + * @param acq Acquire message being responded to by this Grant + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ def makeGrant( acq: AcquireFromSrc, manager_xact_id: UInt, @@ -142,26 +230,42 @@ class ManagerMetadata extends CoherenceMetadata { data = data), { case TLId => id }) } + /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides + * + * Used to respond to secondary misses merged into this transaction. + * May contain single or multiple beats of data. + * + * @param pri Primary miss's Acquire message, used to get g_type and dst + * @param sec Secondary miss info, used to get beat and client_xact_id + * @param manager_xact_id manager's transaction id + * @param data data being refilled to the original requestor + */ def makeGrant( - dst: UInt, - acq: AcquireFromSrc, - client_xact_id: UInt, + pri: AcquireFromSrc, + sec: SecondaryMissInfo, manager_xact_id: UInt, - addr_beat: UInt, data: UInt): GrantToDst = { - val g = makeGrant(acq, manager_xact_id, addr_beat, data) - g.client_id := dst - g.client_xact_id := client_xact_id + val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) + g.client_xact_id := sec.client_xact_id g } + /** New metadata after receiving a [[uncore.ReleaseFromSrc]] + * + * @param incoming the incoming [[uncore.ReleaseFromSrc]] + */ def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) + /** New metadata after sending a [[uncore.GrantToDst]] + * + * @param outgoing the outgoing [[uncore.GrantToDst]] + */ def onGrant(outgoing: GrantToDst): ManagerMetadata = Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) } +/** Factories for ManagerMetadata, including on reset */ object ManagerMetadata { def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { val meta = new ManagerMetadata @@ -178,10 +282,14 @@ object ManagerMetadata { def onReset = new ManagerMetadata().co.managerMetadataOnReset } -/* HierarchicalMetadata is used in a cache in a multi-level memory hierarchy - that is a Manager with respect to some inner caches and a Client with - respect to some outer cache. -*/ +/** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + * that is a manager with respect to some inner caches and a client with + * respect to some outer cache. + * + * This class makes use of two different sets of TileLink parameters, which are + * applied by contextually mapping [[uncore.TLId]] to one of + * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. + */ class HierarchicalMetadata extends CoherenceMetadata { val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) @@ -190,6 +298,7 @@ class HierarchicalMetadata extends CoherenceMetadata { def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } +/** Factories for HierarchicalMetadata, including on reset */ object HierarchicalMetadata { def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { val m = new HierarchicalMetadata @@ -200,5 +309,7 @@ object HierarchicalMetadata { def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) } +/** Identifies the TLId of the inner network in a hierarchical cache controller */ case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b759d48b..1c1c2bb4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -785,6 +785,13 @@ class ClientTileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } +/** Used to track metadata for transactions where multiple secondary misses have been merged + * and handled by a single transaction tracker. + */ +class SecondaryMissInfo extends TLBundle // TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId + with HasTileLinkBeatId + with HasClientTransactionId + /** A helper module that automatically issues [[uncore.Finish]] messages in repsonse * to [[uncore.Grant]] that it receives from a manager and forwards to a client */ From 11b5222d014a425ac65545491e92800b69c01336 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 21 Apr 2015 22:23:04 -0700 Subject: [PATCH 371/688] Refactored WritebackUnit --- uncore/src/main/scala/cache.scala | 245 +++++++++++++----------------- 1 file changed, 105 insertions(+), 140 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4adb09a3..32529202 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -192,9 +192,8 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { } def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { - val idx = in.bits.id outs.map(_.bits := in.bits) - outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && in.bits.id === UInt(i) } } } @@ -621,7 +620,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 - val (pending_ognts, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectTwoWayBeatCounter( max = 1, up = io.outer.acquire, @@ -646,7 +645,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { pending_writes.orR || pending_resps.orR || pending_puts.orR || - pending_ognts || + pending_ognt || ignt_q.io.count > UInt(0) || //pending_meta_write || // Has own state: s_meta_write pending_ifins) @@ -794,8 +793,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && - (!io.ignt().hasData() || - pending_ignt_data(ignt_data_idx)) + (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( pri = xact, @@ -826,9 +824,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && - pending_reads.orR && - !pending_ognts + io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt io.data.read.bits.id := UInt(trackerId) io.data.read.bits.way_en := xact_way_en io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) @@ -847,7 +843,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val curr_write_beat = PriorityEncoder(pending_writes) io.data.write.valid := state === s_busy && pending_writes.orR && - !pending_ognts && + !pending_ognt && !pending_reads(curr_write_beat) && !pending_resps(curr_write_beat) io.data.write.bits.id := UInt(trackerId) @@ -870,10 +866,11 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.coh := pending_coh io.wb.req.valid := state === s_wb_req - io.wb.req.bits.addr_block := Cat(xact_old_meta.tag, xact.addr_block(idxMSB,idxLSB)) + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en - io.wb.req.bits.id := UInt(trackerId) // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie @@ -961,9 +958,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // These IOs are used for routing in the parent val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_inner_probe) + io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? @@ -974,10 +969,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { "AcquireTracker accepted data beat from different network source than initial request.") } -class L2WritebackReq extends L2HellaCacheBundle - with HasL2Id { - val addr_block = UInt(width = blockAddrBits) // TODO: assumes same block size - val coh = new HierarchicalMetadata +class L2WritebackReq extends L2Metadata with HasL2Id { + val idx = Bits(width = idxBits) val way_en = Bits(width = nWays) } @@ -996,136 +989,108 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO - val s_idle :: s_inner_probe :: s_data_read :: s_data_resp :: s_outer_release :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 7) + val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact_addr_block = Reg(io.wb.req.bits.addr_block.clone) - val xact_coh = Reg{ new HierarchicalMetadata } - val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } - val xact_id = Reg{ UInt() } + val xact = Reg(new L2WritebackReq) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val xact_addr_block = Cat(xact.tag, xact.idx) - val irel_had_data = Reg(init = Bool(false)) - val irel_cnt = Reg(init = UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) - val pending_probes = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) - val curr_probe_dst = PriorityEncoder(pending_probes) - val full_sharers = io.wb.req.bits.coh.inner.full() - val mask_incoherent = full_sharers & ~io.incoherent.toBits + val pending_irels = + connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 + val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = + connectTwoWayBeatCounter(max = 1, up = io.outer.release, down = io.outer.grant) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (orel_data_cnt, orel_data_done) = connectOutgoingDataBeatCounter(io.outer.release) - val (read_data_cnt, read_data_done) = connectInternalDataBeatCounter(io.data.read) - val resp_data_done = connectInternalDataBeatCounter(io.data.resp) + // Start the writeback sub-transaction + io.wb.req.ready := state === s_idle - val pending_icoh_on_irel = xact_coh.inner.onRelease(io.irel()) - val pending_ocoh_on_irel = xact_coh.outer.onHit(M_XWR) // WB is a write - val pending_ofin_on_ognt = io.ognt().makeFinish() + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := xact.coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := !io.irel().isVoluntary() && - io.irel().conflicts(xact_addr_block) && - (state === s_inner_probe) + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) + val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) + io.inner.release.ready := state === s_inner_probe || state === s_busy + when(io.inner.release.fire()) { + xact.coh.inner := inner_coh_on_irel + when(io.irel().hasData()) { xact.coh.outer := outer_coh_on_irel } // WB is a write + data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data + } - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) // default - io.outer.release.bits := xact_coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_cnt, - data = data_buffer(orel_data_cnt)) - io.outer.grant.ready := Bool(false) // default - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits := xact_coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) - - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - io.data.read.valid := Bool(false) + // If a release didn't write back data, have to read it from data array + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release)) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := read_data_cnt + io.data.read.bits.way_en := xact.way_en + io.data.read.bits.addr_idx := xact.idx + io.data.read.bits.addr_beat := curr_read_beat io.data.write.valid := Bool(false) - io.wb.req.ready := Bool(false) - io.wb.resp.valid := Bool(false) - io.wb.resp.bits.id := xact_id - - switch (state) { - is(s_idle) { - io.wb.req.ready := Bool(true) - when(io.wb.req.valid) { - xact_addr_block := io.wb.req.bits.addr_block - xact_coh := io.wb.req.bits.coh - xact_way_en := io.wb.req.bits.way_en - xact_id := io.wb.req.bits.id - irel_had_data := Bool(false) - val needs_inner_probes = io.wb.req.bits.coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_inner_probes) { - pending_probes := mask_incoherent - irel_cnt := PopCount(mask_incoherent) - } - state := Mux(needs_inner_probes, s_inner_probe, s_data_read) - } - } - is(s_inner_probe) { - // Send probes - io.inner.probe.valid := pending_probes != UInt(0) - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_probe_dst) - } - // Handle releases, which may have data being written back - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact_coh.inner := pending_icoh_on_irel - // Handle released dirty data - when(io.irel().hasData()) { - irel_had_data := Bool(true) - xact_coh.outer := pending_ocoh_on_irel - data_buffer(io.irel().addr_beat) := io.irel().data - } - // We don't decrement irel_cnt until we've received all the data beats. - when(!io.irel().hasData() || irel_data_done) { - irel_cnt := irel_cnt - UInt(1) - } - } - when(irel_cnt === UInt(0)) { - state := Mux(irel_had_data, // If someone released a dirty block - s_outer_release, // write that block back, otherwise - Mux(xact_coh.outer.requiresVoluntaryWriteback(), - s_data_read, // write extant dirty data back, or just - s_wb_resp)) // drop a clean block after collecting acks - } - } - is(s_data_read) { - io.data.read.valid := Bool(true) - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(read_data_done) { state := s_data_resp } - } - is(s_data_resp) { - when(io.data.resp.valid) { data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } - when(resp_data_done) { state := s_outer_release } - } - is(s_outer_release) { - io.outer.release.valid := Bool(true) - when(orel_data_done) { - state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) - } - } - is(s_outer_grant) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { - state := s_wb_resp - } - } - is(s_wb_resp) { - io.wb.resp.valid := Bool(true) - state := s_idle - } + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + when(io.data.resp.valid) { + data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data } + + // Once the data is buffered we can write it back to outer memory + pending_orel_data := pending_orel_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitInternal(io.data.resp) + io.outer.release.valid := state === s_busy && + (!io.orel().hasData() || pending_orel_data(orel_data_idx)) + io.outer.release.bits := xact.coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_idx, + data = data_buffer(orel_data_idx)) + + // Wait for an acknowledgement + io.outer.grant.ready := state === s_outer_grant + + // Respond to the initiating transaction handler signalling completion of the writeback + io.wb.resp.valid := state === s_wb_resp + io.wb.resp.bits.id := xact.id + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.wb.req.valid) { + xact := io.wb.req.bits + val coh = io.wb.req.bits.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } + pending_reads := SInt(-1, width = innerDataBeats) + pending_resps := UInt(0) + pending_orel_data := UInt(0) + state := Mux(needs_inner_probes, s_inner_probe, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) + } + when(state === s_busy && orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) + } + when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } + when(state === s_wb_resp ) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // IOs pinned low + io.outer.acquire.valid := Bool(false) + io.outer.probe.ready := Bool(false) // TODO: handle these here + io.inner.grant.valid := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.finish.ready := Bool(false) } From 09e30041edd97d9e3e6dfa0bb779c64c4234a770 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 12:56:33 -0700 Subject: [PATCH 372/688] Voluntary Writeback tracker rewrite --- uncore/src/main/scala/cache.scala | 212 ++++++++++++++---------------- 1 file changed, 97 insertions(+), 115 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 32529202..7fcd1408 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -337,8 +337,7 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCacheBank extends HierarchicalCoherenceAgent - with L2HellaCacheParameters { +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) @@ -357,8 +356,7 @@ class TSHRFileIO extends HierarchicalTLIO { val data = new L2DataRWIO } -class TSHRFile extends L2HellaCacheModule - with HasCoherenceAgentWiringHelpers { +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions @@ -381,7 +379,6 @@ class TSHRFile extends L2HellaCacheModule val acquire_idx = Mux(acquireMatches.orR, PriorityEncoder(acquireMatches), PriorityEncoder(acquireReadys)) - val block_acquires = acquireConflicts.orR io.inner.acquire.ready := acquireReadys.orR && !block_acquires trackerAcquireIOs.zipWithIndex.foreach { @@ -432,7 +429,7 @@ class L2XactTrackerIO extends HierarchicalXactTrackerIO { } abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { - class CacheBlockBuffer { + class CacheBlockBuffer { // TODO val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) @@ -477,127 +474,118 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { val isPartial = a.wmask() != Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[T] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[T] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } } class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) - val s_idle :: s_meta_read :: s_meta_resp :: s_data_write :: s_meta_write :: s_inner_grant :: s_inner_finish :: Nil = Enum(UInt(), 7) + val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val xact_tag_match = Reg{ Bool() } - val xact_old_meta = Reg{ new L2Metadata } + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } val xact_way_en = Reg{ Bits(width = nWays) } - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh - val collect_irel_data = Reg(init=Bool(false)) - val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (write_data_cnt, write_data_done) = connectInternalDataBeatCounter(io.data.write) + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) - io.has_acquire_conflict := Bool(false) - io.has_acquire_match := Bool(false) - io.has_release_match := io.irel().isVoluntary() + val all_pending_done = + !(pending_writes.orR || + pending_ignt) - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := state === s_idle || pending_irels.orR + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } - io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) - - io.data.read.valid := Bool(false) - io.data.write.valid := Bool(false) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := write_data_cnt - io.data.write.bits.wmask := SInt(-1) - io.data.write.bits.data := data_buffer(write_data_cnt) - io.meta.read.valid := Bool(false) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - io.meta.write.valid := Bool(false) + + // Write the voluntarily written back data to this cache + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && pending_writes.orR + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(curr_write_beat) + + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + + // End a transaction by updating the block metadata + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) - io.meta.write.bits.data.coh.outer := xact_old_meta.coh.outer.onHit(M_XWR) // WB is a write - io.wb.req.valid := Bool(false) + io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), + xact_old_meta.coh.outer.onHit(M_XWR), + xact_old_meta.coh.outer) - when(collect_irel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid(io.irel().addr_beat) := Bool(true) + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.valid) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) } - when(irel_data_done) { collect_irel_data := Bool(false) } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_meta_read } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := s_busy + } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := io.irel() - data_buffer(io.irel().addr_beat) := io.irel().data - collect_irel_data := io.irel().hasMultibeatData() - irel_data_valid := io.irel().hasData() << io.irel().addr_beat - state := s_meta_read - } - } - is(s_meta_read) { - io.meta.read.valid := Bool(true) - when(io.meta.read.ready) { state := s_meta_resp } - } - is(s_meta_resp) { - when(io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_old_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := Mux(io.meta.resp.bits.tag_match, - Mux(xact.hasData(), s_data_write, s_meta_write), - Mux(xact.requiresAck(), s_inner_grant, s_idle)) - } - } - is(s_data_write) { - io.data.write.valid := !collect_irel_data || irel_data_valid(write_data_cnt) - when(write_data_done) { state := s_meta_write } - } - is(s_meta_write) { - io.meta.write.valid := Bool(true) - when(io.meta.write.ready) { - state := Mux(xact.requiresAck(), s_inner_grant, s_idle) // Need a Grant.voluntaryAck? - } - } - is(s_inner_grant) { - io.inner.grant.valid := !collect_irel_data - when(io.inner.grant.ready) { - state := Mux(io.ignt().requiresAck(), s_inner_finish, s_idle) - } - } - is(s_inner_finish) { - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } + // These IOs are used for routing in the parent + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) // Checks for illegal behavior assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val io = new L2XactTrackerIO + pinAllReadyValidLow(io) val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) @@ -742,6 +730,20 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + + // Issue a request to the writeback unit + io.wb.req.valid := state === s_wb_req + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag + io.wb.req.bits.coh := xact_old_meta.coh + io.wb.req.bits.way_en := xact_way_en + // Track which clients yet need to be probed and make Probe message pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) @@ -773,10 +775,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { op_code = xact.op_code()), Bundle(Acquire(xact))(outerTLParams)) - // Probes from the outer memory system are handled in the WritebackUnit - io.outer.probe.ready := Bool(false) - io.outer.release.valid := Bool(false) - + // Handle the response from outer memory io.outer.grant.ready := state === s_busy val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, @@ -853,11 +852,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.data.write.bits.wmask := wmask_buffer(curr_write_beat) io.data.write.bits.data := data_buffer(curr_write_beat) - io.meta.read.valid := state === s_meta_read - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) - + // End a transaction by updating the block metadata io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) @@ -865,13 +860,6 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) io.meta.write.bits.data.coh := pending_coh - io.wb.req.valid := state === s_wb_req - io.wb.req.bits.id := UInt(trackerId) - io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.wb.req.bits.tag := xact_old_meta.tag - io.wb.req.bits.coh := xact_old_meta.coh - io.wb.req.bits.way_en := xact_way_en - // Handling of secondary misses (Gets and Puts only for now) when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie val beat = io.iacq().addr_beat @@ -988,6 +976,7 @@ class L2WritebackUnitIO extends HierarchicalXactTrackerIO { class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val io = new L2WritebackUnitIO + pinAllReadyValidLow(io) val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) @@ -1086,11 +1075,4 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) - - // IOs pinned low - io.outer.acquire.valid := Bool(false) - io.outer.probe.ready := Bool(false) // TODO: handle these here - io.inner.grant.valid := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.finish.ready := Bool(false) } From 3673295d03672b2a1fcc96e7b402de571c7bf7ee Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 27 Apr 2015 16:59:30 -0700 Subject: [PATCH 373/688] network shim cleanup --- uncore/src/main/scala/network.scala | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index f61cec6c..4b00091d 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -57,7 +57,10 @@ class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { } object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data](in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)) = { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless out.valid := in.valid out.bits.payload := in.bits @@ -69,7 +72,7 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { val out = Decoupled(in.bits.payload.clone).asDirectionless out.valid := in.valid out.bits := in.bits.payload @@ -77,3 +80,25 @@ object DecoupledLogicalNetworkIOUnwrapper { out } } + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Decoupled(new LogicalNetworkIO(in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Decoupled(new PhysicalNetworkIO(n, in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} From 1e05fc052510c2a5daea779564996d3986970082 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Apr 2015 13:18:26 -0700 Subject: [PATCH 374/688] First pages commit --- uncore/.gitignore | 2 + uncore/LICENSE | 24 + uncore/README.md | 11 + uncore/build.sbt | 13 + uncore/chisel-dependent.sbt | 8 + uncore/doc/TileLink0.3.1Specification.pdf | Bin 0 -> 172795 bytes uncore/index.html | 1 + uncore/src/main/scala/bigmem.scala | 80 ++ uncore/src/main/scala/broadcast.scala | 387 +++++++ uncore/src/main/scala/cache.scala | 1078 ++++++++++++++++++ uncore/src/main/scala/coherence.scala | 688 ++++++++++++ uncore/src/main/scala/consts.scala | 48 + uncore/src/main/scala/directory.scala | 43 + uncore/src/main/scala/ecc.scala | 146 +++ uncore/src/main/scala/htif.scala | 255 +++++ uncore/src/main/scala/memserdes.scala | 584 ++++++++++ uncore/src/main/scala/metadata.scala | 315 ++++++ uncore/src/main/scala/network.scala | 104 ++ uncore/src/main/scala/package.scala | 6 + uncore/src/main/scala/slowio.scala | 70 ++ uncore/src/main/scala/tilelink.scala | 1221 +++++++++++++++++++++ uncore/src/main/scala/uncore.scala | 129 +++ uncore/src/main/scala/util.scala | 106 ++ 23 files changed, 5319 insertions(+) create mode 100644 uncore/.gitignore create mode 100644 uncore/LICENSE create mode 100644 uncore/README.md create mode 100644 uncore/build.sbt create mode 100644 uncore/chisel-dependent.sbt create mode 100644 uncore/doc/TileLink0.3.1Specification.pdf create mode 100644 uncore/index.html create mode 100644 uncore/src/main/scala/bigmem.scala create mode 100644 uncore/src/main/scala/broadcast.scala create mode 100644 uncore/src/main/scala/cache.scala create mode 100644 uncore/src/main/scala/coherence.scala create mode 100644 uncore/src/main/scala/consts.scala create mode 100644 uncore/src/main/scala/directory.scala create mode 100644 uncore/src/main/scala/ecc.scala create mode 100644 uncore/src/main/scala/htif.scala create mode 100644 uncore/src/main/scala/memserdes.scala create mode 100644 uncore/src/main/scala/metadata.scala create mode 100644 uncore/src/main/scala/network.scala create mode 100644 uncore/src/main/scala/package.scala create mode 100644 uncore/src/main/scala/slowio.scala create mode 100644 uncore/src/main/scala/tilelink.scala create mode 100644 uncore/src/main/scala/uncore.scala create mode 100644 uncore/src/main/scala/util.scala diff --git a/uncore/.gitignore b/uncore/.gitignore new file mode 100644 index 00000000..555feb41 --- /dev/null +++ b/uncore/.gitignore @@ -0,0 +1,2 @@ +target/ +project/target/ diff --git a/uncore/LICENSE b/uncore/LICENSE new file mode 100644 index 00000000..7cff15e4 --- /dev/null +++ b/uncore/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2012-2014, The Regents of the University of California +(Regents). All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the Regents nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +IN NO EVENT SHALL REGENTS BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING +OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF REGENTS HAS +BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REGENTS SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE SOFTWARE AND ACCOMPANYING DOCUMENTATION, IF ANY, PROVIDED +HEREUNDER IS PROVIDED "AS IS". REGENTS HAS NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. diff --git a/uncore/README.md b/uncore/README.md new file mode 100644 index 00000000..003f9caa --- /dev/null +++ b/uncore/README.md @@ -0,0 +1,11 @@ +Uncore Library +============== + +This is the repository for uncore components assosciated with Rocket chip +project. To uses these modules, include this repo as a git submodule within +the your chip repository and add it as Project in your chip's build.scala. +These components are only dependent on Chisel, i.e. + + lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) + +Documentation about the uncore library will come in the near future. diff --git a/uncore/build.sbt b/uncore/build.sbt new file mode 100644 index 00000000..d78d02ca --- /dev/null +++ b/uncore/build.sbt @@ -0,0 +1,13 @@ +organization := "edu.berkeley.cs" + +version := "2.0" + +name := "uncore" + +scalaVersion := "2.10.2" + +site.settings + +ghpages.settings + +git.remoteRepo := "git@github.com:ucb-bar/uncore.git" diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt new file mode 100644 index 00000000..88eb615c --- /dev/null +++ b/uncore/chisel-dependent.sbt @@ -0,0 +1,8 @@ +// Provide a managed dependency on chisel if -DchiselVersion="" is +// supplied on the command line. + +val chiselVersion_u = System.getProperty("chiselVersion", "None") + +// _u a temporary fix until sbt 13.6 https://github.com/sbt/sbt/issues/1465 + +libraryDependencies ++= ( if (chiselVersion_u != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion_u) :: Nil; else Nil) diff --git a/uncore/doc/TileLink0.3.1Specification.pdf b/uncore/doc/TileLink0.3.1Specification.pdf new file mode 100644 index 0000000000000000000000000000000000000000..23666814d9e8738924e59e5f9f8872e7fa58d725 GIT binary patch literal 172795 zcmb5U1yEeU*6$lExJ!Z$?gI=uxVyWC4DLEO2@)*0J0Up1-6dFpdxAr7cZUFPIP%?d z>c02t-YTklX6?1Q|J`ftu08A@otl&k8z+z(jc#r4U>Xg?0pI{QncAWW39)N=J6izQ zB_U9Vy^|FhyBfsG!VSO)yRLyIB7$b&X#U~{dNKT0z>D9Dx2uIC^k44m8WwI&?yhFA zc-XjVu1;o}7Em{UAvFKrLQkWjO#K z(7z?H%Q(TNfLUc=R<3_o*=0BZAnt#C1OY%i|N6)U0P+4i3?`mk*}~igBH`o(Fnn1s zFDD4V2L{0=6L)lUdQtX2%m0S~ja}2-6#9?tP_l8fLt~e4ay7Sb{bw)-EKI|~3<@yh z<x??;0ATIfH| zyTnkuesDcybHpK|)O7z5%eo2M)pW0_Oh~DZ{ct)`-4vrh_f1PG-2{2xVS zI87fSa(VyU4nHsmvzPndp>UBtTQ{D5M`W#j`b2blca!sUYIv(~c5u`6m%wR%@1&os z^W*b_>NAPN@$>c7!|E549~|#oezXK8QH!nn6>4o@X&}jARizKjiGNH4JQ8lm2@c=@ zEZS{j1ykaXW#`3g@cp-momXzuPMdQq;D^JOmshXC^ zNJR2?xw!pn$$S2nRoxM#AH7)EXnp@gGl|Wm{4SCQsW>308z|oE$e#;_&n#$|(#oo?!egNpp?m=%Hh_wnVV>c9f`B#%A;<`!S{Z2s6} zJ$d2upP!yZX)o?N^fR_sPsrClUhbDWZ z;Tn@N=+WePf0?&XHyS0a%cO?2jD)ajru47_CaU-p3NjM=kcbLD;0yQ#{bj{`18v3( zW3g|Z8WCh0Y7`$;Wu0oIGxh&6N%ba_B@3|7nEWQb{Y37|kd}2WD73d#>;1&V%;riM zk|%z>WBp}xNV&4&f(Tn-fNsjfU}`T zT_2T*Ki0t8f{6E~ywaSDQ+1e928HtISB)ofr{G!k1&!BHRi7aYTVc2MZHYYWZI1zr z7m_zL-aa1?)cuMcYK|ER66G`Kw#cQtttM^|q$)tUCl$c^9@5%R1@+VO+=uTS-Av?GuuspJTM$X*@H*LhNJV@GFK0A6v0IdZ^W96H(j`4B z1p+A@+SIMT@H@>Tswg^tjhjiX?lTzEf6Md;5K6Z|whyzu9LAbE-Y!oi=Xh))0D&Wr z!&{w9X-5OUaR^TlEVhP`w~mkT zH{I5qE%)c4o3dp6+{%Gdzq$}@eV94N6~9h%BQ?+J8#F%au>Tx_v)KY2Hnz=ATg*8W z6RJ>gxjoqRghJY_g$(7!fRdINlKm!=G~moo zq8q%{y1gW3>0;!Rf~nvyy(1F|2TR^D#3%@4@I0;p_qWanzU28FS^&Z0b~3K+o^@W)~PJjh4b=3TsN$GwPS6r<8wcJA(L}^w1B0JoZbmj zkH_f09Xk&JC4%k!SAJIkT3b#dSAGqFC7;k$1FoCdE~(~zT>J`^Y8A<6<4P>V|7aYM z&TG<3`2DW)Ol1)J?DS}D3!A+=GHjlI>0d}|#;4l?Esc_u)B&+=wUJCZ+ ziQl2`U2eAMO6kIHLPGNkiz9ME^Jx=PC!Whg2qY{SxM%O1PsTS%m+g>4gmr*QohB!u z{TNT078Hg#h|PKC^3B{_o`qLv+~NTejbSONbsAuu>i{X1j7A(gmc;#M&CEYrqp zxs&RA?4ot;GhN0tQWY1@5kj6OhS}%FxyHIp3v)779w1E}1T|wkD}-^3OU{(*-5~>3 zYwBHYpLkD<^WjS9w#D&KaTwmV_2d3PxSt)~&Rm>*I?e0b+zT=Pw;tW1>!6Zbe;+3*NYZG`u}Yf%Z*#A*NKw823y z=`QS|e&hC3WmTbS#Y4tt^owVw%`BwdGIDxel^U6Ks|0MvK(NtycQVES_q(7jme;uk zf6ZOL4_WGnFUkTJR$pZr&=IE_(r^M5v|(xy!2Xd)^p(fSL;`y@Ov_lpXy4*vs8r{& z=Bs%ulx(NigI~+Ji1Hf7h7jjI~?TXPsoQR_TompGXY z1;=t1E@>Tze}a>;h6~W@MvhA)u#O1#K6(VpiBFcPu^zAy{46Uplaz3Avo(*ga=p${ zr^m~0Q=xCX+{==3^&a@qe@I1D$NBSZ4D+VK0Sb;ho3!<(fvamJ73D*)FS>g86k>Q1 zj(&)Gk8`<%Obzzr;b~zUv)cFUtPlmUbd{@_1ZYlFq1Cnmt*xDKRH5^UqQYij;a-Um z)iZBXMwaj!Yen?_{E+&>qRvDXhquNWQp2d!U|C%``pg4JuAn&z&Y7u!4{c&*#80DE zssU#&U3Dt=y}fo{I<`b20dWFP2DX_obmch3?i;(%?LCR;qvRo|l-6F5T%i;;sxb~B zk9A{10qHi+^r<6tYnF+0z(B~r(+vgr%5Q~(%?fuJ;#UV zriT>emd|}eP=}Zq)I1{KS=g)oZl`MW7t^Xw*++Zk!%1cvB71VeVSe9id>Cf1N0z$S z@0g;dhTURxh#cut9)7^&mA@r9`nouajFsSHV3;ArW$_vy7@}I z06;x336?s&mK;aEDKtTMqSXG@7yx2I-{cr!fh#RBu3LNi#kbWyM&e>PiZ{QBJ^xslQB6yX6){L* zq2!e%qo53wMT{|R1OUjdwY)Pok;XofFyeC6wNV<=LNEBBLUXhFu+rUe!4YjeCsPmx zbjioLAek36eqSu!!d`w#M5JoEMx0y`>1ka{Pm!2a3jaljOB};r%aW(RdtI@Ttc%dk zUg4_{76911U{pBar&ilPmiSDz=*5pdR5OZ_q~gE+230~34=ZG%M8WY-+74Wt3a@@l zl5tqj`kXFmu_P5iBJ}*1;-UrZhH)Pv7m?~zLt=sKrM>KtZnFhiE0*c75WDo?v;2yS zh@+77P&>_Aw{f;5+^E4Nq7#%Kmu!Y)h^`bednXJFb+2$U9f@ngJdH@#e;RZwqH)3* zf0>3TmaS}9^Wrl zee9c*{LSjRkqAj&XAmJK63w|a*P}`T=0qt{8mVuLRapf>ZOS^#5!44mYg-i?L$s91mhMtI$h%o8^1bUv zTnbT@Sa)sTh5`NdK}O>@Md3I(ae^SS==IwI4A_0imS}94_*k-~%z&p|jGn{o6_Gi| zpgcMm+$Ulaz9KQelT>dP)UK~~`n%_HCTAMAqpiI#(;{X4n`l2@DuZ>T_sW;A%INEf z-dQ>}XIw@qaMUKZYp!{3=DQ^nz?u+acQ82IkM83G-p!G3Y&~UAGoLEhS&wltn;Y4ZP3F_ZJGme0hcWIA`KA4?C6JD!Zn8Eg z7&qxoosN>vp`=19=23cjQ=*6sJlG|oT0A!Xj-oCj${J^|n;lo(FV{MuinDd08IH#f zJ`a$~5uJ*$^i)v~8BVwIxHdXd8~u!}IYE}2w8yAbXjssv)8QVMDWIAMz1_>(-FO-?9ZZI!%v620j-O?2}rzj3gP$I5Z`L;r*flT(aW=u`yx2cwuI46f| zD$ro!GmqPOpZh2vB2K?H@ub=4HkyfTkVOca1EK$*VmiRqX(ZhxZ-it#d4e+bMcUU!B9dn1Lg>e1>+ z_6suFuzC_QjBVGf@&LmSqb}P_!=|N|usMztQAbI9UuEdOw0!Oh0A;;mob{>}@F@nq7kIr;bsIHj0uPy`r zvKpIBy#YSf(si6ucjJpmBl-Xfdf*CFB(8t~83Wz)9$+hyeyv>E6L>LBZHnT_WWk`$ zEP!opXi1gq&fIKh$;qop`B;`{GsTPxYWTQj1yVXeuN$5lGr(>4c^*>H6*AM8Xeo7P;k-)>?zCdJ0Pv!@+v^ok%S7{X)Peuk` z=OLuAG|Q!^HBTZ8CSyt!O8?kTvaS7W5h?m-BGoJxyU=*8FV9zeh@`hBP|kqk1Fji= z7P7&oZN&x=BAu;iVHDb>%RyV5rg0<+KsXX^&FE$?tzaR^I?x%OVHtPl`d%fzei?%uXPn)yfRvNRU9v&`ADuIayeh!Q=pRZCGN4xc z_2ynwHH5O^V2I@|Dz&kD z%s}aW#DQEYT=U_Q??1(8!bX>a$Vf=Qhjy>s4Vj?h(4+_>X?N~YK!}HgoGFu*GJ&$U zoIq)!3QoG7xV%pGim;ZN4{n~PgR4A|U;#Z_)(O9Ol3}EkwOddA_MSD6L5Kr? z>~~9s+V`1KS@h-eZ!v3Rx?^5ng$Cio&qTJf7M8Y3D}J}rXzWcC6A3@=lvdzN6-!q( z;y*~uiqH~C1L;jd4w?8oT`$cf(M+c-pd@dS3B~cO2xO(CcDH6l0P7!uKVC31gqId6 zG$A-p?50#c95mF8Ec+~vfOs+DHo+XD ztf9U3RmtfbWh*&ZLHZi^n*@hnK6$EiYXuYh*f*N9R_}PL`}}D%H7UqIE?E-u*r@vI z)rVg~nG51UKlU{;o2b4-istqDUoUW~4_g9@eCeP#f_i^KLiSN`KM>Vj9+r^ZXy*VH zmOGhG`B-%h8kE^?w2LL(+^3;>ckpc{e~#N5wPQYbeB5g?q{Zn>URQVZeCH&mB@@`( z!~R%Q>&)^)NL$79+YMJ|bm6k<4sXlR#~*wRpqmHxWykyG$lOExb6{w#vc;FCO#4K^ zGoVQO0fbO88~xks26Cj6;IRyE=G3_PdKY&*HO&ul|N z0eSJ-z!&g=sDsjtU+*w}+KbVW;G`Z1iZJz&T2WY!Gn4Fl27O?|6 z4`&t0!`>AGx9^EQcThS0k(mx0Z6T>D`h+Rv(~N?Jn4*gHNwrEP9- zTaj&Uk{>Hzf3{iJKQpChP@9eX#;kriD)2V9#ZNr`BmYQb#Mv&$mLa?cbn?@B+M_-| z<@E3y0iS)Ap4R?4Ir&^R>e;aSAL0VS%x`&fwIHvyw+zuX^TLgw&9uNG&h z8O8GZL${`@3up4`aIg(+&5EV@#4fKY(lH~?Zx=J-+5!hAsExDRly$Zp)Vs4iIPP+G z{fh9VM6chp#KoXZ+2XEP$3J48eD8i29>#mMSA$i9L=SR6AN-E#u#z^gIe^lQ^oC@< z{!fc28FL{^#*ntPxhFl~gbVvgar_F4ObN9kGFeIa4-smNY8DU;!(BkUpzf(sr`8be zE32DN7^9?qXKxTDxgHdsZzyj)M*8kz_hOrS@RRiY#nD*7+-BZ*Ti-TY@Oe|bu-%|p zwua)DS2+AAID1WbvU$<_yYM%1e#jtNqQ~E>Re20;vg^i^=C(jpa{=-CVSnYm+EI8} zB%@1nKC_uJ_64dYWBeL2gB3z$71I8U@GZ5)s?T~@1I_fN^9!Rqs_>Efd2+T?pOM_) zRBW}E0uqYj$hTycID7;yO+ttmxm@{&^K<$4%=pNid8Xm=%&=LxI%vj!DRD{1>eVmJ z^x{u5h1ANpf$QJDp0k;=@IWAD#5yFyQQAJo@;iDvqFpP?>f^c%ogY`Xz96W~3pwbc z7G|tBE#GHtYodCme0}|eQ`nZS_fRAfs%tf0|8?v22SF^NzBCJgp{eE^dP`Y-gARP- zqHmkL=+Fs4z0;)$8MoWj=AG@nynEK3{a#OxK4Ct06S+oW?xfCNhs$1=4&c4h?agV~ z9Y(0%T)heY)%c|!w9W2MOcF0nx*+O@D|2X_AN!Bi*VCSLNgPG(B776DdQ7{wH9WCo zBW-fX0{HxPeMdDPDm`k@8T8GONR}G8x2~KQhsXXyWyxswRAp}6a`~BLdp}JP+3|Hv z=k$8DBj8U&e=n$)#Q324_*i~bEV$uM50l?e{|c^ZA9emlT%_u%&WyKv-9Um}8O~q# zx*%8O@BO-&%2(UD`e~0dqdh;gDqnkNx|1nq!=3>e7gim9ZGAky2=}vPkWKBVP`m9@ za2?l|5WzCO9*b$Ypy81fOY(cww0j0MZ#}|mtJ!`rXa&2{$W08en`pG$5IA>I&jvz&#dQ6L zSPK{_5i^{!?wgS;iLOfY_RBlhGX4cidvUiiucM#D>e1x1H0O@Sab?>hxk#u>_>l?( zUt5`rj~U6a+hKv*!)-Oa+sCk`JIK#a)FCY6v$ih$aJNR~;CJ4GII%S%pGEkNjjZT4 zJVkO>Dq{h6iy<<&_hhay)03$YE(0C3sxqEtVKlG!zoUp9E~0c$!k$GCzvCy4hnPdV zQJdz-;G8*vy$7y~4BNM<7Bd}p*(4Xn^cjr@MISUay!H?$D-JY)1?y=n1+`Tfo^pjz zP1)i$;+k7OkUSQ}P!0rlmdmqzXFc8PpaiSBYvbAObrX1>N^ae(mdnALOqvYgPnN1C zCf$3M#=EvVB4W_0J|h#w6H7u$k!fn8DHA~Cft%o1QI#dC)p&`Mu z!PH)!U$e9goZ`3>q5|Pb%wmLrJtEKUy-)KuE4c{a) zZNrxme5B1wG!m9TqXd;M$5VG&t9QYz`7NU6DOAP1CXI2uK>X$eDEgjj^#|{UG>DEA zb#|5<&|12`Pf**R6^~%HGi=GWexZZ?A;AS9BD)7d2sOyBz)qdpbTy#)vaQ8qJFYEa zb)ea8B@g>`a2Twl?|B_nF4K*O|4{BaMBU08rAc&E51HEdDaRipqO>ti{=1~Q&OP_C z3}t+t3uzZD>zNB@5JOYNFkX~gT%)hs%kzEsT}h~&b&9eh=IjKk;a*()pHkjM32h}( z%t2#D6UjPz;&{ZpP`P6XC)uSHXh*0e>>Nrvb6(DgXf&9~p9e`~A~2xa_ZuZz5gR{5&7;^ox!%K+A{1PH*SOq=b6^|kLL`ANDSTq0T@)-(b7;(cC{cWom%76K zcOyl3qdPx4s4X`FRD=ECBD9;k=OJu;flg(oqrunZdIl78Ra-(xY`tqq-x*zc0xcT{ z%#4nI5^~yhiaTjlpngYHArr4kX?ytL-S<{?^K#%PWT}?rny-5NM}X3P1ftif9oXUP zh&Ks+m{L3uN%hRTWClZC$)Nk!c{>uL2d)0q11j+pr^j{Ed7^${^19CNDO|OCFWZIY z!-%eGqEx=v&mny!bMiw)U@F0s`haSdHwpmzt+^<_jjx}~+a54A(maTXy3O2`%NN`< zWrcq*yp}j>)Wi-asqdREBpZW_;kf{9ZI8x{#!8WVBpDJ)ga|+N*@dSFfh&_fh-sN& z{&>!7v_eqtPi<11#=mIK-ftCp9gwDM z)mx77kuMvwPU-LRKMh!Hp40OPraUm){*9vmsipKu+n#+I(!eb%uRoo2TJqR!z0bqz z&8Ms#j)R~xhqH*22YY}m@AcIO%FIVpZ@bH+`FemhSOtf0YjyRe<}N`)KmU**#ZbIX zOU)4=Npkh5`*F{|{aHcmPVQMEDUR#W?vyY77xu{s4a?E&z@B5Oj)y{m`-#3m4sEUW z#Um$`EJ{Q9_^bJa%s4pNtD`vgKcoY{ly3s0esvSg;tzP7Zb_oepq-G$FvcJBq={xO zMPZX}VAa)_Us9QduD+&5ChD`B3mp}p@DAHsCuIY9%TnR{U;OxO@{w+EdJ!60@Dma| zBf*F(Y@^Q3s~umak&n6TKV8>uL><9hQGsDZ&0JlQi|551K276Ue}!siMOB|em1t+k zYpHsOht=l<2IQVoBs9vI4!h)Zxdwcpi*mT(= z%joVj=y2h)w6Ny;k$2JkxH{q?=OaMu;j2MCWW^2<;)6pJ8_)LAIa!E>*(Xm|#V(mA zaQ#_YcjW}*QqC0a&~YiP zC~Hb4K|!C?q=lU>)9v-o(*AMv!yyDj?WcghP~H!stNfXrIZj#Rxr9UwU$EG$LkRUm zJ|xtW=}3*5`mC zJ`L)cV#I2))|zl>-e!1iNo8%3?rKn2SGYn&e?8u4q%+!*q}7Rz_GIY&L^sPXVtwT2 z4j~I}4=f2o0LclhT4ZQ5P9z279oi7_LTJr|#i1-#dQKD5tQ~Wh0Kd1mKOvYMVltmQQcqc3W_A zQqx`RiSgN9nx#9z0(Z;VjE-Rv(aniiXir`d2=9PHZ+hD?^lKL!RU)8zmzcwyr+Wd^ zGwacU>VXj7Yst8>z6TzY*!-46Ce*p9gpP_s7#d4BBB;ms;}~A5e=fu*Z=IZG#7TT3 zyDoP{$8$ea>KA6}uWVaGP7I69CabWP0(8uyFV|miMP=cr!$1i@6xLRBHwD((+XSH$ zT^Vk>3nVxvaV7f`lsbJGp;+kE37QsLDbJ(t?wIJ)oUnOO3MY2JYD;mtL^P(ZI_$N# zA#GLFjG&&(X4Q-n4oIDo{*I@5Nrj4J58u$S7>BLuOO_3NjwVLgnq|hI6&iVJEYBzi zK`Xrm*3Y;OL6Ym{1$($y1=Yr4R&;|qXuR+l!as&)m*I`V*_dhPIRMUE_g#d)F0=+X zC7vRbve|S}p9bus1mqc;^EA27Bvn7)aULKze^%OjbtJsu7B)DIczJ*EbC^#-%U8iT zKf-}{J-snMY4$h661I`c7OU2MX>)BX@i8)mo~2vPNmwuAZV|4j2nnP8@x^0+E3={Q zDWV_}(baN%%>swL`}zZ%!`sGi1(Y((-ghXHzTPo&GL`pAkarGm$0Q2SmqKeG@l6;S z-{7Tw)lEw5&Nx@cZSW(8`{@)~bFJACG&*jG*RR_VjKwf69EK^!om2|PeGoWmZ08xg zy?Db$cAa>(OC;)nzRx5Y+1f8|E#Vue{_@&EV}fklM9j1LkC_{jU>|So5&kN$se;aS zo|;%7x}SL9A%-RB~tO-N99ASu%jiVKe@-m0fMK(}xHaRqQ zDGN6xB^eZxvks8wK~mnf+g>_>cWylW?*(|38=W5B%-FHihe-?Aw1pc`vNn|CJmWR|^YG z3s)P<|8pJxpzQuz$3JPi|IqQ0!2cH=FZq4RwyTXZ)X5dV^-nbK|EpU5hrN@OmwJhO zsVxw!ishxWoMh#tlp)UltB(F**7SCRS~$o%S~|U8cN!K}Hn5w+8^9%QQ9hCCcE>F0m}eVjlLFPO*pMXezZKN!f* z&GoW89xy);2L18CAVMx6h@X?o7{JX9t6e^RAQy<^B{MJ|4qhM^H{S~}_m8hIDv+0x z?xWPO?&X=J7vG8#7 z0J%9iKrlsMAn%J04@@MO2UeXt+^`AxdHDb!epnO_?~4o`4sO_xFrgqYHxLZs1OxcF zIDi~{{Qu$3!^I05i4$h|_q4E;^Ya6F`C#tcumSk^d0_K_d4b&guvPwh8e`-CxFi3O z&HpXw|0~^S|G+k3$e;ymq<>O{|FEb)9ALQVf7}-^M|+5s+skPr{&F%v0bnkEAefh% z|KDc?6u<@ofqX_~Zle zzYO~S!$5L!@Nja%w&;IBkVn4Wep=GMcfYmXTz7QPb`&H+QRE$qN8#aWaTCM>LTPDf z>4U4?1LA@U_?J$J;50f!?p2h z?_+Uko21y|%E_qjs_4m$`_7VE>Hbowrd63{#Vfd1!sQ5XpoN8HhhCX$X*ekgc$^%A z-_KI(>QG>D657t2tgaTf2An543e? z@BK){?yWY4RIW(8&s74b%3fI;ZcK&pD*yiJ@Jb-dWC2adBUoqL82KOy={Z0TX>R9R zXa^_8$=z(mucy+d=^>RnBKv)QQN)#ar^_t_Mw9!pzhz0;2sGW;?zI$tIsq7405~N4 zf?n1`L6%2+cWEiHY|gPdlnOF`>61_mPV@vE-f-FX?U#)9)O2bQX6(pR2tO-j{u1;& zqBy;OKtg2bp$c|EQN%UHaTH4<4kif({TN@7_F>5kgV-0_hDs*Rq36wgdaM#p_IiD$ z6>LAp+#r#x72szE$vNdUkGOg7|MprGNr#EZ1^>KCYAMKzK?vX%e2ZHxM?7%D8f>!? z$Wen2L6_j-Uh@gtQ}-8L2RA=C^|3z3ni74(k=^h$o}l@QPfrDWV+X-|M4qC)o)yPt z3t_%z)01MGx`32zLv|sLrr@0u$O(!hq>O`t9+l=jNsKYgUrdZC?DqsrZ-Y;8R|2$* z_=snHxdPTiI-OqI!~Rt`@um5%Qc(^Z(Cn!2`xZ+GYW>VJSq}?BtuYqk?0cLq=8e?f z7>9>Z4nmM2MC%`elkKr=`08D&dm;xs-6vLb?K+X_xKYvKDah+BUNwrLnJi+qSc|4M zBjWWpK}7cNSc=?&2d9h}XN|sh%=CC!`fvMo-zZCzcSRg**&IvG(g@L$%eLjlhC=%< zY5cK6YBRe`A~zZanh@3?_Wlvhaia`>U+pQKEM0G$F0;yguVKJhA*2iWf z4LT8G)&1_(iFMJ1^M&M8RN(iCIDa6Ns|oi)I4VD`L*l!A_omGIvonky!;-cmIN?NB z`!1rs*w2uUl*TCOBTkc^GF+4vAA>ezIkBQjex4aUc_f)J?I#}{yyK6pCUL#Hm-fg;MrLxARiDcNGZVr?EWT#@<`SSyFaQd^E(VwBeXqPem&Z>(4rBmuV*||^7M2mM%m4R7i z(u|UNh{{v^Ne#$XiP#g;t!Aqh4ZqGN|?u)p1HMedG!GmL!Vur#@w6owg6< zc|U~lWSVgtHEr-ZwA}=Sy;tr|TP}_zeuwq^FQNDGC+E(tRB>D<%S-qfsM z&TLGuhVV62wI>dP+^Qt~?DG4^!|p5p@80uS#p*?*>c7b#GZb|jwTIz#b?34=|PPg|%+5F7b@i z$eUA4P&6@E(l+=szK8Z~m(lh!zU-6>3Y=sU_yBMTR2 zS)h5I@@i)VI$``UbLF7mVvp?|<`i_gOAiA_ZUOEY@7>hi9FDyA&dDK7dys5T-I!<# zY(IX+wTlyKMLzG_ILtcD!9R3n5G+e#Un7dfBAIw+>`GV}+X70%enVX@<)!CGdHj{~ zdo)JeN=S2bHg5SPs*bXb4mL!c2T>g1O(bjJh0%StxOftGZQidvOvsu{;yfNCIx3>E zALBDQ$(x+73;5)^v~W_==8ve%=@CE zX*JMu)%W=mcI?N?vl>;A$HR*ah6x=zmtx&d_y<2~oY`7pmc;*F0zO24WHw^4CvYUB zAXp*5T<3P_)MHqo@Ynpis+~yk?MK;X+uDp=%QCZH@a{nd1@DSQ%8%7@>J_h1e3OJH zzt~e+n8eB9^L+jxrD$rt9@lr5<6Bb zB4rP(k9AS1$4{Hrv0abm^9(@V*h^~9PVIK!INGb?pYLbhUFRYyvfyF}6?YGjbyWbdOss(xr>|JU2q#`2O~4sXpt9tWc1nyIVU|HKjS%c| z-%tb}{4r?nH^LT`pcImgNr(;q@>cFGDg!+3;Lpvscfo!Nw0TrArqq_yiQ@x1oBUw$ zCy7n~5uSKCZ9*2UKN98bAd_oG7L`dt)xhe&Y(cD_WG9`~py#IVx#u~;9dgFD)+@2b>dRFu9{bHEDR%xf&)=KtWNo`EmjcQ*tEI z`QM^{e5aIqJxHrhORCasBY+swgUbIp=e?sK7G?L2IefZYmN{O!5*mboiMrn$jftwz zoSBJcYg0i(oOzI2L%Jx@7W8R^wr;?_IcJ0ls3EzX;B7AQjSgLo{2K$NIh75qa-T9$ zyyX=9q8y99Qe}|u0A6hpE6uPxslNDvJn|l`u{qryec6ERX-^&WEjgC&d-iq-h91XT z;dV5>@07X7_cpJWFjv3l{K38DYDZ+pXUjrvFBJSa!rGLo%*K|kVWh?Ho#3_}e1^2~ zV|!;SW>B7z$5qPKb&4>i4>9wXJt%a~-9Y-cT%H(e6r+dIH(lYuD+r_TVX3r_AMy&PTnwoHk``ND+bPQNNOCe!3KwF!m<%$w(RMG%ORHz z(lI2zrho3CsYDL8)VoHxBFH!O)E z3tZUom%k0$D3z6~G?|<0asK3+pI^fGwJ9AKTZe*bCFxw%-v*kB@5-g%n;e?qSl*KN ztoJUD8H9U%sFsdtVAXuf(mgBmTLOEA@Ac|=!J9XnEi>!USJH5B^F}dyl^^1cL_6*3 zNgnKZqe4mO!x&0&q7#hTCx6usB&ty^20yjPx%V#!kCAAZME7o0LUdfygnDB(Ts=Ce z{n2kch&{v8kBn3_wBAG$+NL#BOJ)aQ|J;NpFoWu&3t+eA`A*}JId^4a(|yYTKCTG(F&!JN+LtoaBXnj9DQOZ@%0D)V|>HMz50 zj=7RNVBTY9_SHEZqwy5?Tr}Uu@nyV&bXU9RpIWODKWgvDH8XhyESu3xTQqer5zE4e z#-K<@L+ApGl;1Hx-l?DL(>jOcc}LO?7OZ+)?Ru3`-(Xkt+z;}!RetB36(QsTofIN; zlo!?#CE+WfzY`jsQ#%x4_%0yAe2%r^cu4pCAxVug@W(rnU9K~czfpbKP%aa{V17zY zSrzZ*)VJ;WsMyyDa>aa9kh^ZB2T*V!iXu*>gkID{#6-|UR5eO>^hEGPcr{KgTrNc} z76eX$t;a9e8^sOR7RLz14absZL?Z1o2SP3hL_mVDhdWpy7%i9ucL;X~r-(XwLj6uu<(Avm9*zz!PvXlqt@D#o~ynhjS+p> zN=&lmV9GeUI56RSA`u$uP5M^h>Ni)?dW9L|d|N;+trMH1uGbFN=INLjKEws#G2KFj-?H_TVr39~|GQDA)th2o zD7Ai+Dzogk%BN_*zqF~eV08K<=V=5Aal3l_HLit@KdWM>SoV9S%EpkUUOS<2l-A7c zal3I8-k$n)XKAMKb5Bu}@|d!A=6+2sX9IZA`UAta6M%y>qK0OTT4NfaRi(>%rg~qq zFhsB+cd`Oy4F!Vo8c{e1OFZ%`U;u>#5mj6Y2Zs?}+_@6TkD`g`TD3gl@so4^)5kPZ z|M9PLtG`&TMGpub1P%J18uR8>69^wNN%}uZX&gF*M?AWV*H9KkJo4ntVF#Le2ESn- zi8yLC)n-zeAov;C?^5r@NPw+wuaIP_%^D$(s*4e!qD~M^&8!7i*sgo6o9dfX>Ro9Z zY+rh&HZw=tr#YVP@#pNdbNlErrWrEc7sz z!AO~;yUq>Noz$S%J=t>yM`mH0S0by@eLlv@I|Nf57?lo^ZN5;--HZ#f0m0%|ZDde}&u%9_6s>vOTh?oH!gi@Hq9@ z>iKL3XQd+MPqYJHu;ZtPniHI*2+tDV>St5zKgLHcD~>&+GnfN4gx(UD!g^uY`^CiO z7%O%Z^pxKTY zIi&Lr#LQQFkB=W{J1U9PEoFk|9aU;Gj}8!bO=P=~`H?O+z~8s=;rf=RaI=(U33i~E zqtnUzMy*84%-K?|4N{DgW8HmI6@-eYOu50$7^D~kB{~y0*@&9v;#Li|Lm^DmI^4be zHPp%pZsyE$<`Vt9^2b*>?nN1+8Ls-j4$kI_KBs8VGAm+iDbQ&&)k`k0WT(^vm^z*| z#^iorXiecdFcl^+nx)gKr}4Yw%_ronq~%x1ep#|Aj6)X zQ{rUw;=n0o{%K~Ky+*u!8P1TF9#iqRj%0QPK`YhKhXCI}@pCG$GPjrtDT?~{*Q8?a zr*Ga)5c@nj3Ff!(R=j5D4RwZ;Y76oh*%7Mcd?*r`j_3YcX0|X;TE=6AOwj8tsYF>& zc(s;ac`c>}NhkIbQ}LkHsQZ@}7_V&E0a* z{cPvV=^&o#0a|VP+Q4tf*?Y)Vw3_kdED7@5@>KAoQ%Q;Mp1rMzNU3mnH6v}3<TC@=ZOG7iM$8`-TNy-PAqG$ zx0auJ*!%NJj~G>LXC#R(ff+Y z!n72G#~ zX;C^e;)vzWp*;EBO8vE+;RNawcWX;}bnzhN+%#^VM-eMD!`KwPp9Vv$Zfp`NUM{!g z&f;;GLT z@?S~|bTmFk_gx>DPwgGZ73M#zzV}Gqs+%x-G})EI;}iq9hG4#J8Jsxyo9v1duZ);5 zWBRMY=c|WQIIn?0aH4jZ3XM zeja%{ig1Oc?RW!4s0enu7zaq#um6%lBrTchPgzTg`fjwDs|-VN0WHH$6h&e-Asd64 zpDlHTy87g#x_Z3MWIVr{?Mlp`ac0qSm#f`Py0Cv0Y3luHlC*K}u%NDk@zKMql|S<# zc}iPl&0*{L5)=ECkRNnm+Qm#@XKOSkF4g3G-#9i)!dQP|%rlR2kYgo1A&80z?@on! zMLt4<5pF0unI2@O*`meTVJ#*UlIC`u4azRXG#L3}7F(c7aUP9Vm0IAoLuYg2cv-;w zJli8*gJVkdnfBxp?K9ul*^)PP9u1?7PMfxZW^|zpnr}36G$n~DsWi84f%;HG^4#`Z zhQo3r#UgK}y<+KvwtYBl{K#WMw2ZQMLEtEUx@px&B8r?)qFlSG0tE{W6>9b~-OeS4 zIcDEFvk<{IHcRUU71S!7C|x2zTecx+bVFoF_{5n!)YKB1m5i@td3dx8O)s^a;^);vWh^_hb>7Gc?v56zsw)=V zOx_l$+D*qG;&U8G*5_57h*j(n3U$DRF07XTVP~}GEdn@$0#Wg z!K(y-GrrhAFE%{LEfpi_emVQC8}02fmU=RFcqo_Y+n*v_<*46jd9i|to&8p#@BTqn z$?v4zY1v`VJ#ld7w@I7l(g=0?&*uhyvJ(&b@J|%Q-v1AA_ZVc!vMvsPx_jC-roEce zwryj&r)}G|ZQHhO+n%;<>+N&S-uvDg_aFD`8&OfKR%TXZR%J#-tg7FWD>747%^>7M zfQxFzs_*^oW$Mfj%fO!@)kM(>k%AJHe#2Pa82%kS0oaBoeZ^P)f;`?$+}(x<6PJUz zk@9zNIb2l5?_h5As$pkaDVS2STp$d98qv|Q$Gq=fwJD?im|zi+M?x$`_l?m_pxHl-GFJDE@Y z0Hwr}FTt-L%*Nz%v=Mg@7PGr;b%AYhP-k5Rf`eMF08NVISkJ$Og`TjZoP;zg7V;pWd% z7CjHd3XV+ns&BTN_8%BQCLRZzkmx??b+YXgP#dSjKXr#KB8%&?o{w6Q&ma7Oi0r#j zy)#+^5g$hZhgufy*)c>}q`ZTYI*XK#kC0k5LqMd$-l zi;~1?>x}HBQ5_(^+$e@PxD8#bRg|e!VC3H=Vr~5bQvt}2bKJ!uiC#I?ZxhYQ(L&ez3$pI+6#PqwT$`Pd%>Y3Ob*?N5#qFAxX z-%5;#t_HPo9~E!SZ|UyJAIgi$;zs=KCq^(o1EtrDH>;G);zZ!QpykYnn)LpVh_z=_)w0R@cL@;H;{~%8DFa z+yHLsM3g5cBy_rs1{$H@K@Lbuz3*>4%jkRg0QH32%NvPx9sK-sW z%WV;(c`?w%`-|mfaO!ddW*+OkGG|Bb=L6`Ve(7OvVAe})Ijr4KlMoeD0Kd4ZC02pu ztWa@wYM?4(LWzlsNr6daePX?$$&cARMn@zn-QL21@Cs~pAnXL4JPNAaJjfLUJEif- zbuV&u%7gx6Br6JC#_vWLISHjAuGyv@f<jh?_0>a2k^0_!#oOb z;M!zwyS9NamO~CAl?BGOFTIg%Pg_$vV;smd%hKC4s;vIZf&dwB(}NzGQ+ z+tE2UTqS;J|+drALuQ(MB%ERyzXyl8?jIi+>T27EgxP9*|2m7dY5nAiL(1iY9G ztq0AZUKnzrCh(;Adqyt9sqQ}63KA_JzDg=LSMLS335bvp4^Pso^1uRmyJdD7c*k3A zn@C+Zd~3UcQ@OL^{KrMW^U~0T7KhCWpy;4XX3EPR-lN`YF&WzEsH$H*lWgN>v*#(o{ zeeGf}!UU`Qss~aZtcl79LmkoE6S1_|$4%MlEf0B_Bpw?(gO7kX-Iu{vcZOFrfdqfJ z{kh$d`Z>%RB|;*a)MSc0rj}GBsXF2L>NYl-;N=vaAatR?QQT!P4s9B2|G8{PjccKA zmgHMTJc3kc!Dvi8pfaEmCygBmYB1gcJ zD5X=c6S~uL^Hm=OXmd8t3qNgRH=)a_lNI>UvY%sRU3%`A6-iMg_cbXp@u=~*xN^xW z)CxPKD*f7Ag1s&!7&V_THCFcC3+vQND*I#VP3)a#9(55(_;7fFzF^?DJeolHf)3pR zJtR7#j;ji;J#2BCGqQ4z%w4yE_c~}Y8(^p!Rt-eShwMRucZ)=}-Rm>9P3Gn8%43F0 zE^$)5*7~q>e$s}!?XJ=+^a4W}+hyp`{a_0IC zkxYnxM`R9wOw!tPzPV{j)TXJN@bD+%y$f|N%XtQ=Ib|A}EDq;m?@R&798v);3a+Rm zQho?5=$jra5oshu(0D(kBJ+^L&`h&&aAm)(Nt#ZOPSrN!YXRN0B#c-H@!3#43P4N| z(gFS$iyxHD@T*hAsi-Bw3=;OEg=#TqK6raPwc~-xj6Q9O7Q&TOa#R~uF?1@JOI`EE zFQ1O0OsaHUo=vI7j&ib;1b9@jA>ND;DM@@jjbkzr7}yQs5ng%-Wn-_kD6Xng6`Z~N z4zGV4qONMS7L9Qj3c1NyECrga!t=Z)+c}L@p=+to>ZnRRyl5r49I!K_*^%sR)|ih8 zP$o(0sm7{fl$2%6B|xZsmx>@!c1>#}7o0(w)@B;JP=d1|(FK|0bIb=Xq$8&1Pz&0z|{$%>16;cLc7vx2VXg-l zXL8w+Vn%0%f`UkuyvM8?PUx)1^|GFDOL2=5o--ih8{j$P+s{d5&wAMvBnz<`2yMh? z4TU|Y8HgNq^|HRO29o4kKM_kZqjKZR)7`G|>QXB7DMpdVKnV+yU2b)2tMQNbld)Sa zFNC>Qqxa@~|4gpW$tu!DT+hf(j3=uyle?X+(DZLPpDuXN4(6c7?aTry(P-85X}I|V ze4I$LQhcg@D~%DK&|*NFOx;-0;2)s-ROyRXA8xEvw;wp^zgaty=2X*7s7SWHNp%ZD z`7s7q)6u)w2eR@VnYYFY=d+EcZBDG%a-;1AP!||yX!kscX#s3iP-28^aRLm3D#St-A<6hC7 zZsX^S1U92&{h=xywRfi&{u^wh942*>ws;dEzRTKW?c`zcAW@pB!6rF1C5uP6!q~!G zYoo?pu1K}Hy~D!AO~B~%#vpHAOd_#3y-vWw$*FlwsWnY_!rY1G%+%b*sT7r=bbUfY z0&A6~lZ%#hZGFS%S8+;SZ7mo(g&{97WssvX6zrRqx>iI*Ky|9W&kr=cR|)^HU50Gr zYkK=euro6xPl_`nO$1HBWl~O3PRM}awLrUi0SZRF$fNJ5@;Qis1e5;mZ;^bZz#>0l zjM(h)DReIey>2;Pt52)YWRFpIejv z_zk_eJvB0}I@WSVKwao#KD-53UtStrGGaI0;($_aOS!GwdB2c&Bi$PKsntOtO0&~; z*6;Z0eK!te$&EX$KzdX5hUZ9Il5{(xH5-NXM^AJKhc5!uv72=@TGii*=?2$uzkrXT zZUF?%c&83lv%^PX8dPc53G&ct`?Sd}Gcfy5>mm-|n`#TrMV>!`${LsUDlts5{s&s& zqdl~0d_$`LIq{ZNx2*i2ew1nx>&)y4)Jt?95a#Pbq(4&7)6qib`@`9v#dhV#(?%jb zlezZLc&Y{`wJO*HUhuTm4Ivu1kFv?0s)3y&%sBdv>E5N&RkO`41jaa*Zgf^Xz7dP0 zAeDOk*3`KJViw(=9=hJcVY~4*v7@#N<<^h{*vk)JyGA%?ecsD+zITT!V-vVYCK%;* zz3KK}eq~L!#+Q5C$q-7Kxiea+3D5i#S;t8gYiN5?{V6K;4!TpyqC-|JpOL+6G>-#v z>bDXxqzIs6Hi3?oUp}u0o;?oBnK59M^a*pnb8p;M%l79r8N%*>WQ{r{;?OPwUdxZ> zD5mQ=b0pjP+m5=X4|IyKm4W{UU;h{L{>za5U$F9vvi=WNsUbIJ+DD7feoFC>jO)88 z20b|43i~t-?(hN>SuYLRK&k({)m4@6pkB<2#L&i|dM@m6`-jYg;Dr`oHN#JnK+76Y zw+(^bE8u%{YW+ly-{RTPEwADFv7Qp0(Q?Kry*rx62u~&hvEYvjkpZy)WROp>b2WO>)M7P| z4_?a?%M753!HfZoaq+BsT=MxjtYt~=;|`@Shv0%&sn`0O*Xo7lb;kUCua2%VLB}&E zsMAe;n^~93wCd&+U2rt%{(oQXe+&2jC3gSgWdFE*TDbs%U=_^GB7Yu(|+;huSr<{#j*dL_fO(ip)VZ$H9_oO>GWR{#4>%U zXXY<0{jY7dFYx_O=9jPk65kBeZ2vOP|Ei0bnVFi6?Q8dcHdy|{kn#Wi{3nB%j`pij zY^?Y!f7SGNiN9<94+#IOUe>?c{CBCp_xk5}e<%E1;Gc~Ds_3t#{FB4R$nb^Lzx4Uv zDPP3=>m0ry_+L$8X2xg!!sCCRJj-9~{hx$?HvZn>pM-xl{$ki)*XCbf{_k&ppNefzL+wMZf=jWHv^8Hs*i#;=enCj)e`M zj`ho^|9#!)Sn0ny`#)FU3*ys$NqpunwoXgS!1h0p`u{UB;lBp^e+K_wNc}%CC;ttp z|1<9Y7f`2VqNZnI_!8^?I@G_&I@On^r)HyL{0g=Ck5Fg*8i-%8|Gx`$7CI(s)~{HB z{|l;rMFvnaG10Q(Q!&tgG5jy7PESk!SBL=}EyGuk04*Ei|Dfvs?H2ksRcB=W()0f} zRj2=oL!kd3_tdiv7x!Nai!7ZF*3t(xVsVWoag0(>RIP=voy)bi!^!4RS7 zS>>to4D7w3S-(XN)%2j6-V?OGVMT#O^Y;~Q9I9|% z_mzCJsOD*TuD%u^ja=Q(gYQyKxKMV_3YB7||H4?E-{ApYJH+(V#fr znl;se+H-Bop(d<&gFS(ru1B;*>=lXo7)`##N4wAlR|W-rSEm2|ES&6bu%y?{HM^cm zop>3=>PN;4!%H+c>xv&p_!7#RxK4bc`vX~s11bZ?Z{9K)A2xVi$7`&>NtGlFdeYW~tGxTJh|UY6kSnbI`>q)sC&npLQ~*<{P&7Un|KQ!= z@}%#w``k*}bf`?EyN@0D0P`j9_yBikjH9r9AH9tz2mV2flQy>t=?(S)AV6jLlQSmr z2qmvckHe0(3*odt#~m#Y*#?_l9=SKnSdfLEyP*d+UxoG4XT}Av;WMK<32WDTJv4Y% z6&zW!cHhmt$@^sNxyLC#;h@)sS!cy;5>~pgK zxt0~;jJDREJ=|w~6!gv_klHszLC=U@gYep_Fh$z)BYdjg;oQ$-0V<;B#RE~pTH`0; zWAL^1r;6{P?)r$gHFqLckO3I9cB=r;q{eW5G2lc`?IRhR`w=|guzH+u?^GopTL)dQ zIEc`QPJ3f*SsIvMm|0fP_fHOdnNp_y= z1IHdwPu3p6pez%rqmK^SLs|N@&pp3PHqr<(1$DQq$d`tH5hXKQy5G3R!Oo)c2tpog zD{SPY+*Nfd42&I1vzvKrzip7uV*?!u^)t+;=FL;KaK$6Pg+M4_^-Jx zzEJ8}k33SJn?PLD7QAUK1UE9r%0#}mF6u}ACmgfb+a*QGov3ml!?H%Ps3$i0stb7I z^cNPvkem<}`S^R+9?jg2^|!^)$s1ZPzz?-TS7bbB&*2LE#9Ux@ijElKNqJIzokupw z#Q3Ws?APpUt$YWTi3P5zG*v9WbZ+(N_l1AveRv@BwO~{ddh)TkW(vLzBR{CD;7tny z4utoaY%BP<;d{eG(dWkwrEJ+RNS^06DU(=!8CdbXYE~7oMv9ys(wya0+w=-Uagd%# zdOW8cJKUEYxyI}11i|-ZCu!RNRUGhbA$J8CX6`t_pinAAiH`GXP^TcZ{KBpz8-xuaL00C7>_o!UY`N` zxvPOZ$G1ls>4r@sl3C>&z#8?P8U7w|4$%)(CcfdDb*Km5S%^92jmnu&m^Kw}*-)sJRR!U{$Sb9@3X-FbW+-lk9#59r1?P*~enk z@hI;Jd9%l~ga<|tN42cfzrogPo{gitk6E+d^?g9OS~Is+i^h*buInJrA~lK`9OW4g zr9|h4HTT?1B|w?@#u$XPKzwhT)!-t-d46yb`K51BfOj~!9p=3HiZqcAb;_U61)H+B zUI=Mb?92(Ma6!Sm;B0ODp zP?#N1WiReOV?o{eji3gXdeb*C&5O!1s^PY=f%jg$j(Yf7J^0Mu9L%T#y`s!f_T#x5 zMZlo0)^bYZFWJfTDu$%=3fH5poVHSToQ0kOuXmJx465)9@?d7s%gj(OA{#vY@-|8w zCimwN@9Q`wT=rHLkl__s1i2tRo$ls@%@}d%pQAJPT|;dJo%!+jl(-R*<75U&ifC7T&JtsZd63@wVn2<^jg$R!^Udou4>cFO$6Sj|E#+y#VeM_(BvW-m z*KqC}G{4}FqeXv4x_Esqv3fmkY9r`ZD~=}f#?++k*wXxAKxBj+N-dsOA)7|v@of!9 zFi3a-QzptjqV}~6q63C~rke2tRPLHl^eCPE8mZFCdU0rR@vwI1`vB5KxIo3@wafR2 zX{;)Oy}_wNXBl?CEN*A%51?Z6`jx_CzXQL{O8Ah=yI`0E5w}y-Jop^=rZ#1?JlTIm8UAvCBh6EiBad5FuP6+ zbTYM2;G|rH>|;MV7!wI^Gu05c`WBU$CHX*|L_)Y=%*(pq&aFgUH zuHMpR{>DU)?)!Fhyz!tb(kHEFhBueah^|SV-A(bf{HA;_aV7nZbfUI>W?MX$bWY96 zXuO!iQHnkEJ*B;go7Wq<=f`+9@-8%uIAn z8aHNFVykqPjkFV3=QNjBm&+UK+e(Ro65CSpg;Y}+DM^PSX6XsZ4lW1d69~Nf6c0_F zrxLA8mRAeUqRoAoTbAp?7wEUgCyMvF_uF^qcbTW97nYZ%7pK?9C-|q$TfA$pgHLu@ z1PCzp@1r1nzyOe4YEkrb^i}o3B(?czEE%b?RNqubrgJm)g}YMn3YTU>zKPmY(erhO z00*7`GeAR3a5D(p*Hi=*1Zg8Wwy=cDIuw6a>@j!+?%B@)O)n5%FW}9VP zYOgxyh;p@#2Irxx5e;V7uY%{{t1As&C0{HW3(XGkMebAvaW9G<71$; zZjSZ=3;6tGrL+#-i*A2>nc*>O{arDxE^MltqwekQW^O7&^PBYxvD{<$uKj>X{diGY z7F^~eTZUulZAtDg(4rfe2KEHI(&v)FB6%6&jM@xBQZLr$^$}>&sjN5odG_arkq**N zMYiR$f53fC1lDG91mnW zqgIB~>Qh&Sf!@M&2G!(6&w*NDhtBjKy8=-9cJ;b1!xwEKaYCPN8$Cf*cH31WZUi*x zApe2w(Y6rjU(|H0D?&aBnzz$H-L3#TY)iZQO;O<}hLgfw~)vJsF zva8K0e%VnCM#2j!=`Ti$S=jACHaE8gEGcr_iHGYSBsZz$*FXz`>m!~F`RRYDgRUa| zAx@^_j|^#=omh@fG4k^UWu7m2SWsvlfj}nCXt++FX*IV&P@rB=aq?zg#xQ9=kk>{#n}MpPAJs*5jN8+JxQqDz9N z8&G!z8UkRxVP6qABg&*CvWo7C>Cne>iTnB&fF1t{1VTunn06t!Xrdj$5u$*%jCRPUQE=CfGBB%s9r44 z;T9(w*w8kNI~eH}QF@@vmYr>gItkB?<~{OriaLSPNYx(84M&SGy%=l>u>+B0>J~Rx zysS9pzCtd&q8!tLER~74gMtj~yr|T?B-On5ZgB)-o+Fk_t*G5ewsa~lvI&yIjD9Nr z`n&+TJdG4Fl_>Ll40fvUJ0tI$31BtZa9WZ*i~DB^F?2`2VXcZeT-p$>LD(|V!*{j8 z-?c{8hwd>Qz0B28<_v4Sj@24xun#1!92~=Nd6gf2IC@z%Q!D6aoUe%*`bo=CcOoyy z?Jv>;B9CpH2v|$dpTgdVKj^iFw6;&5!8&8CdNm9f;IFCN zDK5s>p5Z#9&^3Hez0^VUG7@lyuVR-dVgkgO>Ayo-uS&Cctv$h!Qa9^ zA$rDt`gslW?BLz%y|I3NcPg2Bg7l2d+R3_|dxPN_>)O(~sd{4b%=DV*+D3nBddqs_ z_~7aq_S#B%GWY;|hJ21!U1<#~?O}I~dhK^zZ$2|=f~aXzR1mgQgE;o8IkT^Vt!;mM z1KsdPDZy8TdMY ztdFG-7c?w%15&jW_aw=kl2FTcCe56jaJ}LVl{I*HRpL(Fl{K|{`<&MvO0xz1Z2v)| z)sKEl+zj9;JR6a@E}AKFLE}n8btm>pt~xM#QvFksWkjVy@xXv=B#4>)!AFCg|(R{tpV`y@z)al&QW9o<3 zTHEEsyA{sU>)K_z+tsw&CEmx@u-wy75T1wo*%N7&r{m6E$&^f1%Ra}(4UW}wWRv@l zTIVj+=5^ww^O#fnKF5bqr(U(ktF}(t8q{jzh*icCJqu}@&~3S)RGPjaqh&nqRnfD2 z2d=or)NX;$7q}OJaZ2TKdke728 zbYz2aqWf5Epl#{_;@<3`tL8b`%v;&~A)V4fIp4KN>3k7x(P43Le_m+F&y$awQdz7L z39`f8uHyh}1x1tpA_vAvu;jNLiWWZ}WDp()LvwCqTv2MMUWIW=GCB!*4TQ&0N75CL z5W-88JZt(jeBI4EhTKa71g3os-!M9tWQ(25Zy$2)~IamR7bD;rGQohAAmkUjOj zQ4tc-qY+aVt-M!qqO@}x2o>jEAi&YmxSU-&^}dGWo9X|a0zs=CL-oe|7B)_IfGRi*f6O3L{6+s!HBJ-FDW`# z(?0q~I6v%#U?hrLcY(%uZOj9`tBcOSm{RB=%?k(-#da3oG+7G|&cHB`p58yWn0+U8 z2V;%=I#DW3vZ9cCQ?L#kvKBrxrb$3zO`qVkL$jtEy425_MVLRKLn7A({!d0ds;^2) z0f8#D?Zd+1@@rW7=u3Lw$v$`d-uYSje%mD(jA zab|Am4Te?QeH#JVgw=X9I&{Dedtx;a7W52J;a;vRoFy40A@K{s<1UX1;>q^HPbTq# zSbok7frFd1ye0XHrB}S)rIG4r4JbDv4a>9w7W7u(O&tWDeOGsiFyP}L^=e&wii)d0 zZe-_kaU3HM|sE`aM?`0)Z7C% zEgiTzzaRq4ALj}FJAN2GO1^U8VR5z?dLTNG8gv_#5lH%>#UVT21yM9I{k3Jc+JvKN zV|(iqq;=+MMsc^h$XRKPyn$80u!i9X5|&ehf@7)N`ID(i6a};4jU{eejmJC)DCBGL zBOqEtU`yq`GX#EFk|BBD@m4CT=nhw!UGIx;fTP4#z$?azX|;aS(kw*B_utlYct-=q za2?GP#v_{RA8;?k=E7$}v@gV-UD;aQFs^7!-Tq7b{igkz{fPa<{*qHWk1(Dxb92fk zcEkKAgn{_K0bg%|2^8+^+Eb6ob7V#p{h<-#GkeQYGW$(3`pZHxyQd*tfrB1q7Te#> zG{o^4DA255^?*d=DYYY}Dhu%}-#cpBQau|c;*K@vI@#Va_#r|wJi;TqU~mQLa-Mpz+F6C-+{cq zu|}5!9k)SwWOY_dKOSwOx+0iB%?jL|SCCs8SM|Afvm53c=Bp&r%&;^SsI134ptYi} z><*ftrd0!`!3; zK~u(62EvG;+=-O8gQWb%8rV-v>qhXkzUr2Q36a>!EWtoZ>b|P^*E*ZvYt38biP0;; zmAy`C^RQR6-sA#Sq-Sbe_T*hKzm^ZQl%8+@|TIqr$!A;ZvPyQLZ z1e@gAMS=XF|I zh|#{~y8a$glp0rYp_g2%Q}n90ly~8$vS2E?f?P}@>gX1q+ooQzo(4xQWTZP-T5%!r zedXA5d0~KSvOqsP*zk*>>+!|4$!JHvI+|C-omx6s(>J!vX&kkVX&_K7$j`4FTSf{D zhl4R#S4U`w7j>c(^nAE1=-XF^n6{J(E=VkhCsdf`-jx0+)mMxiEo>~EFwIk~S}a{K zRhd{&fHhTf*fXd7_2*mTh#_A;2gvtkB<5c>sE?rm8URx$Ctk8hhrIO{NBnjE*Fe6e zHm;m9503@Guph;Hr@Z`&meUnlL9(agmJ6k#S+bgjP0^)Kld)9(l+*+B2pL*Z){kW* zh%t<$V$G#(-UwsqLSnwmsddHmDCyU~pfds%9TVRsGd}j@aoY?(u5C6Dq~WM87n_Lc zxmy_HseohlW-W6_{b|~ur>2s^*w1_O7-MH+HOg_GONygJjn3H<5*^D(K;mt;2zYea z;*;?bY)??Gf^E0tiUh|hLKAQGJSdY$AQI<{yQ|`F%^1bmqy<~3;^soxEyN^#jk0pm zs*p3RY`1id6YwNnzb4RMCs>P;+mUt3$3r&!3T;+bCK@O}Aqd%$4{a9BmDlga6ROn} z!nGi~DV{|Jc*WZYjbThKCefu2(5HMJ8wm!DOV(Cc99Q@3V9Q=mVEK^2Qzf&nVwSeA z?XJ+5Gdl7@#BXw$*Jlfe_=|f0*aHHe*wz!&lOrLBy21t`qQsgbf3c$Xq&@B=_uZZ`JJ|a? z@pBP!wmfBg=JvMC3GZCz4>&rslq;caP?sd1FXWagqTqL8NL_|OUJDi+#DWT~2aYM6 z>4*9_eUS;7iR6d=d3?GiN*nM8JWfJ|tU!$?v@lz1ylf-lB6b;2~t}m$$?uD)&?Y`WRpBCg?77=93 zc{He*4W1fwP|X>y^LIx9P8;FQ08H8-5C4V(Zy@;`E|k9S%7U#Th27e$eMWq^_Cans zgeBqef@mVRQSIModZ`~Q^Wp5HA^NV|Sa1C9O!iP12e8JpiPH(9%p7?U+)Damr@D!h z2S!;#!b8>j5S?|fB7FqlStTE4B+Wg(y9=c65iBCAxo{(eVt30^llje5LpjDJ=0o3$ z`vJHJg9vb6lOroC{VS@GyB4m%zUE>-w%rRDHW2kDnMOUV!{Nq%Rc^L4Y?h_PYMIBl zp)&%0gWd4S>lfL`M<(isrvfbiC&crERps@m^ex@adoC{VcUhJ;j$tyY=nTX zP&h%XkU%Eq1xVt$qIdTwR%76DWM*hq;@(g{a!ko4U#KzQ!?Z!(#LM%Uj-@V$@Fjws|T#dFbE7Q%kvciNR8qY_k;}2s$#i@3uK0gvFm(V3W|T%P?C((jJo8?v z^qS|JtW7BoKKCAA3?khM*$0U~B%n>d&n7Fe$4QaFBUv*^Wz?{gSc}cH#*BK9nQb~Y zbxmkA;1Xq-MIQ|9Qb5I98642^M89gj#01$yCMA(h{g>|(*Si(XPGbTTRqcB0ND`Nh-*FcyD0N> zgwXWqd5C=cP^Yg7ZKNj@x?I#@pVo*qHx!xxoC`#!Pm(MJmc-*IW^D#`zMeB`1YphQ zMyY?3A&#+^=*KvTfq8;^h$q;uEr_LFlSV`lFaOR_*Q+skJbEmqEuCa&)R+|OE`?qm zqCD>ypshornO3g;+7|R^iUTOoN!VzxZ`1*{#2EIx7FVlW7xLA zbqF)Dc?8xks2vJL@+V^o>q9+-UuXAODrs!jWYVu>-a4T&ax(7N6|QBnX;6gGgd=n8 zS@@ybkS>Hj@31Ftz|XcF8{utxxsDS>jhGP+-N0N$G$Wfs(1b-Ud^D@UH5J5Y6%sH{ zw=zG0I)|>qJK(f-elqgID8cQ00!!ybLs|Ah#Ww&gZO`M$>U9=^h)c4)fHwiSzfU`X z(Q?9Sj|-=NfD{^jGyF4AzVl_yF2ufZB!+wfb*qFl14knOi*f>OvHKq`94~F}l5tAu zl0^|Rp>_x9iMv5`M>&8jeVeVx7-Ss^M-sF^k?Q>>l?7t#h=PBL&1)w)Ht1-3@ojhk zxSn5$hz4T1m4lZ%vf*-HOboJ;6ojecSSQ$^(w!RMorI3!yWQ1cEfCz|L^ zG3iv5Rhpi2FUS_r07B7SRu4-;o9IBR8i+y%#x~_L*Rqgp*T39;?XUX?>g_OTxkF6e zZp>z07wk4yESWekjjegzMCf)QY*~iDk#AJg zTh;bfb$@^O=iBOpf4a&8UlgC4;YOBc8cm#jpxY%u7ve5F>4|A%;%_Ri_lRnavz8Js z55VfMB$FxwP?>NzZ(AxN9WZjap;Jn@C4kAY{=?IIRv6Wz7J;y{4Sc1z9c}Mw zVDXSd6z@d(aPL2=3x((~K9IB^i(9khGZh*>dCl^U`{~cRKA0<1l?P!5tzrGT6IL^v zMFx@UBxSQ|4Wnx=&Cc46o)nHx)&su0QT{njx?SgDOgIulyHG5UB$D~anCE=1+DY8m zA+v&%(cPrfwuTRkSriOjuqs^CjmB`OtRsni|3YW_Si>j*o)VL;PHNMTnD=g+{RqOyj%dIv2`pY%9H_${BYGiv@mnb znbyY1(xCcDKEk85*{IsS-2qn)Q7p45A7&bo`>**2K%Bs=0U-S_NK0Ua~X}M(eOLwuZzCu&ya3L=}-G zyj3h1aCDY>;S3ZSCju1b^2a@jd=Y6?<|C7`LzXmD!B(3D`) z_{{>>vH-Y&ZIXD!$E4RbqxDb+KF)_)|4;Q=;L}UDQZ3_K5UIF z!<6X>+8W;V6;#>;DAQaX$^K&`W@qP9g0xe z8|Gke7_FgVK&F+?uPwW1O|GaU=Jx`0w}h)b6^*32NMRjNbuEXAqRsLF%5#f0b_gu zr6(%oLW(IY27~LZ@z7XAeoVndCm^7>CiF}XsgYxgwrak~FL)YSTWr+NXYSp|W9SoT zis6^9rs70eB9#w9rV--E1_p>h!i{~&&M5h)#rU+tx`G40*|i()TZ1}js)Sge`eja4 zbo$To&y}k>xxV{9-4T*S3!>?#u)+=H^2c=LiT6Jljc^k-0wLq+;Fs2_{Yivk=gH~g z%8vW9!RBAA{C1`a9FcJla-mhfkW7xQN9Z%6XJB2y@VhC$Z>#P*_W~xDS}Cb3B1@Wr zh=P4!sl1S)Omd_bl19e&?;mzMd*WErY^6@*sj?l+1j?e{nxC*ryY00V$9JZJfH&*;1;v@N?^mtfuc(=8zG|>5D zKZWf+_$0VDWgq8Pr-q{_IEq?r^VgTyUV>(AtSFl_pcTh}n!9bN#3*zQ>nZ{ExW011SXZlrs(bHqGfGOER*^Xqy>H;+| z;6L7i--SrU{^ArgmQ*oZ>j{OFeG4z+LP7S{0b&FKC98Q>Npw%AmFwqA_Qeik;o#PU zC~n3oiE^&HRfXJ#f0`yTJZ#9Dxw6Gle%#oL$vdFLRDM3_y+Yn=OsV*KJ@8C36(|#X z?F7mWXF6xK(tAZ>;&fbz=+qG(*I8scpLl~-9!2b_A61ErwNx&5tQK|p=kNoKe^ErZ zGp2CoA`xbx=SIigoZi&EQ3vCs+hZhwmcxa-JeIi{9ky-Rv5zA#&zk!AP(N%Y)m%*4 z5YrYQ5jeJcp^KsY!x%4`RuwDhE!)Az0aeXJeP2~XGy9ZsDBMQdPq1zxeR!a;r9qAx zkweNg&%F*3LMBvVrY;|)k`7$biMLw-Ld>267?(+oG#};%rQ-vI0T(qP@|f%V5Bad( z+VSz)TB!Lvv3>$OEmJJpq142<37k@I+N62QI;h=|rpqp>x5ekMZJEqfN%jjKq15{I zw7Sm_rA{o)kJFYZFENua2_~ypHy%!z7b~_`%{ith9`w$;<LTXEk?q&bYYbK8bSScGd-Invf;c_UI649YZirLnI+l?4!nQVLmy* zw~e#ka+p0D{jZeI9(~y&;+JWEZheN}rXZ~do})UJbBl+xE{|Kt-#Y8pGXK0^T*ZLU z1oc(HOl-`T3e7apc)XhlherDz%A)skXtOsn7$+5=G^iE;4-N=6q@v&yE9y?D8=6&? z;+kwd%h)r;R$RzmfKip5y0U0@+RH#Ha#JTWRKcSBjufzw6gZa-19 zy|C64vlCYa>B3pP;`TTRY4;#=+n6bGq!*=bDFV$B+gcY#a!Kgq;hHYF!Cm??a)3s*d%K|KCsz8D5KL7cjb z5dmaD2wfpK6=y70-36xKuTrH&C)cNm%E2&EF&j`yrnIsEJb)?$IYu)k&ls+sI~Ppa zj2#AFMRQiP^l(`fSFr9tGsCyUK4;;;RejyTTA#xD6HMj;;YN`7%u?-i%@#E^?ni-C znutl>pxnu=h>5LuJUcQf);N8#9oMY9Nu7VKx7#Ta=dHiD5g$V6HjFyh$VpFs!ScG@ zn{$aS*7WPTY6yp!ipfB}Z)@#~1CnI}!aVla#=CA)Z%ru9PmrOXeA2QqWH-{l4OgC} z$76hXp|#1t!%LVMRb#oX=oX`9TAf**!4vQ8kkFwlW|GU9DUy(r-%RD#jAQ>l#_l;f zvZh@a{$yfJY}@Ww6JugKnb@{%Ol;fg*tYG7ZTrjpoO9NDo^{Up-uJ&=dso%2+Sl&g zYgJWWzckr}sfDg`6KpS4FI5za@%LczL1br&Z1WWB3=_H2l7dNgAoD}s24?Hx34ZVo z(Q}yAI6752)oKHVazh$ctf&bx^LWy-!Fq8Ai70`F&X`gJ=sc2K^sekY7R*RdmJ(Lr zx`tWd;)Ww0rg%hU9Hi_doxeth=qdDx$x)~^JDx9o%nt`s)>Z;+9y1Rql!<;VX~_12 z*wI4CFo^}n4Yr6speoDsq9?pte3iE!q(@Qj z+j{LFFXHeeBttWx+*MB)8-niR7J+Vpkv?ns^WpQ$oQJ!ao1O+J)%J@ksmV6ulZyV_ zY5L}8Jn`z`_%KQdRD%j8QrZ*v&5 z4`z^Js?=1g(o4$cCAe6zr9@ip)L4u;^NCYSO_c@`ArkAt^YPsE+tm{Ugxe?Eweij} zrfU2$GjA7q&P*A(braKaa6?_l^%%LaQA<0InA8u)RI#i=Igq;QSbhYI%;+&goD)B- zp?gHC_Igs;@X|yIA(0DahV37_U~at2SYfWP#83TPR9aNp`QU-0dpw*Savc@-$LhHe zMyXA;Y9tj?Kt43Kp5t_sctabK*L)(IyiG(S75U+9>JBia6;&Q=@yxDP~9rrdO5$E{X6yt^X5r|K`JHqnX z$B@PKwYIRU&3t6Dt=lIiPA{=4>L?pHB{Yw+?bQ4MVcXlSH-?mC$F@4LKH*&aEQ7Oj ztSATGvmJR6yXea`6EA+Hn>UA8j|9EMU4Uk!I&gK$ZZhCH$Rj5P!zD9++Mw6mQ=Jn< zaA`NSTgV@X5PTR9`N*TW*fDT+l|O5p@6LI#Q%$bgq6Z2ifu-K;mTOE+`wTw>qh-YPwag?EdgQYqTfp5IbncZ zzf2Uwga_aE8Y-4j9tnhsj%=X6go{q3Py()hLjZ*rD-jFOhx4;xmJHA-kLxb1tvJ9Q zWpx(qS+kddKusZcr+f_;PM)J5N8qBvQ=rv^1{>Qmufn8W3)M3MTX3s;AU*2ykb3y} zA>%rAfJntfxp%nPTD=st<*R?I{`4!((fsZ&w-}FN5+}oF5BFni4+S4dpGy~(_jP~t zRjP=(k!<{1rSPlpQ#>R351&7qY#^#8bgM`_*g2?$aTfyd$|j_XDpdj(u_xEo)|Jto zEfZ-e5qCF@^LJVip%lf3X$JGC`6-{rje9=<1Y_w2N_&tL2HiI8eNG|8)z&o@?G|yy zt=HajxaXd8y71L0WzratTPfqB9h(MBE41kG0|iTJ@g?j`Iy9Pe#sNC^=78(y{y@|a z(Du@BVeU-wc==3ZMn1Y^w^{Nc<)T`kl8IEEm(|ky_Q$pYX@&z#m*(k7-2~QQ zmhPo3pBU?k-x+oVR)M$P8=%3EQV8kA1=0^wM#`;X@XAFZ2b^a0Z!tJY=}BvfZ)Tza zGB-P=6cUwjocYYx#fAFVT!#|U$vIIimy3UbYZBCUA@gxK@C$ZZq_kAwCKXFr-+(;u$>zf zfP&lkLgE(!n8rbPo5}TcZGHB}LFnNRBi6t?aELBr%$tiol*LCn{B<@@*-NH;{O9A5 zL!WG#b2ck?G9X(Q>vKiHjb97vR6ELt8`*Qe=VhKB9=QuO_Pzt$#JGode;|8EjGIr9 zUC&Oc%fsjh)>6rNbg5#rROh8;hL8hrv@OvC(BTgeRz&FW^VttUnl-b10;NOwChd#} z5P=lDjYoR2qhIBII9eito@Y_Tt@_ZQ-ome=_DPvn$WcArY|C5?Lm?3?onG}$TcAgG z;nrK-C?Fz-!i>op?>VnUkR|tPQ`okhBsc=4%nBPX2?z2u(GAEu&5#C+QFFFG~-Luvh!L zi-Mr!O9K3==cjv*^tdS*hlu$jyqbiyzAKvcO!1W;ChcMuH~u3SmUY z!g3ORp!EjV0W({#ekE!$QvW3erOx+@8l5|VR=STDA$!}#7xP7i+iOlow@aQ1 zoOUmcn`+nZDBqM6gEGls12VCNx@VWMZ4ZQKeswk^*3&rFIyN}lC3XM&73P#P*Dl!6 zD~u%usii_x4dm+3F+t(p}t znAd*&RBej`nwI`-{srOHIS(ZiFL;!RY3_*sBPWe+8MboPABgXCRJ}w-KF{fa9(8o* z6tG|QxD>R4{s?~Rr(+g#+VO}3t+C@Kdg}-3O2r{Z(E8zTVbI%98%hQ=&s>Ntp8>x$ z&>tj#MdwDsWHL(UqXgHavEY!P)zfkW*{u-Yv7e5@Xu^edV|)9{Z8D3jX%V)YP4IBj zlIH!c^0c|scUf4D+ju3Q6xQ2C|4JW~S@jFf=kwP8{7l(Ve+=DS)Unj9Q-`H>$9-tV zuV>Jd6h*u}nyry7+Cut9&BDn-+Cm7&;FpC?#v=A%t-C|zj($aS#zycwVtyK?a>Nk! zrM_>3f|OQKtQi;f-zeoMN322L1~i*|>wFZt&8mx`vH1X}fXMfX{9!0Q6nA^S>pAsZ`3^J^#LJVvChW_sw>@I2ympQw}x z+fZsG1SnHFJi3Ke#*af$c&?(Dxsy=KB0Fp=tqe$jDFPr88xEhh{`ejaEt5%d7Y?EW ze_q{LUySPJD8Rj}np{Zd)->7t(JZwwP3aO7dG7cmM`aTqfRP@Pl8OZj-heM&v-0Q> zBxCZ>PHb2{?#8#RzcRoKN4cZy9N;+-Z;eQX_&q7xh8g;pN{*m8OStgx=e#`my?4M4 z;ZZ*o1yMHCVRveR4VnvQ{LOA&psE=CnbfFiG5P-P$=TWo@?oEsKobrB4?O5#M$BD2 zaJxmL65zyU?b+i%a4LKUIg-WQwD#_DPjvA7R;ZM=zgT%DbV#+s1iS&f&bt2e5k0nm|* zR{Vx8MeVty0jl5Ea{)DkJi%OvUe!+16YAu-#uAfYiD=1kWL=|VU3C*yO8hLXAM@+nuw?hB|#|U(V{UFqJQjx_la?fSLICLX9+5hY;FYqe>&jqAY%DMiRt# z%9C%@%O!4-Jo+s+6z9k`zj&A$@8Zt4N3y*{(SwF)D~C2xCYgF)ZNHBEf|8MGb^RMLTm7 zc^OKjMVBI(h13Vt`cfiq4wQpiqErX?6Q9C+J`{RiM7dFPY#a%<#q$x)v)L6~!YQmn z)>2nUN!YUw*^ftUY?yy)r+s-T1Ke3xL4(!h zL?c=NJyR8%XymN3Uv+t6rE)3Kn~B zxe-Qe*Wpye`={3SK25+*g`1p8YjuRi32;<>T{`os@yK z^G4SYhHk?|B-xZ+yKcb>y{D?$^hd{3PmeJ>3enh%yl0jR*&m#jye~&@_XEaJBu+77 zc8`A00rf*MSnpbdZYrOXI0ttbD|5yG(xT^vN4AbiV6S>@j;8Gq-b9gZRjPxg#f=+D za*KRS#oE~AnbR6TtEZ*ef2T7HD{kA>mL&GB2vAz6L@E(<@h81?qqGWF6>>@VQK4y_ zyo#`su-3!#v7gx~HMd_0-A8pt?VdMyrpROWQu=GUOv8No!fw6%J^lmfob^3A5KD1eO)Uvt*NcD1VwgtBM^|`$f z=7s`~7_t|}SA|6^ciuQz*zrXSHrc@aiwU%zD_NT}EHQ^yem1Aw2*o1n1m~sqx`L%R zq4>V!-l1%wvBw8p+oz6v8v;>y+Gs634EYn3ZQGg%QQ7sXXgfSSuiY1}(cMuz*_q;H z>OR;54CH%rA%ACDyuLw6#XA)E#NgyRZUA=!_uY%@yVkpE0+wP(3VDigN=g{i@AFly z76y$UD8H#H{OWD7nt0MeJuaPoe4Q@QL$4q z<)DEwuJxxIN!H;-`f!?Uim~R}-^^e%n0~uw7{_l*8MkY94kSf67Zo(I)=39kWHs?| zb$&8Xs%OUjGSQzxh=F7`rDA`AZwZ!xC@rikv>-e$J~uc$#TE(qMFQxv+^_Pwh$%F% z%cjVP;kIepd!pZNClenSvjWpIb5zSl)j4M=B|0WV~oWTk9h919V$!AVv{A*glx)f+T=CugyC?hFjDGLpU z-Mm*|QWli7wv(!H*Ak9Z*zBj`D1LhS?h~3MXKwQ=cfMpnJ~%f8Y0Zg&6o_x9H&{z$ zz@Meu$z!3JX|kJ8%24HD`?A_HMPK3O&H7upV@+|lVRHWEBbR-P@A?B|Pn~^!5H*38 z`{vBiwi!puc{Mvx!YJ*d70EIKAiyG6%odQEV%B=ka-4QvysAT0>1>R}Pn|4_BL z*aPjQB~-bK64F{UHg~RitXzz>6E^bK_ABAL5PXD7YEs%#O6kgYyp{$URL|TWBRfj1 z=$yJo^H~;qAqahO#|X{mkDpaJJ%+!rHu~|h4P+F-=a}uQ4b-JDSUl!4PtB@^$J;ex zOEV%)CaL^`*EB^0&N4$&0_2!Vb(Zs*K&+^%?bV2N_Pr2NHyYYT5ZXH7b%`4lheBiL zb*8N|SuB5VK(A_Uxp(IybO=IlWNpc;o5J1|$UiwgxzwB7Ud$M-tVieTBDWdhTFWNR ziv9Jq>OoBpPC>3iPNM)rAt&c4^jWL%%q+ zkz>+ux~dULlaziK&-+ItpG?AQ-KHT2 z^?dj%J$@p6QW%Ma-?9%wo=X+aL|rOfohVtk-7hAK3JJYfX$dgxxX4r%338Z4(psTI z;P?bevGcr8Zm{!p=hWgt`7W}F^f@0#sgrZIj97GyGAzyDzB9h5m*`r{+eJ_ViF`0u zphvw=9$ev12(2GUr;c%zVUsqeRWx-YX?uZtuFHs`{o9JARw4K?9|}_CD-n_EJlG;&=Z?$gM)h z2ojw})j7H~oR%G>aBvont6_#SRkE{ye8Pg$Z9R{XeP*+WuY@&L?Yu+T`S;wJVtD;$ zP+X&Ui9aryr-a}BoXD(MaOraygr_vGDVd%GsW1`Mk9g_i{yD0I32G;{_EZW@2T>h} zus=+22?nDJ-7v1Q4GrlU-K4T6nk_Nrn%{DlG_@=3j6(n^;&TxjgFG<+F#k2XY9eyvBY(PdAi+VA^RxiI)Y zlI~||7b<1~0j~fPKr$=!bTcaoYhv0$Z9#onsOah9U!B681T(`2C=m6y78R9eGC6NP zw*JTKdacbN1_zk+z8L+;+pLopEE#IK&GNp~*q8V!wVP#+UmOw7YfUzXTX>ub28VuW zD81kRVy@WiJX5>zrldWHanh+uJA7=%a&^n@AcYCH?JH z+Ho_YsF)9tlWzgq(-z6;=D&b*%c$qj4~Z4-66AcGIXC6E`-^FULJUGw42i##3e!TK z-)AK!)^0-HnmuJcRPM{)+E%R>%*h5h5{(i?^9MskD<8@Y(_f_- zZ+^aY!I)?M0BXGfQxeKVk?-6Rc=_z8(o8Z5N(j72e(H8$R7v=6UIjIc4X4!Qs1~dP zDoVO6Fu51E(GP|(T&((O!o)LEMd3=uPrf^WC|1oRqXuY5$M1PYaeXb(qf<2Od8+eZ zTu~ZEus=}e-K%W6p$t(A{5X+SOW3w zzg0FMY;J<54Hx%3aDi&XIJ{*Xzz(^^I7+pi(5RE8kF4C#TIjQ0QzQWwX)Etb1 z=L_V;ZYZcd{9_^-7lg+M#)$Ym&=NhKcH5pqr|yZ**T}A!ZCfn^lGe|Yb}~eQP@Uq8 zk73JSb@sKq)dav7ch=dI)OL7zQr30CfBM;0W-_>v{|zehxE)+{NNHq$Y%6B_56^{U z%pP@bcn9IB!qK=)DUBXNy4PL7D;k^MzDJ5!diFB&&Clbg>dz}5ytLZu@Z2sgPwlM* zc9#^It|oiMOQ)%GZY~`W{BWz2r_If4$l;#zHC3-q?p*@R&Ik5xItF)^y~dOZ>zf2m zt&jdY%}0zPh7A>#h86YhZYHeDSlJiZXNq^@#77Pu9q>rwMNqGpZ}>N!6C&=a;(Z$1^Qn9I&M?klZnm=Fp=vR6i)2~)Hw%lDcwlRm|&z|ReC?S1)5ES*4`EF#3BiC&uDtjbewD=!yq zVhd9?ehuaN`LT;r=24F>Fr^m8RbLF`ZcZ+yHw?rq%!#UNu{D~YM%1lS`9hAX-ReW+ z#@?vKCDte3D8EMILf*Z-{N3)C)=~Vfr8D(q~xhB>LQ?S(=od!3|3j2Ihz7gQ+Fh$Ia)H>Kd1!%dY$D{!|l$i_Z7I zxoge@q}EzaQ^Wn$Lt8KJbj@IETb{cUy6rE6r_=WrVI^){S0aAmN9WTqq;5r?E$v!t zQ;Lt}1xl5%#!?pardiJWK%#rT#LFuuRPFt?GE=-Rfz_imHDJ&dwa9+6jl~S~J>Hbq zM-0ps73Vppw!LJgQU}YplfAY;dW(>bWv8SLgOB;^I}m#1!2YMBG>S$!2C~K;R6;gt zEp2fB#18!ot#M6T5ReiLN}IWmE+_$Am%4aiNjl(}s*>BnaPvDtuAg+`Si(@*h6ncr zLHq7r(sqnXMORbLegE(S0e`9lK}x8GPV;G+#YNWo?Nfh?jrInU_ayMKQgzeyGHPiH z(winwC@%Mws73yYJKa4Z$WIZu*{IwKp zLiv);m?6*9lR;~RgO~W&jJmxm(G(R}*&kttTn%--Mph}Bf)OD*Gd-OgmKv%WhAZdp6SxG3 zIKLRsgW2v55M8C;>Fwjx7Mq>#jc9k$iSv>NcV<~0jgzhgb?^!sTvrEF#?7U3Ez*&EUer*^6SS!y zBWi`eR1BLe>ylO$h)I-`>?ZV#B`b~#($RJq<* z!avL4FMiEZa<>t zWVSJj2l}yqquMk@nRS3Zrg?C=-u$s`;Umae>3OpyUb!zY-0s2~A>7JY69+LzA$SY2 zjOI)Gvo2yeu)R+15Vc1+%~GZXF7bDp0@1cav@R*_>NvbzTk|Uii$CW9k~P1JKlg}w z%i%Ek-cUWl-RWSjPOAU-A>jVI$uW!V3EXhUdNt_r#QuQ=GNo*SvM*7pk#_pE3yso&}Gr8IX)q~d`2A&9*<9eh} z#ITcx*-Bj9Kyy@Ci%gv4TkGn;4ILcJcGzJ7x=om_m>^6(&)yI(8ram8ImG;{2tj70 zC-Lo%-Vs3uqCH>EkZ;eiIWQDNn#`~?pe^$kQf#f?rn<9iCKq_WE0zk@pVe@mJ1WOt zqz7a6(V`J?Us4MKdP^!h)O917@RgL6>x|3@6ysM=*(TPWH}7LyMjQLsRCm7KUM*)= zTev#dmzKZcUxyI9JQm|DS}!^Rz=(2t?>cDaOuAu_bbTt|XP22(5-)Fu4>qro)vuI190r=#;_*=E>&C$BqhtUDnBO|9<@w!>aJ5e?Tp9Iq$riLwVWmi$&FTIb|qd+_C zKUIQ`Tp-zpWNm98)!~AMQSSW|cXqd}Y!M|@ZDDJ^uuIia;t!fVuC%`f@V3FfTba3) zuTVoR9)h`}qh$Z?oWucT=RhcVURog3l;lrX+xbsz!H%+%5 zQV8O*JH^9_jT8kWMuvu~JN%H3qdxl)vvKiPTgD`F*dvEiyOO;i6zTw# z<jbzLvbZ;kqlHD}m1(NrO1&~dG&kJ*w_$az^oTvs|z5F>8%NPLKVyrOcnY*w#Bq|a&6rOmm2LPj=^ zn{R&goIuZA)&8sIvc@-lx07$hCxzj)waM0dTyFEF3hQszBk_9g47PUYmB2mnkoCXZ+`>rsJl;0R9)% zX>Y6;dSAp8Lrzwolg2oZZ(+lq8fzl=Ux47V8EAR@yqZTZgh~`^VbE z6F$yXiI#=VP-VULq3gFxq{rpx*xlJF%WCe;1b#pV#ar(O&9kDse1%GUr>70t^7qa; zs=0fQ;yL<5^h=|e6H*y@Z}<*!#1`Tv&(AcxHJTgMt4e0g9t>`@3flTD%EYkA8s%mt zQhyRo^t!6L3w?$(Iw(%1xu}8)6e#V3)l#t#6Q`~3RU#isd>#jR124g?_(Fb1ul1G< zo5$-D=tsIUQ&&B;A|AU`Q@=xeH2%;PtrdY6GV4x*L3Z$Q@KV-`t%D0_vGGy{hS%df zm!J~uJ}rhp`nR#1+wK zPD+dKeuh?Su+Io}EZv1!NcntwM%&p4$rxCc5UXwCP&-#YU8hJBd_ZJWs+`>?8&|SV8i@v;?1p17lskwco1Q zvgTE667!UBez|w3J>}+2@v(S&`C#`j!#(IDx& zWy?HGdD0PMRy4?r5;c1Ar~wA__?=zDO{8M(fO4^ zbhNO$q#Y%z+&|Q;5}9#kiGYy@b1k=$8R-5vw~&bW%)dM^pf?;|;D`m1O?$8V^~In! zX5w!!?>ATwMEvm9b;Jj^*=zDxIBkqcmoB~{U-|K~#b)R?0htZ%eb_`foPD4W$v_v^ltR8;^k z;`>l)pZBG?ml|c@c<5A?4Ct^$P#hAC(z+zBl(`W7Diin z7M=1As-%*zHpdk|M^2(=F+*4BaEY!6xfOJE1Fi5belzd)$YdsP?e5kahzR&SqHD_( zzXtGW+z}AX;E>{Tz;r00ug?ZovK)O#j89mn>c~ky6ji4m0q&zd0i2ECpC8^RHXS~W zZ!dVg2tU1h!Yqbm#<)w+&!H;ScKR2>=N2T7!fw}aE74#w+Ld*b6g6oQRZ6U{T z3Qt%>!pnP+k0{GZ08VOIrIjZoo<)t$D6JSbrKTIxdd@C~W-mS&9fg*bDl<#PQ%49h zj%gdJh=uPkvwUsfd9K4ohA)yFUT*>c>Kb5J2!2rP&BPVR5sVhc8KVsjl8_jy2^Jp6 zNG$w5ipLq7FLExvCEI8PF^-!-s*O&My7UnUbX`U}ry^gOgK>0RKlj6f>Ive!K>7Ov zTq^au3}&F_i!txn=WN)Y`iOGc4w^sIp=tTS<^xVXh)}`0RnLW3G1nHtkLksNHh8s_ z@g7VPdZTy3sbxbHet(U{8zP;%Vk$@36@yX__yM_P%qWSNreQvh^)FEMp z69g#$FcSb)oms`ND&Q{`LD-_$t`pbl3+>sJ(p^<<#u)~1b$M(rs_Tq#8_vyl^NIC^ff3V+M0(QJJikII1HBR2B|R1 zc=h;f6QvB{+lO=SlT)Z zmiH>z#Aq!aKi7riTPLZ?_5MrjNHyA!f67E9T6XP-9ekPr;cG(0_5}kRxYx7v0R&nL zy`QnoG1U7$^ZN=UYb)0Gx0PDJ+3RFExe3-BMqFPkU~u)J5^zZ+qHz8OcwbtAV1#vnI#m zF6dnsCn`}S-QV3{oV1^o-Q!Fw{rNJ*}s1Es)8l{40=6Wyta?Y2-VKc_8Td(+n1BW%OyNRK>eSUYd0mg7x^`IME zyKK6h=iK+5K%H|S7~UtbE!ldONiR7eG`}Tzu|a5>wN!`p;L=p!iHec>dC$sd}#-RrPw0X;)Iu z7ktCst;gOSZj;iJ)&2sG8Ux~SI|^EHZViD^+v%7wcXN_((6e9h@rbE4_IC)2C%y-g za{MqVd+PMpC{1!5&@rD05D1u$>81RhP3?{yZ4DUyi-s6T_IDVwK?(S+O5xj_1LkOc zMKBB*lj+vCX~D1m7=8B7r{JW9xBiuhd>3udp@26U`>S|qgAUWHi;`ycbE3kL4;b2W zXl}~#&0x4jr5r>TI~o<+&w&6(==+5kWfqPE%!_ia%JPv!H39gK(rN$3Voa2htb4)B zKy_IpP*qIHyCl2dSOrF?{1}!<`R4fF4Up=(38z^3A%J|w-c{q}cd$M|{^OK5OE~#} zB%--S^xVxe{!gGE&^BY!kw%$@c++)s6oAJCfvzGhT$_-{; zCKCs;A5A6;#^TsFnJ7Clm~5_#PRWYk^L$cAE)!c`=84I_`xngbuy5dtJOp;Z8Jc9diBHl!@(lBdmBRjXpulGpzbtx;(q&UOX zt*X1%W$~BGcAQ;c&J#@DV~sq^WxDiLoF`n)!JX#dn~&tj4Md?z|W%+R2>g``RYC0yBxt%qG|H1rpor$VJoaP^_&a2Rw}%E8BZ4e>sh(pqCSi;?Gl;vinN4})n;&)K%0d_@ZhU|>hK3Km!DG`+ z%f!)$<`ahoMK^bDG?F(rnVFdZtd9yEvZ+4y(Y>xj36`yN!}lwt3DWt6_i@HujsgCZ zd@%ymPUj)`smfd3rZOTS{vw{YXRfqBYB>~F7ABz>Tf=!Z64SvW5LcD z7%`MQ&O>bAddW(^H&HFxhhsBdL$`qDgs9b@^yvV70z;+|#JeRZpZv3GzlrlhP0v8L zPJUKdIa8wg$(anDqk6sht42qu^3;&U!|6oPj6?vjt9kZ|RGPj^p44}3+pIM95|x?p z@ImO$4mimG$~CvoL&Fz@GWX)C1$F{ za0eGZ8vM{a)I5yXwlPoY?0nC*D+iLa$^5V$2U@gYjsO=*RKdcy9!G9(2NUZ`wPNzXluBxx^0MH91r?$KBOb9F&cm7~3(go?9}t(0t7=r5PPZ z(XSBz#VKmnpU*V78JWv𝔪hJX)CF1}+vmD6X+dL2~2fzNx;(|76w!%VSP8)%T6d zn*Di^>d1?xh^;nw2SvZT?RNz~1D3PM9P6HGHr>vUh94d@WQqZF62y^42nx|Nl`BLD zP6;cb(%0~!OpK(!Y7DNZay}}kT!aWV*j5_SFcoWPZ22uyuhBt_EkRZqX|7g@wsE#g z)xZs27ZakWNj2ajbL*D`A}0uc7j=xPq&-(y1!FQV+29ehy0W#^4N){OyLv>ZWTCdQ z(qVaB2Y%a@>{O-1jZv=3pjl&Tv_$Om5w{S5Ua1DGTcckcxDY@TNt#olMP)*SR5D3H zwciC9`}L!sS17=sf}rdzLAJx-M9n^>F;WgbXa=6*CFQ|S##fN310pljpH1lg>Zjyf z$pc%-Y0iosrV_=@?KQ7I7`HUHGDi+%9m}mT zTdY`A;b>|CmY>(J`CAFJaiomEIRV(BqodjS>-}R$cc0mEOJNnaFzW zzh38)_Y3m8Ad(OCvV?=?e!}k7V%GgH?hzB=|C2Js!pi<{B8qn0hIIfT>d>VZ41H6C z=+80>eW@}l6o!Y`7GB>V0~6-hoQ|t=NZd7Y5`mjrpC5-2&1DD;7beI)%kpfD@l{Ks zy#Sp`F#F(N)g*&qC;RORPouf4@@E<8MrprGoY+O(WW?W&=gk^n-B!!5;ilx@tkITr zbcg3$7C5Y{2H$$WxswB^Ve)bu*gF~zD)D^#d#n^`iXU(5?D!BOst3Mp`(YWAh=IqO zku9s}5{{N?f{(~rqam5z%bI-On9O`LvtpFZ;4Jje0OJDRQ6?0X=O#4rL-qp-Nyh#V z_bA9OCgq0MfnzNoHFea4K*szXfdnST1uiK+CP$+vwjSct@7z1r*h{=*UZ-glqm!@= zqw!mrAqDTCj}cQh9<{0Woe1-z-M`RVcO#?}FtpJ(qJm+NwKp`fH?uY& zr2MaHUk4SPY;7%#to}iO7{4AL7zR~B&Ht2@{^=tNBNII{8z-waAq(3VQuGBFu`qt| zK3_1?7Z}9B#P%Q3)IaPLfRl}hkd2**o`r*rU7L`V=^u^i3!wT!h5l9iKY^+LQ}fGr z;`l1V!c5Q31mGZK%P(M!o|BpBA4`aplkw{~Ga)M*>({ma)yjXFSO8{rdS<{^F99r!^j}gD z6Cn#H8$IA3F6*CK|JX)f4X}R|XZz~)KW%*t_n(jdW-|RNbpL}P``=Bbe+~UB%>RT> z=C{??)9^pT{6BCe77qF^|A^zi<4pe{Mg5n}#Kin{Tfc6`f5({^|Nm4bCPqejRt{Fq zFXoAjlm4qF%NIVz%t_C|!OTI($-zhuVB_HU?<&*3`u(3{{4bS>iH(u@|DrN+vU2?U zE9l(AOif8_@ru*sGU$K}01!_IMi&r~M-(ssqG8Kv*?~mS5fUTHBMMKam^Sw4Iis+S zVb+^RN@y5}j!Qk1*xjV1NFhr{t{KkIcB+#{FwMSamUaGRPJ8s~Jbb<7ow~bs%+z!$ zJDFc8SN{2}rW*$F_fIb&qxWKmLF@0KFF4;WwB`7^ZiYHpRKCHj|Ip**>uN95;VB2s zsliRyFDxo2thuZ_eDC-CLAasD>mxrlkMYnUw7c?+yrim%7Qttu0O?#7jQ7}SaxME9 zuK519t@3$XOZi~*#J3|I_`jGeO`YZ+1_(ULu!x-mKR;UauZdNirh2MDIaFuUSf~$@ zmuQZ&=Rs4h;H+7eDrwEH2@STyO!Y3jx9=`*(x*@ACnd2^#-E@JYa&>LazfJIJVhQ#wlD?A|W$hV95A`72Cr9%J}2@ zQ$SghH)rOA=^^EtI6(U({&h<;K(_IvE=1THasE@tt`~I&E|uV+;KAK9S0hSa_=v7X z0v7~Xh+Fe(OVNk!{`32fmXuX^M)_6Ib98h5d?tzp-D%7j@h-m#(dpJp#0=7Abn;oZ zB`C;%V)b7J#yF6z*qpkErU<`lC;U)N-@9OTB!-~^#}pdA z@9OjTWNNgDf>G^wFr1`)$a!t)N0>b2Iu2&6DX$9z|^83D~1Inh_PRV2n9dt#X{CtyX8tEWi*1W=v&&N4MYIT?B= z_veKVPBMzH@`kcrsT<5|zu8950taQ|2^@F14Bi-m(XCA!>92`6q-y>+HfXe96(Nt4 zV)pD!8jcwq?wfRGzf!HxHDt%~-|K($Tc%Cc)LF$1_^lHR z>u|5pb2^Q{+J~vu8La<&g11NRXYLqf2VbIc>T%MyF@HriL>z~zLJ9Y8sr!K(!^slm zDr6zQu1V4bhZ00|g&4tF2Z0^KItSI%t5tOxCj5&@Y!yt^{zJyMU_7vPwrzeBYzaol zf2oG=n3eL6V~XtCCKYNA=qyBdKt;?zzb28o1|BwsZb6c1!98RATqTJobiZVa9y-Fz zvN@l3z!On}N6yc65eN34<-{`{C`Xdm>jitgu3U4-i&Mud9QP6kJ=gKro7n~HU*fh zY|dv>=!ji6q@Cq)J>}I8o+Cal0nn{`3$(%QO^WDr}fj{g`N?f56CReXDaR z_Hk}VcwKJU3mL{YWeDeIQ1JALS*4r5$bH~7m;J-fUf-NARmkHOduR1oBi=A(Z4Ifx_;o6)~b=TMG>m!*!QTcprw zHHisx<>h5E)o$!m9c_|qCjL3PNS&wQEOg24EPa0|hR00FVxn}gReAW6tNNqO{0VAm zSe~u#lga)uAFLD zO|ISlN8DXD#TB;gf=)t$OK^903$DSP;O^46ySqCfI0SchcZbGl+}#>)oSmok%+!A8 z-7_`c<_~nOuC?x}Yjq#zbt=v89{!<}t z^j6>U-pXh?yY%1g3xDr?-$9D!|DTKT9p_atb2o1$8>XLH?7aP65d2qWyKjej-Y<$$ z4Lt9gKwBQppi2S!i?^4xk0x-oJERBZ<8gZ+9+}|vFwJyjBv{xNl%?RftVk+VyVk}Y<@($O47Ai)`%Fjy1N>`_296#6{`7P2cGDSL` zLLmDNQs|!C{i^?{|E32Etrd!lZvkfSw_AN7x$tGb@C|=V3rUjCpqNU!Jgv(giEnf- zoH%j1?wZ#VZGZ4P-+Jg?U0zf09&x)62!GhLIY-)>=rLizc|{~S4)_`nK={{0a-=z~ zO74wNmRFlMgAdDn-}hk@kUoM~!0k3s$1&p1=QeRo9Ov&}dC*F{?XkxCa_u*wH{wmu zb$I-~cUmVZxkT`^GfEdgjt_@LsO3?3C$*aRId}8pWFyxHauTE{yUcq-zy7H0@hz9= zOB@PSP1i9wKOH}<%}2|oJT}B7b{#!PHm35s-Qg&f2clh=j>GrLz~n3HRw(tp+#OmE z0-!Lyl>>TZ7WcX>@lrCrpS>3Qp|^nrpKhBe4a#1Wvh$mN zxcg3zA#o>^L7(m;%y~Fuo2N8rYuoM-Rxki153f7Wen-6(+RBhWHHFQ8YW$9^-3UM;dTq70!D<4I47J4RAkF`-Ev2y5C^+65%laC4tQ+ zfvf)uOV`fwnMT=&to@rji;<#SzkV7++BXC;_$wZtK-y*e7ej+Sx)3-{Qr($v9p6dC zjOiW*-qWbKLug2~9pG6-7+x`9wuR+J5FgPLU^li&2*O3LXbHkhu2>0%cDTO?aHB~Y zA=~up-Wd}entYBMX4Q-^aU_Tvf#XJ{HbPYxrsc*g=^N0Dq2wk6Y%g-7`*I`5)*((f za;k3=R1UUjhA!4(SzqC0!Jrs30!O}ShTpe-$FD`jHbQ3_Hn-`ctc@+d-V`BzL^u!N z3Pr6Y)DO^eAn!!3nju9Dc-iUMLGt*r8LqLz@uLsyfYqDKQ{*st9Q4X>ZwD+*%t0Nf z!E-y9Azjf5W}knC$jYCjEX-%)5bxEV?HFM-NULCKp?wQct>LSgJOHa5rh2s@z(3cy zyYsmI&IKtQx+`gHhW_x=z@>K9lR6ET5?*tl;q*#p zFrhZ!5vEpMRaVoyu~a>9Uf;dMbIEhSF8gZCBX5P?WoujI=WXC`?_S*lnVU8_9!+^y zb*~0(f#jnxv0QBf{CK3@J68RmjW>Ny*ATqNV~HrGKX7rz_uB~avR#d51w8KAo$VX7jFozlZGq_T(4{n-M4#IFut=p(m&O>6h z>#Blv9j5`kBx%6ME6Jk-WiQV)nZwEZk<_6n(wD3=x=7h8@8aag;tpB55_ZNmSAVcc zqu!UT#OeG7J+TexNrh@E%W5=Ig)CsBIAJyavr{~t1Qc&sbPro>9Nr;a-mfKTi4GhL z#12D3FVtiw_~nGr;zNuu|5=aa&5jftq)cZ8JI?obb@4-K&sk^kSCZ8nJpo{K+9q?VlTEO zs(ad)+FkqxwFmevM@2)HCdQIsNcL0yviA`!$T-OI>mD4{_Zad~@+-8En!y8Bj}ov^ z?%$DZrQQR1cTR!zJKcGDc`zrTt+F0cYv*U#=wAk4ErXu->U;d9o_2l5DxzAiA*)%H zcz3<{dr(!Y-GJ-+S)3j9?)BxZAA{!KyGsOKfjYJ`VYE)TE);GFG z9C12!T7II79kR222fBlfqx$zB*xqVSQ zWHFl|8a?9@uh(FyK2>R%kuRO`H&A`K8af*OZFR6hwwY{`b%R(5HdIC8;_KixV+*8f zAAsS;d^aM`bX{{I*fo=@0PC`ahg1-g`gx-SA`Y)qQ0rhkrXKt3Y#z31(dGBJRLNi; zOONq7UVLOn`00MSWZ+XoReYI!=XL|lf^MhxUVe9_TalOECS2WDWZKwE)$ja2uNZ?r zsny(kE-wn7xrnsWx)CZpEtI~|G3?gudiH9^C=)s597DlP%Pj*88B$sc4;Ii)@Qph%|E$ z!#YxJrjm6D!#Y|GO)fvk9QQ_*RWaX(ZgIlO+u_4aPXiu~r)|UPVD$Y0OBVKD7DX;( zk^YG-sFLxw{I?(ySvoUgtpHSViDmMz!tA*xOXOVRG6f+_*8TEEoNV3;(Xg4T?_2%K zx%S%u%EW)3(Zsx1pb}SAZoeG11r~_>3}&e|q3wXWM+R8lSU4Jqe2wf#2w_IKk>d1MN`RjI>;lWDLnZ^ruC@ki8R;Mls&V z>~^rC6_D9SVyGe&V6@knB!3$+mF*UnDqidPRK92_-)ydRTQ#&qXWgLIz>8_$O?WX& ztIklTs-w+7&v?dAKcp?^{$_47)YOXVzD>j*Ks_^gLtGge>r#@+5%D>&O7hf2({|S) z+_G_M|GTss1?j(Tl=zT6S;z{zkKlsPO*Yx`#cdrh5b78~#`lN_jpXsy#eo7wJ^?E3 ziIxIo-#0fxw7DYuBg~%Wy|Z`&j~2_Zfk5B8e>1`IStUPM*wJu{YwK%w7v!zvO8+ho z+rY4Aw)%5Ib+OFgS9Jd1&{qZ|wW%^nYc1`%rOGBN{3e0b&le%)v!dk+F1Gg({?5ny1u&#H_Yzow>^}L& zJtrW+{J>$(hG6|LT|nJv6(bq7>9TZ~*QCMt)})Be0{ZjMme2#&iDD!g&3P-EM58`%i`j4@@ofK8{a{Pzu$*O7tHkQYrGo!=CHeOpr|{H z+qcU>Z-;S|n^K6EB0RzK!CCKE-7)W+l8_tSJ1+HnL*x@n@}12SQmrt?f`?NVUDOlr zp=3TNL?nc(O?JB;E3B=28zU){fz)7`oZ!3a_k8h{Q&{#DiOea~Ad^Oh5dQe}=tx2G zjU1!tZTMrHhd?mKExR2@;WzJT#8iwTYKo|b%d;E-^$lk1S{7zr)f%E z%>qQN9WCEM*P-inPgl90% z)}A3dX;ZGI<2x|rkp-rjCG6d44w*JYCl;U{EYDK?bx!xFG(-(_%$glyYr)+ri(l3X z`<;Y`R;$6rnxl}lMi>=k;h^A!3W52f>nXoIVsXwPdi)`bZYR)D_7`MgP3+qozb#s{ zN9qaiFx-tLE^=Ftf%(EQKB;3>l1Tb1g|SQ0ty-j+T3Ro5&T7)^rV33vG`1i4=jXn0 zPZBEgMC^rIJ-;+Y}z z>eE-OICbFtYT>ju_fOXhAXxKsD(Ln{H#_-H@cQ^v#(@O|3|`mGhpR};CoSU$2|Ec7YpCZ1G_Vpe{gkg z;yeTim~r2~+N)Rntqv+OpS`sH5M{kJTl0wEKC^6QlD`OU=xV zvn2zR!Ij~jmIBlz`6*a*TqtF~uPVb15g#WjVkRA0*~K6ki!uJgv&Ha9oPQGj06%&@ z$&9wSvgmhyjD1s0lYJw&7F$MvLIm(V+|r;-2UDK&C_wHU2t5S*Tl!l5a@e%~!L($(PfcSW%tM z^HJA6AGpVjG4Mg>cLN$@wngOUa@Md@loLKT{P@bS4@F54FSUC_t>|4*uf9?m@vDMC zC8TVY1}lV=ie4)!p@c*ZNs`on?$6i#)uja-4)G~|7veNzrA zAN8-i6q+=vlrz<&Ifl5E2}GI-hDf>7%D)Q#&Jy2IP8>tCi-^}PUTB$a`SpFP-a-yZ*rJG8jeDfwnS8wxaMR_prmOzz`7L^yAd9)2yCAPQwbd~EQHB+V|sqVlTT{u#QLW%)!a$#|-GF7QmK z%--DK=R(*BqCGBC`#n9`?`zt!m9lSTjAhDJHcOBD^tA`@oyOc0u_*Z5-IQO*t7TB@ zL(zT~zLs~u6uCGLkBA}B$wW@aF-I^**2JamMZXy4j%hoxiQJghSLHY6i7Q!hM)Q=j zhq;_DjU@;kU)9Z-x`!yuhW3xbl(^lxv}fKRgOPEu8>&Vtf2~vPId!nXAOZ6&M1V(= zPUW*o$ZU=KrfhuhWWid0Q`aLC*@v-ZuF`Z%l>hASBfD7l>H`>?phAIDYh#r+?Yne}o*#p?5L z^EFgRa*5{As;1E_nUkE0bGQb;2|b2-ie;Yp@(oy)StU2e<$9ihns|h-<6K|?vUQ!= zq*(>tmS2vtf=>;3sW{gaWcJ;=!%01-Ox{Wx)c*M-o z0XFfL4A)S~qIk``sb+mC?Hz9uFR2W0`U(c*@Ce!#4cXWV)?2&8>Rqi4J2o!~? zV}0s%ZP!1e>&uXa@lHm`b1WE1?FWZq(~=KZ(vWu)^PKTQ(l6eQEpbbL9*P~dCmgCH{ zfK!pP(x5|1e^x9B3vH#KlCE&ep>;7t&NjmCP>$d>9)7NYDq+1?Tf_4b@}eqv$f0Jb zswQDUH&7h8lrj&)RpAFbOA{meUnKT6|NK7J!=3ERH^Lsa$QxueQGc*DXu659R&F!Z ztiFc4MwB^LoHk6mx2mS9R+3R^{x44ecdwzQHsBfXl?6duU0!KQo!sYQZ3N*nvGdXf zM#q>5cf%xqz9v7K-(ujw$BVO6Zz-nmOET&xe-@>kc(xxQAOZ>rztDy(eKNsV93fPA-Y<$b!?Uk&z>Y;x&Y0>R$(9M7~OiM!zK@7E5XB6YfrzAi1C;@HxDURLr zauM^_IlBIw72uK|u$w^c;Ri#5b50gDVyA6<+!upGYmU?)L~1SF@7*OEa^+C8@2?~p zNi`{+636G=>58D-CYQIByfVBo4c5qFM@EN?>Z$F*$rdaMsmRGV3=)1hbB=0bx*#&v zelof|8DE?pNidas$!uWQ$-wbNEqE#rifkhtH(l$MbG%+w9V7ntG=q*CwD&hb+B8-Z zpKEgb<_Kqq9~=7>q${vHwg{@{{JLMlE(HUft?)+_ket7(Now%Qk$0%j`H#k=t>QP1 zer1Y4t$GW1l({3x!Z;%e6`tf%y437yeugzOC3@sBA4ea<#q?|ya3oJK`?Kd2r;+5y zq`OQQ{D|Myv>OV7Uxn%XrTXK7bCZmO1edm2J_4H}`bsm_n#>y2+RruVHOoxg@)Rs_ zL!oHcQn>_1^h!{$Pgh}izA2{(dW1Y01DEu~$()guQ__x=8pWkRMq&CBv%s*TV9sYm zbZD`WdVBA$FU?`Jp|lLFGp?q2=K2w*B9YZTVx_cB$BlWJ92Fc-1}+KkSUIP!Z{Y%> z#`DVpDP8U%$GihlWr}X(?X09|>?~}eUgxL1)PbVxP)(&o!e-E6xvxW{>+^L1QKk4H z2ZSIvu0&JQDBECZHlwRbfX$zPQAm?1g1HVn10@zJ zgOPHgMZwi*yh0I~Gb%5;*1FyM0Mf_mh~Ec%o1#XU6hMu_ok~q)tkOZ@`?$)N9)WN; zbSNEKss}O5A;w!fAs_b@J4dk*%>JWSFuT*Bpx(!f9k z`NH2o*@SYxK1cdJl{4$@!Az{(so{4>gu2Z(+TCpRZYyw!_1LvJ?kDlse%XfMHicqU z)E@G1{6j<_x{x;gNI~gV;0vN|(0kZ=KF^yz6yR>!k^j%EwGa;XtaTpyFzZ{!E>hN= zbrR2J_O;1z9pB!_LgwCeR(F3d?wds%#Ppe?36J{(^doogjr)xq8hEV1uFIzgT=jHX z0`3{`#rvOm-Ywl8R~?OQS&a&nRQpMb%B$UzX~>A%gL+9tp*w~bRevmSEKTX{kC$1O z)>zXRsH(L5?Xfnst%_FRY{#R+r^mjh`Sz;fm+9^m0X*gfl~rpgU=;u zoImF1ssrEV=Q|{Zm6WsGhA1>9Lww%8F z4?J!6F-p(Y1^5Ds#s?J21!>k;=Xs0sUf>$>+{qIp-A2Es+S8vRRyLb11avZ{@_P_$h^iCA^npdl8-6!=VX7zKC=8%f80p+%$yO3gX6eFFjyi9A_||9)j~DEDXPo7%a&#$WraH1@hDVdrVF~BofsQ2%c@gZiVYhSp*F<051A^` z!t4yU^z6g$@eAsk~>p-Nq%c*|#gQTC?W}cfI+BqjA<9&l9m6*6j>z6cA`pzu0x1 z@|e0CrdY0=mP-p02=wJebSn~7|1tuFSDRWo+g2-&PWp~Er{5y7f!=e9ZmPusLm_2Ki-hl7(P2-afSc1qN%W>mX ziSsS$jvd7>FQtd9d_qNG&pwryd0V|FO)^_3pYLPPzc(F&87Z~;x6?7%Z${6;ib4^~ zQQ}J81Ais0{Wp{>%zE@~VeNx>i)ASR>GSwP-aisr3^XOty&s>)XIMu5Hz4sprtSZr zToRTxE@n>uNTQ9AiAJoEqbT_o(R|CMh2)1y`)EZ;-Hw-{~f`V8;l7eTPm z7eUx>dCe%>RC}DiOd?hgwz?gMH$o2_+fOP!Oj!S_(0}3a(5Bfk_uU_Usf-#Z z_jHA?FgIN2@j^?!;2#BV=J6xka4YP26{(U$hLnc8nnmVc7*KcDj-kjnVa zn*Lwtk%jC3vv=g?`bQl(esGYma&dBz@UZ{*FShZ2IS~J+ZRBGA=Su(Gj{leQ!N$tT z$^Bn^^?4VJhuTnk>qnKAH`o+xT5>Kd#cU%*lLy(17XE~Z`BhcS<`@-I0PC0PdDL>; zzk;Tp@!$GP_@W5tH9w;wJTOtt1&92;_XaGjl8W7VJ2qWZEUs$}K zj)Z6kLsWpRSNjF}?`vwmg~A>~k^$LH=0{4@%xF(*47(L2xk4SIOp2{RH%F*o-nU81 zhGKAsb>`^0SmJrIc*{n{>y`VFBH#VvLU+okmp3}?za7F{{=~@~Tp&Nf#}$2%A4*3{ zC(j=bukA=qHrp!I(}=I1^>uV5&)uhhSJmB8N#*);bii3H*D->fcyBl!WXVGQZ#=t!`GUA4L-UOtzGQMz*eN z(7^lylqBC7^QwLNGEs;GnvX0wb~-oPl{|f6lSWYnip?K7tv)1Lwp@BcUsy;lOPG^8 z$1-r)P6v}*DC~7(!%=?+8rsakTHkcBPLTEFa=Pt!Ew497vX!>bvJooNu!Ei@6p~s@ zVQ-vLn5#PLa)X>DtE0XbzGJQNv4dy^{H(C7K*6M z6+0OqoXGlq*ZWlaYP+B3P4q^OOh5v7F(v+V7L1;)%CvEOn~u*=%sbh$4JMZX zyd2eqc8cZx?ARxRTIAY__pSJP-&DAGy%Vx_$1suRK$)(Y5BOSPOaxNacEinyVg`*& zn8zHYpgwzwGvyj2#bz4}=0$7un#?V!yCM8|A?K#$**$PWm!88pd+a&g&ZWBuKA~pPbA`@5hqd`s!Lpb$!*iZ< zqJGWeHern4kKD5nMaT49D&u)shgxt9VkYHmBO_?Gk#_;-N{rLsd@|zkY%=_9E#>5V zcFzPmk?1d22)|3Zt^@xiU3qI7c86{FKxXOPb@&fvsY>CDSB~plZJo292>@&SyGvW} zCe-T&#Z*i0@|3O{ISBYiIt+}{&r8I{Yq$ja6s%2+f83v&L4FzgL*G89i}rfyr*iPU z;V_uX3;X&xw3^Gt`CPCcm(C@DiL07go#+HeeSeV-4@U}qa-0pDh#l+q^c@l()K$MFQt>PGAx(xjrzC2}LR`M(Or zOFXFp@DkzO3(a}&ke56k^D3v=MKXiE|J1FGQj%r-W`%lMSGB5%3vi9=>;=4n&Zn4f z#c(jVDBeipsB8D;Pm2-$RyRdt2GBqaK>s`rG!GF5!jJyj_b$rVycNmZ6K992Gz7*uw)-bF+ z=3{-QSsT_Sv@cUKKAE~Qk^MG4y{c{wH1_t2QX5I5$`m}r)ANyC9U7h+$ZkPujD9!9 z&qxxQ%CLD(Gwd=QrltS0#bh{jAR>EMiDWVh_Xh``W{~~I{xs{Eal0m7MT}rddNZ5g z972ZH22BI|o6azSTPm!dFjKc)fFLo<-p-daq$91Ummf#61_MPzm+v`)UDTdCWvj&; z5$@0zXL2lqh&YW%mTz-eQo;L8hr&UwidcOclZkW|>S)%}I+f&V&1TCe#tz&eHr?J$NuG8-`q{TI0 z8YB-_G>jnSiP=VUSt6aiS~~uRI73A4_|}v6j(S^XCVK!${tu3(_fA%wXHQGq!*$2% z75n)`gi3Jkq!-FcRVx_a>=#kWQ5b)9u8-ARDGY7Nmp`vsj@VB<+M7$m-s~vAnZ`fZ zk>(?Qe22Zf&XCNU^71;o6hDkrQiru^NIZ7f9d41zbPK-}#{||ohiQ%%tiH8T?{2HU zAa!;0Z|5B-YtS)~OjwmDr?#*z#yAs9f0bmBh*v@eu$-Cs{0Y4S`bQp zV4i4C{$2Ond(f5Sx$Fq&$lka{L_b*12XC#DaAVH9{~$ooOdxpkHP*UC3Hc>>h1{(V z>tO|9oG%f7zPvUtJMBJi%!5io}S+a z`CKga{Jt`U&3toBO-~M|b>}#sX(24u*TbG$5_~6Htq-f)oNo8X@~W0#5@W7TNC4ye z49Z3biDuu8<^Fu7s(zY@t)N^J#NF_YtDCu1$hFznLc8f;>w|P+EH_RI&S%{=nA|Ud zD(-LK#{S4ZQ{%aod?`t8=Mz{9(kmbOv-lcdkTRK8SPqNpt{e%Ysa$kiEC ze~YLoGRU+9v(1Ii0lmwP#4jP}AMok&YnS9(Og`sIjXlIC1_?QG9boazfc5B2KjCNJ zW8biS9D_5hJ?o+dNJ)mX6gJ2{UtW+e<@xn@OU6yk!Ltx;Ph9xtd$0g$=u_>ttGW} zuh-&o%^xH)U%g7Cg4|W`JA&C+)88WoDp1Gm6Mp0+{+me^4xtr;7Q2M~edlCJ)ktPj zy7@V>V^f?DGk)%m!FIrYS6CN;LbI+G(N=Ihaw2m3lh z{47}IaLo@2yPIi7BN=o4LMDIYw9DVuO{Rp%+%j)nYl_($_F{x+bsVbH!l_J zFeBt{8tX`;CWJ03XJ%RjhHa5UN_3TrRh+QZ->8Df?SamJtQ5xqr zv|DuH?3RCy8~pthxvU6VlKiLu(v~3JiII+!9>OPr(*N|#yW=-XAQh@KBPZ%*e=OYk zDanN=VP}lx$9;c$7TIX9F-@ag!qbdGcY*y&QAd-KT~TGhvPM!7yumFM529yGHjM^} zeM@e#EVsr~9XK)iqT@jMK3jM2-67FSE3VG&xAUlf9LJnv;-7Z^`F-3AcLcXH_@2q+gXXq9(2*&*ZaLX`S%s-S?r&AeL}+of znN#I`#%+_@34`iD_EY)j)A!7+N$tz?MtvjxE4r#_?(qEhB}pJ&$^}D%8y|-ZPlfb^ ziZ87-mLC}5amS!mA+Fc`anuv&6?!|}QF+#^3`ie$7i|2XlYVHs zf`&96d8L$z=8A{79cc;WX)s=`>x{C`slWHdFpAiTxgApc4&=vD`08U`R)6W-J@T;m zwZT?BidjUfipSbI&WUq%&xqrWvb$IOC^FCQQ2X=Aw5JcWL89B|#_&En-RvVra=F`^ z)0>=v7*?vw!GC2e4hHD^=}6;?hz@n^oo=y^o2=c%pKW(0d8eWJvu>&F_?nrOY9(A* zrL(@AkSyigBSs_;=W_<(I`u}l=D+dK*yQIX@L!b2HR?V^RlB0>mh?G*vet^$gc8tq z%Uv*s)r3}@&URY+Z!vQNjMrk{DXqyOszPY?u__Bhh<;)vFN23qje3>qTi26%yUE_# zcP16j%6s<0uG5S2UPajho4A^t^XARrY3+SgZF}yHMea*RS2|n#tqXlXl`H;rLPP5N zWu#M9=us112Ees@T8#=PM-u-)}>q;)`G~c3aCQtyn=nDelOtV1-aY z6B74vKA_4C+qL^-+lk6BSsUX_j^m!Ms>E3Ff5 z_ngLKdE4@yw^lHqp4z&{FAPJO%dJn#PLE3T3T7y0YMtQ@=d`~a(|2mHY4U3&d$Z(n zP3W82GT>{El4eEzDmneoFxU4(@hdp7$kgt>E^OjCM)Likle}Gwa{rdR2^EA+>k8c~ zll8j@S{-g{xr7c{m$KpXX&m`mpi`>H5it2LjPF@FtF7Qh*+4Fai|BHX_)etb!}rD6 z6?@+Sj;;d<$bEAsr0x@98km#G&k2{G;BoPP>GH)~NwO_L(Aj=U ztb9{e5E}O-Z|&g#2~l0RUcR-&-1z=%+bbmS7J?^kSbq=k;f$fQNAnaqT8&@-V6-W! zrnvOQB)IcI0iCu|%MLqUGK^po6>L%&SdZV3ZWYD2R5O!(z_6i7X!)lkE>-PM4cQ@~ zhKe&cU;rkgTis9r4PS5vv~l*VVM<0l45DI6;!RAG{%tKN?pfPsS^R95vU3d4d@(xh zK+zBKpV*n4)jYg%0BVP;Y*j3|STgfzd8a%=?_CL*8_EJlJ~9Uc-N-Fp(kj5xJ*Xdf z%DlU>TzlHpJkei{Y7j@MtCsiM63@6~qH2%?Mwj1o}*QQ zhYoys&!NLg&+az;!rSB}c(kk!DD}4^oV7Ee}YFf`(J9-PWA# zx*;DcYtI;K8E_#ydk}TJOm%ewbowa5_j`!NL7c>M*IKmNU z@E$3II(aj!wO9@sk7oYw1Zhu`D6}@Svbva;v8-1{eNVo9=(<0fXa8v5sLoGRXd5qY zpsXe#0L?i!G~2(A&p+Eqn71;ho%+6e)W< z!eIDrkFQkDZCiIvEAbQ7d8lh4fXsm90D}dlJ*zH@QnG@9=SzG~))Nf?On>yM5hFlr zULC#Y8DikRM`e2u*t(JOw(mM0Zw$m4TGx^DkGj^0npJ`L3B7nr?$)3Rg6IMC5X+|m z>LYAMNQ%7;!ZrP z73A=sZ942ub-@RP&lYgc{%_0eHD17mo%H6wC|^>$B-*8jwvMU$Tssa=5c?KruJ1CA z!Ull9GF~Wj=XL61H0{!TCC|dkm%#6p=PK}byogO0&cFJ+_c{PU@Oy zWFa{GI%ii2PC%uMpD5!?A@;C>%YJuUV;A{B(^tYvyy*Gj=E(hGz|M5)O$W625R+zsJH2)wvvulaXfk!>~TJGBiOws$j6 zV9_`SKT2P85X=m7s>0+5*W`8o(GTHW)b_-eLGRfQS2l4 zZuZo=&F|zz-lYN92IMdc{Gl{!!WqbqMQi!0KM~sfZz>c@jKc<#11Oe93%lN!Mj)N~ z7A~WnY2}YYerH-vKhCGEL9l^5PqNg@Zx8=Rxi1fUgYHHx^Jxu3bA#P!UClDuD_ROw zZ)~?us84Z)=gehb@$s&Zh8^C10|)5Lp(feUdHJA+Mcod#%qNr*83CG;hla=b_+*U_ z2Kn%wgjyMQx@!aW<#~qJMv8~+;w4Ij>^DI(Oqn>|93QUlOZ*7%d}u?d?V(O_WPVKd z8!`g%hEN9No1ygzBPh-kHq=fR&XL>Ll&e(~^TTdzOp!7lotFaRGt-=3%S;$A&ibF2 z5?`1_+)6sxS2`Zr0hG7vOH4pRZwg@_oUcDWswv?7Hzb!ikI%g2-UG6aWafCL7Hn^U zq3rwAxTYHs-EBe@bVDPjy`4=O&k(MLIxU0RL&1Q8I(hOLNXoH1{@uUnZOyqGuXl4p zo01hGn{dFkER*b(mFPvkCce}$xI3faXNuGrVn39|LS%CYiVR4>%o1aHde>Q`lq2V6l^qh=cs+P#jR8#t0$IASY?B{M1c56<^h#%OG+GpCw*7&1mil2Q} zEHyqC;LK=vazt9qdT`>EeBVV!{KR^D)-+8&QEd;(rt(+GHt41T=dB$z*BNia}TINTl93nObKw zE(%RFnoD&zsX9$gbI=AiuTt3$@Xuc@cJf=L`cd_38Rb;axIe6r-o|`2Ijl?3~C1GSA zV$)u4r$I%Y)7?0@-oQ&C>09qifkV&rG?(0omoC7~B)ibLULC+@{b({{L8wPV;??}2c+ntEI5a(SODF=GFK+O6yk{PTakxp= zUFZY-UTja_a-NBTQ}3qPwV?b1Hqw0zdQ(|adGFb={R0Gk`xImwT-XAca=+_C<@PlJ z&AoGOMQzs4a#?i`tJU8`4!!v=62g4DLp15mbA z9-+J02Xj0>xcnTNSN4Yc5YwLV1*8Tb^Mkw3DR03*;xDz&@`+w2T?3@?t(H`;SXmU|uO(Z=EayDG!e~p1V(Pn- zXrJGZk9Wp$mRoWTJ^8TTm?Vj3dbVtflmME7u3@^OxJM`M$jEvX5c{4o`ab_ciqV>7 zZy@vX4AsrImnVd*P&-jS3ax$F+0xr((aWTXdvCJhs*fbLWAl6SDSj#Kw#F36tjMLJ z{I)M$L{>0JsrUvr@_^`jDqnB;9*w>g5|!b#Bb-d0$yLi|`xCa5ZJ13;(ifbEeDrIaJtdBK7wgI%3jfZ46;Hr@4`Wo%B}Up@KrGm2^VZ{4eek6(rAU@*!2D;Uy(r^Lxo%fDpd)*bc5 z_PmDOwFst=kwc{T$lW_`63%}MJ}8tlE4?Fk8_f|*Se4D2%q>Sqm_0JxpT1lgE-=PS zF(#c_Q;XxhIPwf^Z%7IlaN6MSFrV&`<2QQllg{Dg$rPqjXAT$E;M!UX`+A4Z<>c9A z%5Tmd9+3hx*Jg%h51*#knucEaxeP~>s`1V|PnKD1E=HE_6KrC=)_mP7b9mUqOngOp z-0ni!Tjr%imb1R8DTnuObzeZ+ZI(K4t%)@bQO94WV+nl zv%cKThIoI!%zGF2=#u1GQdb-|uIcSY`EjwLvsInLT4|sqMBN~9INA(-BEa9y`S*zQ z_)9P%(o5A-#ZCXyH_1-^`YYArub-)x}fH3g+^D$7B;z?9Dx7b+Ib&{Db%y zM6-GS-Q_9v?y#RllFH)*oj>kI>I40bFCy}HiRZ+SHaI^ngtZo4|8^JMCF3?P@+xe`;+`9^jp?E)L7TczpsssINiW=ob!r{$yME>Y9))q&<1&* zq!u?2M=Ph&^U7HLs8^ydsU%aApvfe@;ils&X69l{*XM#1!WZm{q9vTB->)q=I?K78=)yvh*+}x&d0ww!WUn=3HZqBmA%_WHQJMJ(!$i3{ zA{)6Q$gVT*$YhV0zh(ciIs>d9?MN=|bpoauzDNK^O8A6P9*0M5Z#WA*l;@eKTytOH z=A=y&jV6odP~O-V$ap4lBsHBmPCz8swHn6&lX9GnZ%&0>s$Q$vG5%|DoRS|H*vesj z+R)MbxCqR=4sU;bK0?o8Ewes8$d=0Vi4|O*Bce`Bm z-7Y>#U6pLlSO0^#w~C7E`SyLI5P}5=?(Q`1lHg8ohu{$0rIFylgG+FCcj-?J^QKFs9M!MX3eU(x@*?@eAjBkY_L{U2T=rUwSpkNy}zxceCOw-rv!QPmnADwfy~=@S=lT6T zNga3Jxgi8DkF)L!V?8%_1E1HL*alcLMLurEwer68<~xC21|Z?s+>>9Vr_T4VNv<*F zNW?cGURbKMu09ym%Vxb$&r1U}eu~`y2`?aZovYh)KwfE=_siKz!KlndFSCa65=!7*aNJMK(q@0-H7f@--z5}y#I=Q;3+{mRg&Sk(?Q&5RJ@_B??kq#1Y=8yZax}t zZBw)|pv!eEsWVRH$f=sGJ2{pR;9A+eP!UjN?O)^mZ6qOP{FasPg_V}jv_7ATgu-A; z+~+3;s#B8*m$qNzevV_hEul-0sw&Y;DRMd^B`@-a_Z`uzKgzX!q6cpfp2cx5D7gC> z^o&OwdFm(O-Jh}`kz zj^7~2f&)3~@7ul6T6$h5-BZ{bE5Hm0jyP6(qXia9Iu0)CCV7_kKsMQ(FSKyxT)vl; zgqGctPRH&o6h%x3?=|QN(cowF-E6E|ygwe5m~}2L-L|Vk7Sm}+Z%kfDyiWMM7K?sS z)h)AP4(r?AI5zg0PlA360b_Ly@M)Osni7BKnVI}Dz@t?5rfu!7k8y%95|MH;yW4)Y z#=Z58@t208#LwXQ#{?LUhwIRfEx-Z7qRM=e>!Yq~QJG+`AO5ue@(Ex1(Kj2Du1|#F z)Q*&Ewj$Y__dV(@V384AY!~IjO?q zbsPI)qptPB_%ptn+R!qHz?8dCHYLJmhv*WruZMAn+WIJZ5mkm`1O5!}DFo233l27V z{PI?A5Zi|PTwN8h03&wv#YIX6u`7NG?@rzDfz6=)^xsCs@dhZ?Lk~1wS*80W6hnbg zEiM?qAQ;mQ)!>a3jjrMLW9*+LraYa@E1hPh_z%3b>Fpox>USozzz~C>cSoMx(5uTP zxMn%m26r-rzyM^F>tTA=|CV%L5P5DXXQ-rJ9b^9_QAed3Hu2(guEm?>v-kOD$KA8 zqAyx8W}-HGI74{y&@$5=z|C4x`1?y5App?rSl`o z*NO-9-9k-dUXu_;VMIrm)R?WnIgx5ySaUccIN&dkXL*9|xn0`;cw|R|S%uyQ`>@f5 zhuM{q9KuG*&{y z`GBJP1H(@|M|h;;Z3e`89xyO&;Vbei0^LSP8daC#az70Fgn2->jexj}V-{Dt1UNEc zD)N8#H~w=@ZLiYn??=;PRto-!M;&}|v1ZHeD{Cxj^Wo{WZ^^_HLG!JVWRT6(fR95_ z%&Y-8s5V^eSKY+~L=^_H`z_+)mD?2^y=klj;yYOX)?L$ackG?nF1*k$y_Je-5!>(N zPq6LC)N?$zzb5)2mj!cOrg8VhuBg~9sgxH3nyEvq#29VYOSnj+Pr}Nr7&l01Cob^p zF1Qh)PPq0n_>$YQgM=*jYM3U*zh>ul0`iN1DPr34s?#spIPJ>Vnz`NQgC|cUU$^$( zG_O)-;`6FohhfX=rmJpnFajScD2-wU<-gUAUlZ@!wESZHkt}b7Ag2Rz zOyGaLwz}x`Vi{AL{QI@b_iwN;@AHZAcfY)SQ#%EHdE9++iK&kdrI)h{vp6i{@u&wI zp4bW=iK7v#w)5&L(PH{*6;D^}TGtPU_aykn7~9(~Uidl45&_(P;o;SUz1{(CMxybOYlRLe;7dJys=d^|O=AZ=EC1y2Z z++_N!7YgJ&xE2W5fUCRU-tncs9U^IbFl&7-rn)vCo(|K+4*?F|wXhzOK8B zA%k9>?_F7df>$uRK+2zeItQLd{>YHq)G)Cj&kt~f@Mti4_f!9G`m%5iA~RTmsKa&l zQ&g=s`QInWbP(Z4R)0W8r(U16nD1bA?rk<{BMzc&o_j;#NHE)$48A%-u!m4OM=vp4 zjbH7$6~JUC@o`ojA7Sz#LpA&WOX*_*P3aYCD#w$GB9ueTiM;p z0_hL;1&FIY+E~)^1AyykdKEclb-et*Dl=M^djZ?*y^}h%OvB4$b$cb}qe?ZalIzm# zQtHBqqkZ2zFp=FV8e+A^RmV#1OTeF=44Su>DGkHEiKjSQGOA;y=CSIw=~j(LH|le2 z%rZ$D{J3+BW4yLM?nGmB;8_>yUBWT~?)B4Un&g7NX6&qx&C0+$2RiG5%T>45QMX2* zYxK%VND9A@X2w-i({g78cp09+8zALf;}R!9e3Z$$m*tB%YxbuX8!VfE>{fiV3itK| zE$N!AttxN9EsuadYacB?Gp+2LXt#p_6UYeVGhPInz^zT&@S<%YS;uK(8K1YKD^pD8 z%e|j9*xnfK4-|}<7a64-3Hj|4~Ou=Qk z2j<3P7M{>Xeax@OnfsMeu@Z~Y~7Kk>B4bN2uPWCou42KG~)%?S>7^>ww+zyq_XP3IQHzG6jIG1 z`uVn}b)WethI7A0sH1TP95A&m_a*#~l8iSvTXkPJ`Bx_I# znOR#gxP=|*j2fi*boQMSYG9k~RCGs(V7cN#QROPD=(;z2$+IvWZnp=aox1Lzq*aa~ zr8F4yU4Mx?rrZJWlSJmXiu4oQlh#Tk03BXS7*Pj83;eG|B-DUc6+r6xtbljlu7ULS2oZx;WxUqH@olt)qnhKmc*ZBj|H#{Wx{PDJ_v^z*1``y% z6mqc$nZ5gXd|;3(cwJsip8a(+6Me?)Xz4UI5RUBQM`AzWHw}*K?9(S-z(cU26mStC zSSG2@Uq6sm=Z(I$F31M%eepX<3HQUj$L%I&xTWJ?oyOnmUpmw%i?cbluf;oanOh)_ ze;q%PDR>J(w?lPeAh&lnPZa38CO8RxyX_O9}ey)<9toiuI^ z=s&^bs&#}TA}minkD51mV>EUdQTsfx3s$>V|JAJlJ5HVx6NSV5`y4@8dG5!^9P6X( z>*Lm^{$>(Nu65U*f{pfY=FPG%^4`E}y%5u(QVFB}Yuk9G(ebU`xnqCsyY@Ek6U{H^ zaq(fJ&{c&;aMYU#tY;k^stb360FqiE-kWL9IGr1ukR0mHmC+d+< z8cKJ3ZujS7$qgX?PR6H-WfO$34+77G+@0lvmqYVMX4aQE*Y}UUfu@t!f2<&p90`4` z3V;xQMF$~p@6YBPL3c&N=UZh)*(3-AbIdI6)0^5j{2l|Ir0aL^4xQ`uzE{WWz8n+l zgKEs?pywQV{+~X~+Hkp8O}f>*kwnAJbIGR9Q=|9pgll}goaV)kAnlW7)4%=wplge! zf)U9Q`2gE!C^7W5Z-lR3Pmm`}#v&Gu>RJjGL#k^&c8s^tX>cdHGUQYWfag!HGb|(F zpx7Fv=)#y>XWw*x1sUSq`|F+%TdsMI9Dyy6RyvHg{pRgr+f!gobPgbA%iB-;Jat|Q zH3>2;T3PWbB8)>)Xlmw*AsSFuvgP!AOhjJ|PFjejuPyyFqytc$mC!5C zb9#RPQY#L)wR5LhArzFAJ(XTFxon=9pM6_da1jE;ip^aQps&oaCl!BS0<23_=KNVm zK&DYCa7YxsHz=j;d{vgy@ThU=o9ra!-S^vNDu5C}>Xq(U*dYuN-~~i=Cd6zLZ$y zH8EcGP_KE?ow5#x*yI{EgTh^sAJ@)o9E=|nR-5nLmjiZv0=D4#)mQP2vrpJ2moWcN zyspcI%{~5K4SYY>qO5;TZpGsbJEeh-3-b2 zg^L|O0B-vVhn|lrZ&%ka&T&i-&r&ZtU065R^H(rlMXIiW0`!%muqx1bYya_C7f0DK z`}}=}%^VVcU+9%YcGHJdp*|(P({yo|hvynARHZA(;H@C!?r)CF5wG!1Q;ff!!dLWp z`7Tezg94s~;7fknR>`(~Tl}k0+P%qdXML<d+ckDOS;>y}&1@I!mSu2O6M~bIZm#2lca_#{$8*^z|`L@9x;xQ?=M~*TuoR z+ekhHi6!mZY%uXXMDDzZo>y9{>#KTVKkjw2>BmBzNrYv_okPaW?l4V${Qe@UF^@4W z>*pK=yD=uP1j_yxHsn5$0}6gb=hrer1oPsluNAlyYEmqc4YHH8f7`+J?d@Oa=#N*q zDv+$qYP6xgae95@NM!N%fJKC2n-3cAKk>Hxb3ojdm>hE47};sAq}f^>69r~2Gwn$f zZJq59oYlM^m_%>lhGwTGht^gis$%vy4c>F2Jh->|^tWK4a5PC{V*N?+_V;n};Do{o zq$X?!hcO8kJ88TZyT9yr3hoe-6|NP?6uoL-;=pnFR}&-# zA*yLI6xgmqYOrIq^Ks~fn~(&Zd{7Cd>*oNl8>hc-*zDQ^X%N~?0opOv!H@A7)iCkq z_v-s-zl(vcKHrfsrDG+Zh+WZd-MKAzi&Ye-nm-;h*p#qYRmTZ_C)#%gHEtVB#JXAu zJsA$D7m-@MOst+$_&{8^Zhsu&Uz~pKc4efdAhhAPt``uM8wQms6tlQII#vZ3Yjjzx zSxt>w;g(;oT`Qdd&zNrbxNg!Q7=^%f@`0PRTV;-Qat5vdPs$jswQt>U$Hv0NV8wK$ zu6wBG)-P8TkbBWRUs0#$!*$06T!ih)??pAA=5*Gr3AsCl)^WW0r~divoDVIrA-H>1 z*7q9e4dZ9WcItdRK2Y=e!Fic1`D~HB)mG8JD#bZw*5mIGkAgO4ZI0mGN0D+%I7E!{ z`W_OhRqR?^^yDF3>_#PQ{8Yu*UDXM}Pjlqa0cKjc=N!lgO(5n1xGG|7?oC&b_hlB$ z4I^qg7Q6|0iZQ zaZ1nE;#eD(^(+Fv^~`JBB@rd4xW!>#k*0vUH(;-b*ve9 z)i4|Fq(62GJa@gmYZOUdzvh^)<)0&XfbyP=f2g|s+43s~d}dOEdn|`&P~AhQ z`KoLhT)+3h`wMxH$G+e}at1l%_W{LZbY-(+P4BP?=WxR#1tx01>saE%&8k^25pSaGZ&vr65N8NR$;U3?>6fv-SAO)$A|g(kg13_bRBJA zdi`s?*6^o+#^T}40XCReK_%OQFsacfp|%cAMogwkZ_;A)l^fGM5;Za!%Z0Ar{6j&&V_G z?X`5@U^(0HSa+(mKrix3^v`%d*D`!re;hj@yW@F0xgJ*JC_IaHJKl6C&!L~((PJ5- zFJEx!nF)2&j<=*}g$Ql#?`-TdRHv4j_uanUJ3{hZOrDDJO;4JPzIR~+e;2Se-MgT% zsa7tjDw$3NPSgP>YNqpqd?~6TQuXoR&2z&A0ghDx+8&s`pK?@W#v40jV(APl*JL_m z`mn|ttm>te`DhZWm$lw!F*1{4m@hARW3f%-tRW zE2|H7_1wlx?dkoR1^K$5Vt3_P3q2ZOv~LCA@4sPBBz+m7ZtzPLWi_kE`to%PjG-ipZ7hcu;Sk zW$aHitDES2d+>8~gp+i>GurY|SIPc1hOVT+vF*ZYCWs4kZ8pe{_4MEBTYbvO>P)-8U$PsQey3 zip21!=S_#)#WwK@h<$MQCUSb>z%DjTJUR|Vx|-hpaOIlMC&tzJtZ;jM7xEvQE z!YB50zE{%4Cr15K&-0#rOs}N@V+JM2hex;_~l+nTY8Q* zElT@u1{DvR4-ic4ZYYwC$v_~;NKY~QrOapKnh_R2WsUKH;qb)}(XO*|u+KF@#pi;n z(+AGi>X`H_@_@cXaD8y>>^{;`c1v8heWX34XLoZie9HAL=gWOepB0Vqh?T~?oTFci zpTO|SEr*4%)mjWdV_Y8OmvA%L`s5&2yUe-f)1_>&n*3pJi>cOf9TqGH;P>WwM zF^_ZJ+pIpOvD9oB^rf#Uk_gYZb6Uc1)gvpbD8?4%(`t8+DcG?Z`yv(M`K|ITEarZ6+%jJ) za$!!(cR&edn>==Z@-(>jpdSu#4r=IKf}GTm<*^Q8ck{_l1$e$T#MFlfVsbW*Qpkbt z9V{l}Cifs8?`DPj6rV`{(px)@%x50kX^ZibV?U-hTUm>#-^aro*Gt2*sswZV@-Dan z)aC`m4s7Yh-7akduDGVg3c<%0!Au827EAq{#bOE7!=zEhzjlQSpq^Orwaf|^j9@dz z`3rUg@ejut-1Yjg#oVDR&4mx(Ytrq+Ug6Ar-=K#-@P);)?K#1v9H~d<+R$|YJ`Q7) z=VbDN{lfF1!TZ6&b-EivjR=Avhu;}3f{#R|L1&MKN1ZyO_Xg0&Mylj~L-KP~u&++;-DH7asPj!|2 zVO#zRNUq>M2@a>N1iOPq?^li4p8uPdJght@w+}pElqPkKcC91db*?v6&+SgpBUm3W zN5%ys&z~~N@>bOMdL(O456Hj)ach3z7eP z=I1gT*U9PE-NHZei|TY;(1!XuS6^icre5d-VcsFZuVb(H9yYm~mb?Rjr0$K` zeEUYp!8KFpPojZW!}tpqitwr@JT>w;1%lOsy_N86{~^&YmVOZ5ig8?+#$tKP6`gDR z-}DOU-THpo#7DxLueq+It{+s1b!`pLx!?nzigL6^-R35%K_?37(K1$MGAwT0&z=eH zbk)tc?ZQR~WeEe6hDoj&9{USi&_}`=Hr@b)36;b01no4D;fgi^ve_u4z7LQ0=s>a@VYIq`xE5%!Ee>T=~rB@8(ycrseaG$UKrO4P)li|JAn}vT=ob0 zwdHHgM(u{lpUVx{hVviYKg>aVZ}5@G(b-T%NguS&V>a~v@cdB;GDZSpo?$NLePLjo zVVdEXp`8(#VV>c87Ukj8AQKekrgbwo=qzMTs8-NRa|2Ep%6ZbSrb53=naPdQA(#)F z3R@D^RNypr;fXyqn_8Pjn|Yh@o0sPtn>L%eo6OzM*d)w#5)iAeCv5Hf^=5caTzG1@ z@p~|d5w1UdC*^~)ml^ChX^k;)dQGuHu|mEg_6=1JH8u5ad;v8MwK}!7RCUA(fi96Q zo;Pnj=cWA4=s?Cmf4n^*BcTnU64!gqFe{Q@9Q0-?4QGp1mg8fCql_bt8RVm4oB=M6 zZ|_jw6%zou=7Kd|F z8N7kKe|Ycgks8rkPA5FX7cc9zhntfGPJgiy))o2=7Si?u1S|&I6!uu!3YWA=OWDfL zj8PkT&KF5()APrVc8YOpW-16O{V4 z25s$QCl8DOerez&<>Y>D2;Gg^nE#U_t~Ed|k8TmN6e1Yj9_o`1KuA!FKlhS57#@^M zr8i7(q&Q3|6q-y0CXy-oaNF2SUCA*gYSDHDgGy_~GO=SrmP77C!^36r0Q4a$7{;+6 z16$~>GIaOxTw+tETP5r%0@N7eXw(%`g$5c7e9=U+{=*sB)=y<|u&ND}?BoR@dR_xBl9R+_af4RrWJM}97 z)6bxL)FSqZ+EOR;O)nMWSU{#W%wV&;I}R1hvZC4AA{%LmJIbb~;}$U4ZpuieAza?t z2^-i7?H!IZo`Fvp0DMx8QKF=VI0z~@HQYs9!6tph#dla= z;SAtiG{c0RM3>quZQPd!z)>hkW@9omf~(tD5MMy0mZXr;;KQV@b)A9a$Elbz?FfYGsx4(RZ9>cI4-0j zG@NBIraq=-^>cdo@MuSCXS${ssML~iGdo%lRe^dSq3rJAuH`;`BZ|LU>x*3i zEHrmPLtV+q$v`Gg&9llC)wZh3)S<|ss$_j8KJ71^261RF6!s-|@{T z_)UNutoJDj5e4rp*U@p7=+(r%VrHvJ({`5YU5OOH#n(2_aQ@tD%sf_;X6WqDyNKJs zG*-@Ew)HpC23JNKT4k+0Sx%)l*_oe9RYYYFuYpx5yVCA{tfK#%Bsra{X$Cc47gdcr z_^%W*CfR&^y`>+ z-eC>g_U)?X*R9=J*In0W&C|osICFsi!~U_q&wh!f80+1|7Ww#t^Guvb07JGePlLI@Stv_8`POf3-#cg5e&n7TlGqt{HMh$;3}k?;b!H3nxWX;KiCuy~ z&6{6~j^^NOg#67=rS>X<=@IYQv=0 zNE|^~KZKChBJ*>+O)z|@if?0kZd@Ydj=gqp7v#>i(v#zO5I=JXf2 zZ#DoE+n9@hZiE0?-SFY*Q*j53T*C6j_E7-I;%gr)%ab5K69N*eE}cRvF!9<3O~#-s!l?0LW9pzrL$mermbU@=9VaShD(*8{xl3qY@Mfg_*BbV!sXJ9k(MEV9jTVVD8U_+rB$_nH0 zRcXb%q|Y@B45Xs$V>R!%!p~Qt^gVy!_nyNtkNf$SzdE6MZLY-P^x~u_IdO$&A4<-C z8_<{$iu1irr79DW%zD{JF*8s5TPI{jmn$U&(*W#7iLT)~$AsohTV#W?Nm z9Tz6bfey?v_01d*#+w@AA=}43(n{MXvv!>Doh?MyaTEDn0gg!@;DO!5jq2fD?HKk% z)OliUX6+fW>j(b`c0nxvMcS8!@4&C3&AJJ8gxFAIsODd7r?S7it5bO8LHEFDM3hI*oH(Fd2d;N!B{__yUwTDP=b~b3Z9-+Yk?07uN1h$y#89;3-k6 z5GS0TBn*JGiNmeSATp(g$Yw zV^N~gwU`kH9wCp?1TB(%g59lB%~*ZKk_)Ju6JW_cO6pNsEaLdj`&=2=mMokTCDE_C zGaEP;hB*nsDYg^-5SUNIQMAVgR$GivM%xd5&E#4z4yxR=+;I$Jdt;-|yPfD)YzwZR z>_1toRvI$Nbe$O8(4x8CsfC2p#-vFOmY`F*iRSR`G}MH)ro^4?72s3e6#MpZ2h`hg zT@r3TpuQuIGY5>$H}9BE2E12!kB4JvfNHJ((lsl@K|SfCnm_iof$UxAU9}-Y;b`c| zTe$-_Ns*MaKFDD8qtNA#SZ`ipg z&6VoF0zH>Q=mBsuJfto;w4G&-lF*RpuRp#6edY1W+i0sxw;iKWYTx=}t&es6blWN) zIkKg!a4fSva#Tw}Mba-L4_vX1`Ml{9o>QV?gGs6(q90G4?0@=8mZl*n6@ih9>rr)F zCO7M*ORAOgw60ns1Ro3LO^%o>PonB>O^%q0AGV6BSsA2NqH;XvT{%1&(nLchd$$}7 zx=-SI*V{r`TH{!xnv~4f3My-?v2Ag?qVt1Qlc+-dWv;#>*h59QBeNn+anvWf817}0 z+T^78^0y7u+QfXK6_^l_Wcvw`DjzH!3KqIF+(d?aWpo|O4!od9sWXm6*N(C`ZP>o)8qkuEJ0=r|NySV9(ap~8aFXqC#yGNy$)a*< zF<3!&xCkm{SYpDbuqZZ7SfDoN#K=oeN^C4|tdQQ~)>r&dNKa1!_7;Jnk*jpr*rGH+ zOPW)p1X2(@E**W53T=_n>7Gitlk1;ru@GJIpio?2o+@r$xOa1Fif&^rICHa8CEQKv zid+@f9wxZrgfeuCSRBX@-Y^D=YqU-`p5Yk~3 zb?OSSo~JPEhf7*mAy%IWN;_!Bp!;AtL4XY_>lPc|etLA-o7>a}GN9 z27~$qz_DeA`L)3LD6|r^w!Rcv>3_D~KVyD<3#!7<9kC#F)hHPzIq;K^osS}Q`esNM}kE$z6pbL2(HJn;&}WTo{GS#=lLp8Z#zPc+svDbDKH!_ z^V&m@?|Ucy`MS$+JSG8+C&{JB*PId3xCi77%ssL|gZD7Uw)W^$v;-j4rNvjjU1b>% zZG z_v6F_h?}H~^;JGG-fah*Wi`Y80$$4flthu2P2ledqm@#lMGwY_N3)?)qE*kx8(XxS zZ%gRkGx&|9_jhm7Ab;xBwiZ>AVc&w}#whb)GL6>f=VQxZP%fJ0Wl z!WwAeBj+ciAqqkk3G-64#7$0d=^+zNdFf#S#?(4a<-Xxfmk<(0oE-)Mz1a8&PPy3R zK~AaSA!nD)Oo|1jHu>rl+udgpA>LruWv(y<<{-AW4q1G_!1iUDFr~_ndc8)t&EIfO&`3TUU#2oo0VPj#ENe0i?_J6Fkh4PDn<-KYYkouuIg&Ov|ue9IZeI zC>b#B0f*Rj-=8+9aqgm}o2ewp>`JvM77rV%JS4I%=5*>|I(`sG}@e+ND>SttvWZOx;@y3eVJU4{m2uEOu$j%M?+I=>2N4;@%UY}2!~RK872!JNlsQz*b!iltz8R1;N7<5W$#iwt9J zSjI5KV<5U|tZDdWOXW`$$OK5pt%<&(ije9tN(xaRz1aH9M(1%(MM?Ds8;~GLGqVy! zgCjTON~B?@v)6mF37>mKOJVV|uZy4+GVQ_l-r<9fkNMsUC81c=~^oDzqy)zNxTj;V^2+ zSMr!f+&g|Bx#~`NV-bu$7eL?Djk-KFMYIv>hh5Kh)9pXlq~){u{Y{m<5x=S(tr5T6 zQaPg$KTFT4A|R8B^+cxADJYL3ayJZZ_{f*I3eQMn; zQ^+(%Y!4UjF%5s;5FM8I_q5g1#jMrE!$DyWdvdB)Ns`tJ;tw)O5@)g`oA^%OQBz5< zuwO;IHlA%HC!oQ|opcK@R5wJ-&d+uoJB5m7KZ&19HCM!LSg~*C)tVaQ)54cbU=x@a zsfT!ot=jxy+U;`^L&oP5rr9a->q5TqdkSEI2a;8AE{Rd$IGgX$Ic>Y@N4e{Z2@|5UT(NyKw_KPrfZvWz6*FLVH%PEb=Qx38~%@!Opw|@BIh(ff(vhWd5 z_bc@qexrt9zRw2abY7l{*e}+?nlNpZanKaS2Iy%tsD5sU<^@jJ1}Tyv_S(2(gYk^# ztdohn;?#?qW9plgkfix4+`A5cTrLq*R`F^*XVT0h1+Qt_0edrt`8Mfizd3th3w^_U zpXx(N7d~Ld*wD#Q1{;CQieDBet(G>=@j>e!`7AuF*os z&2Den;!;UX)E>R5_`~;-jM4>@OI^peB?I!eiqbBn_t3H`#YY_w*=gebVOR5qW)zMP zXBSJ=-Nv>=H_^JL1Qvq~_gU4C>_0<92Z^mqF~~aOIH-`?FxHYY6zXi7NjzN_#|>L^ zKTX4Oqlb*(NR3`f@5sC9yQ`bwFu=s!`>CS{IV(o@~643?p9)e4T# zZI*>$e#qt`dhKaZVyUKCx%m*0^9ZS2U4f6+UtB&W+oxDR&F%^%J?P?u^a@ z?{nO1ks*5L^3xfbiGhO&A;B3xq3^eemx-5ryZyU0LvhZNXL8v%SlLchc&O63_`k7- zcnR}*hNy{ZNE!cy($u|>bd}@5iTa!`T!EV8@?LF7J%Z#< zl(l}EjwAsJxk?D5JUiYPnPr1J(=Ja+6C76lN3COvk@F6-=;mow_OO7fMN^xfRN#?} zvmrqWja-z>Op;tMtdggZh1KPzDE!BXoL09AYrK|Fd9}lqAF7v1(?cM=}eI_1@y6HH@Ds9s0evf;S<~Lq7uf^;+GSU{OSl#enUfo3*|#?kAZAUem{?3 zXb;{tADb)+(`@sGQ^NCxKPc_(XZ({Dp-I1`@$Lr!(VHJ%QXemW^&y4Za^o#dM3RD$O>V@t+9wPH7-wfBbcHCFA9+&$toNbG^-UYOL@i0pfwm_kkjomq`1cl$e~VuKPdN9#_;}*rVHe=}FAu2XuHzxLwpcn?fU(rtrvO~hWp*?T z&)MqrX}bed>vkd}pWF%vxpdsgkJ_T2AVoZysAM1eyASmkafRpoQUQS zvE$RdBP^AO>ZvE-Ar6da5qlJS*j8ay?0CG)!SjZ_4tP8UBRHNe$)BzqpP-KO&*jqu z!zR;p9NEzWY77)2%>7tYBya=PH>XS*c;Y|0euQ>QUIDl#10MU=9}gjtdXKPv@Du9_ zDk1lm($lzrhZ-=#cpmra3DGb1xPJQ-R`F<{O(ry&{6?wOUp&gYWKeUeF^9?KkPe3b>^DlqZKlGq88nPyd2w58B5^qyM}J(}-YB7qb^Ud! zWoc=f#EZ?0b>5jGw$IB|eQ?XD2=gD&d^6hWzm*)^5nAw+WYEB;-MOWLYU6br8Q9!5 za}bL&q!L}`Gfyv_p)fVjhCQk|?9?yU$R>A{^H!-^Z2M?(IY;NmzDt(pHYCCFVdsir zX37@M*k$gi%lZ(#kRha8P~(;qyvtDSdZHPV_*XL~SSQxm3!Bm~_u@`2aE=xhub*~c zOJ0TxVOEN${zkm-{%9pP9q{dAPAbd|+wU~^tU*XSudVNnkvSW6^sVX%39$U4#JH*b zUj2||=?*1@!CLQ5Zu8+y6f8*{{DZEQis||x@z;IwRHVNT+Te1wZ)nb!aljY(Qs2p& zcSM8A7~J?5C#)jVv+r259=PQsmL+g}j9ruuOQy`%;`c}nDH+aq_g8-GxwHu2Z?$=< zz|2>&6D6xUrxXtbwb?|~1V?hH1X5O@(vaVG@#SsO2mvu-@4ek2u_DP^E;j!0_|@?= zGB7#z+NzL>-lPiXVZKUI&l~lT-)*mul@?<3G1FU2jQCFJBQ|n+HmYZ}_&rz9I2Wl8 z^VUr0JLCVy1bn$_(!>Ee_1Tjvv%1i+0O@~mkdPqc?2>;wMzvV3Nh0* zry}jI$}#kSBcuAj9^8sieJ$!Mnj&?0<*$pHCw8>#n>(Y79XoP&+p0qLqQ5e*pA72T zL5Dpu3suM1vMOgZ4zbRKBG_LeW6Vex7Z-#xnx$JnA9>@7NlmU2kL+dKj+sIfT~Ubc z%6@9Xr;Dpht`u)3dF;`DSta}|yY(2MJtb-fm6e>uPMgTgY-G=WC!K5n;srwkrjS-e zLOA`3t`v*9V_eBs4S~^xX?qoI+Y{-$f8vCr-X*qNZ=6!vPY;0!Ar?g_$^MorpkwpdJ?Dw!0Q9VaK`&!#f-IgTM1v6Sl3det@PtMDLGz zM(sjhCVb~@QF+>*ht$%lTyM|yg>Al2opgJ!=z|b;7AQpa5tAB-8DLVccnfpio;-bX z{gG_81|xJWni(y%xcfb>nzDlnE$ZiqQr1G^(miZEg+p@@c_NFdNdtk7b|RL#ikt+Q zvZ>no{E3*tC1ox62ks&Byb5P@O3gxnSIi>rw~@2&P#b*aOQvSu(MRysTNnEdWdUB7 zqf>eoas;W9gIG#&J<2PDXQgk3+=pJ>z0pWJXlj$ETvjP-C0C}6v0O|o)}ecwZP#?^ z>}l7r4_CW0>9_G{1A|n2cC{P1iVP z@z<8rw!ajrqt^2J=6-aEvB$pBr@k>@q{y8%sQZlcR|UNUrW!_Y$WVY3z$WO4jk=R)vEH0xyhAQxS`0{!#I#FCM^+EuzJveal;d zkuqL3^pm?$C`dZy_1m;<0bVi3lL65xMLorY?SU6VI70P~m$syIDagvtbJ~zNAlYbA zc8uH8;TG-etKOuz`SL#UW>gP=_RoJ zv)p;smB5B<*Y<5EFOf~G`J0CF`<*1TL|T~s57apqD4R){n?aCtx*L03{fAWYK&DuL z0LwIb#}dC|xq)D`nKv70lZFDY!1@I@PTvMRaldk;Ga-z@uab9Sre9b-a%k zXpL!n7^}55Rf_=5i0K>#Q4cxscDu@1V;@jYAja@62J%u%o0h3ns~SF?2JzYSlb#H5 zKhrBI5`~PtWlncrMiA%_hjo?306`yz?LmZV^77}kaS;YZii7`n{W?F z*|spMP5md&tm3$~YA=z!cmeHBva5m2sI}Z2JPEgcO7tR?20A@s&o}3cZObEHnr-G5 zhJCsP05OedW^FwR7{j#F`m=G)`R`E!b6lbeDtR`t7i3#Xf;fhM@>``YqkneoJLG#a z`pC5P!Vbd$3;1hVNR`y(hw8^nqAYQw>l1VvrKKsLHyCLk6yOPJZ8YK|<< zPs<1RN-W5#?5Rgk9#k0YK2t1Zb@keArV(xQO8%2Zr1Oq<{G)NBB4mlOp=eEUrsT>PCsC&**k`xP%kfvuu5xGW@Ay%%BSdI^ z6=8g)uCsebP`vle)~>;zSEOp_!ux2(9Ut8iB*Et=pT-u8ZZS*!< z;Qk#F)ccv;EE^Lljg?}}oQeylYnBG{ZGvPr^_wq5=<|`CS|q#d0RuySyCCcQitJ$0 z=TQ5>5}X6%CZ6%_C3BV45EnPgnZ47n$Ek!d-A^`AR37%pXsfH$KFRzDahfCZf6U^h z%0IonlSqh{YIRtDOgGA*E~+qae96b<3E{Z1u1dZCN|8QFYSJIo%EwD7!3vFaV=3j} zYdR*xY&G!!yvb)=pM>7F(%IJq*1nJL=Hh7 z2-I>WgISbg_+!Z8#ovYPNFpbjN9-~gicmAH97lL*qQ4sscPj8FIVONd03wZv;Anc& z>p9DFEO|jmb{nqo>v(7vp}KP~;>**pMdb*tLFhwZSqqUyfn!x1Fbh5>&hV5Me4r&| zcqgItEWPRw714f+%Nuvz91!}9mC=4_tj^sedC2>jD&hEtEbSM7v^IJuVs(CrA#S=V zj=G>Cpmy^YZUr|el{2_$io-}o&MV+JN|DDI3kKe5N>}{@jvbCGv+!P1l0Q3sdD)<4 zRO;H$Wsz2Ti%_SWZjT8<0gjeHv<$_X@5b@kn(0oKtmyb^R%i>mNwaSA23^4~mWZsV zl|}X{bjG4LVpO_LOOix69QE+%HkaU0d#yNjb%=6FLv4(yR$Gq&IuuXGiuL=Ado7)W zOW140*q`yJxc&4go@F&T8K~PL4491Kj6)~?E#<4x3K&L@MVdTYD;NqAW2tMmW_IgA zcA^!z{viC*E8|7Q2mC`o%5Z3K7RBQ%y*A7za+#%`9%y2ljrV!_;{Mw z!;{z#37A&=yg#cN(c&;mg}Ggv=t}PChvV`<=z5rJO@iCrp@vB4;qw*an6l|O+|DNH zZ2fO)jaw8YT#+(b1VlxP#+cHHBgE{ZMbT>Yqf|`xKKHUfqMFruJ2B>F^M)QOUe5{o za-kBn_0yFD&vxffjH9dOrzddZMopv!5_D881nVU!@}hp<-zWtyN z@>vfl0+752%LYJyi|xB@XOrucO#P7{z9s0!M-p#-!;2?U2M=J$%#U4?y}~#kk|H)) zT;IdfDI)aD={QHs6XirgA-TfxE;(>|V>ga0vTP4I9>&l3;t%ANhwXN*cJoM~T*d7( zA}roXRoKQn^L?LhVyvAg_2!b+oa)bKZ%Urz@uSx3`;(%~*b%7< z#>P}dFi-PuGCPe7^mr|N${%G6xr+?Fs@f;cxZIeAKi6+Xth^=KhTUcjmIltw zCWIl0%BRYyzW9FxEu2(H3seHbOJX`{ZR0{3L+ns)7hfP71Vj+`-&HVSjt_3&SjVZx zQ9;`0rj6@diTw=e#~=miK2=i&n_vQ|X!( zQ(}o_L_~!RM`n}ry}zp*XcO>v{h`c4)GV%J{Syc$O~`3gSZSBMi;A=d-UH{PNdu%E zt{UDtyem4xt4+GRcX;h{`VIC85c?LVlT$a{|E|kwFeiY-+`N`Y3+lKY;c;vMY$s6~ z5AY#TZ2`8Zgk~f=QsMTqrHiby6VWn$Q6FfU#zz=WelzFLnX3p)pykwcFwSc2)Qwrw zCrT5lFYonVg-l^du3WWGeH`R1GG2bMtB?z(=H3n@PWmc$~wVv|t7ylT8VM_?y zaRa00Z)e^Y{(9cive;oGl6C;S%OH;mL$>N^`R^ z=u)}zy8qZyhMcDA%$h=?w}l0R4Mj_evOOcx2p=#E3*-GaLqE9&Cd<6hE5CSd$l0E833zw!o>Y z1s>SqOf|@a{W9xr?bmE|zGCRsUp6kTGzyFGEE$mKd?|^<70O7;horl=??)Pwv=KW} zMRh_mT$$5^2+k5$bXv2TFwTrwE{Jyhx0?V-GY-ZdM!A?Cj@ZOc>A}Z(SNro!QUM1mqqwW)#^#^R4Vi~(#;Ec2 zj_6merfFri=`d?SV1Hh>U20yBP+!!2sQC=A|Jj#`toMkclSI~e$r)Mt2HQiEIZ$A@ zv=ENGqCI$+6bJl-nA74wD3@*q#BMw$kSH?#(I|g5w~_LPZYdMFM0tkY`g~%Iw^Ohp zRHunB!p$PzdDdp9#%r4=*&*%o9p2r5*xyIO=&)bc>}MoG>DeDcJTShBmQD{A>>RNW zgOKWSf{g`jT5t4sbRdK^^Po56ZEFIY zD!!jGokTy`BC$H5rBD-}vczmN`tPxif8A3OKb_+JReHMjwquvyS5QrcI2FM#y)-WHi`CoRmH|xe8j`1xx!cN%hx_^<+vPA{nV~tT{^&{E^Cq;^JsI z1EQ73-bt;U$4ejia?}n#R&@Zx#Sgcbibs7p+vgp>A~I1i)H!5bafbTq&XBLleowD9 za56t2LGrTD)43FS)fU$8e}EQUjm|Wc2(}sGuaHd+A76be-e2ClZ=G9xs#u4g?#cf4 z$)%a23~Uw?80gGpHoPhXDOF|jc3<2j;&cR6PJELkkGmrACJqsk!`wb7EjJ2s%7jSv zwG*&dXH(EuehWUd-KGtF)KKs?%a&S7TPFxH(^=O8^PzJkraFZh=4!D0Pq*y8qVdVrN#<=_~w_4 zQrVlG=9iAH8b7yUAvP@WPHjSN<^+2;LuOOmA?hfJ+GdYm&C*vib|W&Zzx^ygBBC>k z)#1~pR33$0CK!-hmm+U;NkXo%rT-dc=%~`k6sgKQxK65ZMv)0CqS>9_?Z6`H;fNtY z(h(Em^-V$%v%7}(>#{!TmBEZHz-h;Ur2ZmFom%htQ=ZF|6Y3HdhQNE_@4?gvy*{n7 z#(~{tZ)UXbD?K?*isl6#Es=5Ms?)_vr2zGVF4B<5rmq(H$-a#Dgc8uu={W)@JuZbDyLZ!)(|408Rs+U`+*97zpCAj z{b4P;Qs;Y6{NzbSW#^qss0sq_`EzFXcSXz9dDdAFvjBtdr}apC*T8f_fYQHvWppdP ztnQPh6|>2}G`-LCOd$~g5b*U@FLTI#oZ3}#tq@1~4oiqxGw}$-2zGI@$(qO2%G-|X zBWU{U&}P)Q3cNVyY)Q=k#tWf6D#-Y_K?E=E6P#Z^5;i<*1`+t10%IMc$l z@{@b}#`D?7D$AqeRp?GJW-(^loFk>-`@qTNXlV_JgbC8Fs>*D;Ayu0?HQDp^kH-|v zp@B^VUQ7hx)7R8t2k!pI`6PJI>_{$i$O#&pwPwuVTGv>Lbfp(oRgIPCSq>&O1;996 zfVNvFe|jlXQH^A76QNNe61=TrQBSWP9rQS!e0*&5(zoO)LG_kjdye=kA6@c|!^>hMO|vChJHV$;K7L zIB42xb3L>42bV3~bO_b#zvYQO!6xss+Hx?$Sb~Ab_*-PdFA>fO-y?Yq1-) zm^Nl}vRXX|JKBrWH6nCoweal%w$(&pbq zK>Ib!fNW867@Y8m#!!?}sU64DBV8F~L0K2|BT1A!d*RGTu(r;G=$l*tZ#R%5suIn+ zBEddAn`A(muu&d%w<}qGB#WUZBw=-Ccvq`r{mw$f;$asnkN;2c7NzQ7vSf6CM~ClB zl_9}tP>Y(0klF4>1O62nFJPnb9+mRt>J)r^5`TOyR_wB$FkpD}umUXwjEi!1?` zP>o58hTMxP#`z^Zp^dkO9$&b|AL680~JH8hJyGk4ATG7h=8yXHV;YTyF zkkSZHREb@usw@4&6Sr-r0pvE7>+e|ps-KA|8Q&=R`j}?V-FM$j94X4`#mV}mL=zwG zsxQ`6NPV8B`?J<0yL~OU2OuD9g|gmZ``xYq%a@j%jC#s-CW}CfpH$?#+#NiWN+7b8 z4L-_YRFx;k@z9Wrz)BaO?9P-7xL_5;zih5j-Qpr6<3Zwnnp_t-Pc1f96EPF2N5s^% zWhrAi5tL?9%d{kOt==V3+jDdM+<_v9)$tBP22w@YEKuA;Y-%Cof>hjsg+*ckZ16%R z98pP8byXQ$byJdb$m*Q?AFTn?e?xYR)H}TNVLqqj5bUF@psFmH<1FrfDzyTTMFk_O z*zf&F+0IdDux?WO4da3Hk*0gDrg5ZLDCwGeU#UrO|qDwWy(nQ|<*d|hsw8|LtDiw9Vdj@$ayrDnoE0DE<1sNIlI(M^tN z`A)**c|TFh_?*HrbLqnIX(v+~5J5I3Vo9s?wG%5TCQ-`)TA!=O)$hsi& z9f!(g>TqQCs#*;+vI$xbXa1N8I2-ZL<)*41RGbz2i%WQ6ID(shh{HU1PNK$AmR{X6 zE;;hziuSwCr?G8|1{wy{MDJtbsM=!hC$^YYh{5U00k!Sg86x8t`C|kE|io0k53!mE;AhthF zNh*Q@cG#~{we3jx#la@125=&lAVjm6qg{N64U-M9KfiImYFn%7Bi#Fh=r6` zQ4O5=Ya3(0u`@B%u}2>;yB(P`Ibf)Sgg@Rja}&{-1N4x%X_^!>ohg2khiqlSo1xnM zNa=BOi^TIFVZ$<|2uKEQc4EL+b?M@rd`1Jd1BS=*j@b?5i1|vzG<6?;t<(sn+3Usu zOPf%VkEXU$HfqLjM{0K0QOa z@<;?ah>t!R(=y=IuP%O@*2uW4Btv!6|27_2a0T~kT>LNt4t`ROADcc0GL6KErMZoA zDf>=9K$EC+*?!Gc)@k}MM*sJ|sWaxZ&x>mEq^Z|s^p?Sf!K+QbqemPn1Y!*`sviknz_$yKZh>KCN8rlQXQ)u|m=9`WfzP|& zivOoSm7-glTS3Cad^Mw9Aa&(!+67}@k5ut`&9}H}U4P)d#k^{kD>!ypX1I+190(r9 zEr6bHOXT)fX>ZqHZYuN>zVBYZnP2Sc@g*>5Atk#NeAN@{7w6vISf1N(K>2l{(sVNO zgTO(Hi4#aXRahd+vv^%Y_n6t7iOLXC*xQzQ z62VHb-7)3F5tUXZ_<@huML*j&lP*Td1J;F8D3le62t*jd(%#gFVY8&EF|wD+x~+rU zmG#@O-{^90_~Iy=uUKK%NOirQ1-(pU7l$z0QUPa6suHQOC~G2SJ^-eNxR394RPOZ} zOFFTP{2oB#N&tkPKhs4?07@@>&Ke}2J1a`EW0bK-NAZ`%PsZ(D1 zGfIjEZk6wiCM81rl>NBMNtMvj)Li^Vl~9N9X%gz{%iI!49DnU&nOCZ!zLcjI*<7M@ z*qK?+V=`s-3eB1?m2%}#`=M=kVH?NAGIMl&>b>Gp<~#AvneAd`{OJ9a)`EKeEZg+p z${-R6xZYuUJX11SaEIKSnw!$*Bw3LlxUhSSZcP=;kny%E*6$5>G*}iEx%^{*^0{xB z<%=dZeAt`gF1Z4=@mVCMSqy%@LANAVOmIBwl`r5V5>*3XM#9ppNEjhQPzf72tWbAI z$*~FY&12$Dg1BfP{G67I5tFqf$#h*-6VUKDJ^7v483jciX9^xQL0p2%ehQUll3qP& zoxYX~_7`{UKHk3by1V(aOCD9N_3UBb!x2{U|&IKBQ?968#FKt zS$t+^7ka>Ss1a2t$p3gwoRLa4))s;dnqTz@Jz{Xkm^fm<1{zkCty<=!zQL>s_3E_# z4h=72cegb&+L)!O=@~aO7-P)PZ;{5g=7OGicVhF$lFIE3G{usBvm=6+ugzCH%rn_# zT3qQ#CChmK<->EpsNOx-iY;yi(Tj6U<+(JXw{;2s_~Qusg8@xQXNP8Q2-rb{#jptC zkA^m0u-RteCQZR&Nm3C5b|1uDvkrFY_kuaK*RZfXyAw8xUPYKpGC;}40Avdzo#7rt z1}NhLI-0q!7y9mWGj2aczwzMji+YPM(IJbYG$Gj4S8u{sHZhz4^yg@$uG3+ z_vU(o<+;rH|EM9!=-VGW`8B<=FO|7QK0UtbkLsy*ykGdwXK$VIc}>^hzqN}F5w3bH zt-GpNW#7J+qFzV23jW}rm@!T(-@CobGn6hPPGiSU$Z#YmNd92MBAKrhtZ*rncdd3E z9;c4IJ+?gJyp98}8)Q0y%bPKFI0@6^rhz1fWQ~2k2m}M-TeZe*0apTIry|TImJ>l* zCo`%|6*3TN+p#f1gpfY1)V7J*y%pyR@bym%-SWhxS6 zE$(Nt!g5c{IA}S(_UdB(9QqA%mCCr6JuG7M+&U?seg*areMR&x-IK^vf8LP5wGM$@ z9kFq`0z>BtO^x~ll_4chizav9|aZk)hfw#GRxk(Coa<(l_x^h&akoA&jX zSmTR8qOJC9Nj#7JW13uA>k^EW4dCTTyrY@B=6gMO1rU4l@N+F^dk)jO?f-6hEf^P^ z@xUCR!}hLFU+BpC`OiY*=PJt+)!2XaENK`I6>mPeV-XG&?7F8sqSCNTAnPNQIUoxA zZNUDb{3QiW@g*KukvZDei}_?}7#U0TZGXDGuljG|-LmGX&4A(oidxgh03lq&mBX{z z=m>L;Ac1JH~p13iFWmhk&{jsGjWnih}(K-eg9~ZTbQ{n zJ8PHSodFDa?l-0IZ1^+RLSP>3hN9Tqr~YV77w^rgR_G+IkRV%GlYKO#)Ez`I83A)S zqgGl$Q4uq0934!(z~-DO&BD!wh*QX9dK!b-Qc+)936IBoYT1oy>$IRE&P)a?*|Ro6 zo<+ZBhQNt6^k~q(_6Zs#6Xv@TTRlt4gfd{ywO_2dG((7BT@wVQ+A}^;4{VWs{6RVL z$0jMQGcg;&LFVBSK%VC_Xr& zfhZn=7pyf_wsVHA0cqBw*v-%Nn}U*wEzg-qAP4%Lmcu%)wkFIvD5}amxBmWQ2=)wD z*WUx40nLd@ou$+qW>CD^s(REbhZ=N~D2x{hQR)93jU2jT#hwZHE0rQA-eEc>_va8&EU((M7tSb@-aWg4T3J~U+6B^ynq<|hE7`@K*(NOt zRcscRjm3}hJTf6+9Ak;Dhgko;gBkWsn1Ns6q1I|>F0um*` zNBiP)Z`n0Ry(=_+{Og)DZv}`otQ@@G92H5-NNzOuw|I%iBtB$R_Dl`u(@*-B6D#o~A`8@~%@gNE$u+bb96OJ+T<-Xj!%Q__ zX>7T7g`4d*f7SXMsW-W2A?fctv8ieDz-wfcV9KX*$Dprf%r~0Rlr6_UvYqA7*34BH z{C|H7ai{W{TW-_@KDgM4(wULat~|cjd3*Q|?K~~Q-y_=ilt`I+L<|ir?9nqMT}&5C z;OH^kWpK{$kQ!=K2b%KEik6qPY|DT>uJ&-@%w1k6QS-t!HzWkHd6;FL*fZ}h!sTS^ z*ztFMMV_g^p9q`Tx|o9t!$RzXMdq-1Mmq)(_{vJ00N=3hFr@$JX=dqA61 z&Vq<~VwE@#LUlki)_0RxA}Nx7i{D^Rm5*BK`{gvZ!fN-Wd$>wC`G1kRzz; z!)Fs#=I#q}K>Q(M^e4Iv>IZ4Fvmtp>_PA%Rd;ScWYtbvB&x**RQP*?Ej@`dba+ z?Zq6%J4WFJnJVd_cr{iPIKXQw0n)SBGo?8d9&*s*N|Ris#M8uqm}q#>uUPLUNyoXk zR!VV$H%+&3L1di0?8JRNqST+ktE12h>pjxTW|BpANedXviqlGEBaRl9B)Uw|^UjuX zNRMJgb8D=F&2p?+&om1^VcXSm6=lfIBx!%_f5C10Ac96&Xla~~Ak#|Sp*q8t9Vujj z+tz1CRSAPiNzn9f)L-<(+gM_aGs?-G0{BjhYIx%hJiK|->U;$O=bJ?$vwota?_L$Z za`O59`J84OD#5*|SSSK_vZa%FlhR$(1H)ies%M&rW9A6dvPd2OZucw;>Z-&=zz8MS zN7mvzwzdUKqrzYG1HO#L|Izhd;YQ+0!doBFQ>QZM6@YCpP;m@wcK^t&+=YdAVjjJ5 zdg#=E(4PA;qGKmpdq0(07PRZN+W$!FiQUIt>Teir$$Y9=K2m1+Pi2w6<7~6o65W|l zK%oF;f6yVg$ ztJTqd7l3$mvJS;WiKcc#6a{F;)Tal^qs{+ijTUDKwnj6smlrh`B_3-?^^d>RHN-_` z+PgzPuW;A6Pm&+UVTYckryY)^qRyVw(uUr%r}So(b;dCb@m2cMc5_=KKz;;UktVkU zefKa`s(Mw!VKX*Us$$2ljNXvgZP{r>f?rk#Z#>yW1+n8Fc8tK(?|0)8k9jdD%zq2B z-_cgoQgJL1_^1(xopyRW`YZO|-CyABtf#5QN;aCPbmjL|=MPHV)aTZ$!7p)N&Ffx2K1dfcPjp5kD8C%! z0T%pcxcOaqXV!H|OS>E7Hjm#Hd1$v~bT2?;#3^#Y&!bv4--@Y)(X)Wi^m++}%7bKs zz0kl2t}BoZ*pxx(gWjV(POZ}g|(?0l`yv&uTg=CAg9u74F&N48`R zDmvsU?$zY;9cS+}%*MyjkOIfxKB9FmcK*iVepOG)?Vr*-w@wVJJGTsM@7cHo0f|1K zcdo6QC7eoa)hf{OzH$!8dJ-9J6rI_~tCFp)>A8D3xn2)Dwn|J(djAq&|JvPNiVf#O zeO{YF@SxG=`~j}Rkd+gIgxmD2-c-LDXW25&_#D0Go1tTov>ONFx_QE5z#p-Lb-b@S zJqK3LJrwXZY9s5C`L~YX75%t>9;04s8v2FW7_cPad1p86a+at_#Urym7PAbPIUn}_ z&d^TC_AVgK84*iJSM|c=Ix;721;)9JWiz1HSYiY~v0ma04erKPj$ZlPvGj}4_6bg8 z+!u}{yYED04+*t*a!MD?QINWQ`krs&e4tA4AJY*Gn>PFm=XeT8eY-qyC9GNkZb|D* zyj9%p@7yfc4Y42JH1a$W`PRxigmnYl5^X(4WQ?}8If=0UT}>ZEwl^O=@@(hVxM#Hj zh^*bWwHxebokOl-;z!UE2*(qhAG&_`R_cV1-MsCFFRbVs)ly_Zwr+ux0<*8v+k+Fq zxQPln-Wy4f;F0n8o5rbP6+;+nco1x1?=DwEBZgbIj;;qWu-+r^45L-NEiJN*2gX;f z@t_}+0RQL`!RshMuIPa<#p0@#+rXm1*oeQG^TBKDUhabpC0~8TU+)~ zC~ROS;lS$`uCxxBwALrUvROFR?ly0>HpC@?Z!~UCpZcfr_JLHBpH-IA74bkHSs%b%YG%F#}NL~8>#eSa4R-V8OY zob!Qc(v!sy$>@Ux=L<>>z5XJVONC*GWNawd_?#%tPgKVC%`Xz!%^ z;6fCb+u*GM;Ck(uqAI+7jnf!!bY z=zji#nHZFTj3yu_VQ0Q^yz3MQMfEaPxf!1uw5VA)oB6<;?SEw&hjyz6`<8m55`yeU zZ&jm8B#9D1>T^2!yD2-(brVfhahEI2D!h8S9C0VF27D1Rbt}(3!AUufb#+aX`~BAv zxU-rEGwY%*pOJ|W?E*!c99b2fD;*SZm_Hqzuw$st5V4;V;LDZl>kw(cA>4AT*80wH zXFHZG%Qp$NJv_hUFE2f}YAWt=>u{TB8SQEQoza-~jCOO2-e&Njg7EQD!dsJ|L%Eu)IAd^e>Rw zXV==lYFbw23v7K&POR&YITzSz7S@2IU$+5x?_9YiJuI9?&{Tfo0hAi@;44#P@H@j9 z`kSYUo7U^;MLPicdZuy)E0PHRuc#+f7DrK>?pX0ZfZxn zQs8#nGd;X_*lE< zw6|QnYv+?(cXx%|jFpC^h3`=&)?p0+a5oGfJav5G84QFZ7v5B zwXg$F3>UM2!!4$V4`PpMK!KRUJ6;}`~-bG;db4cWX{UDoHnpW{x@>QX?fW{ z`1{P-1J^FEko_rJgNNzj&!U>V6=UcyxV0F00{D$0Td-Jnf{w&gQD3Y8`qnX2pwdrG zPiRXkqpzR6Dd>JiK(Grakm5u|ji8g%VN%Xf}Xhm}4<_qOa)*0-1tAnAbTHTykBDE>Jp>scRb@G&2 z&|jZgF?*o-r%23p*$(D3-tv+lHSfBPTF!mBC*?3gUg5Lsp;&jFjbiIKFD4c7{-?;^ z@V5<&_HDyW;`HG0If}Q~cb}BQ0QW+4$li$}yoLGUd*+3$?u9k;ZB*Weah@n|iGVbj z3gqw;&xKRSvXRqLXy_Y(?w!syYx7C1ulV2Ru*FoZj6GtxTKA5^)W=o9!f?LEB>=)+d(=S z?OJ8^%yykVrb|rfJ-r6Es;S=Y;k(j^@!_4^+=(Z=shN|Pf1#!09td70-qY5vUI?mG z*u@Ekj$mz;bn!tSK$u|rjE>}_u;8h>czEbfD5oMDedTG8j{#{{2Cp- z-;45Om4)q?jp=3;2W12{Z)rkepTWAzW+M9>l+i2A31yFGywE?~`tMSd+0KBFy>S4s z5Lq9OS;CB<@*i>21V`fi9tH=fL$N$V{?`nm)U7curR$%1yi|B>ZYj^xRHKm9=&0sM zHu+f0{k;VQeqpfg(#`~vLuYPjXHt?{ILob?i=AAacADFwms!q?*yxGQ&q=!oxgHpd zjM$u^$6cE3i#^YXBnRpMFDnULW7>ODhwYna9)_SWkSYd)e|SO%zTYn!h(XiU{oSX$ zzt6**1~#(At1Oud&ZA^1bd%9~1m9~aS)uM|A*9-1i?$Jv0^QfHZ z#Caep#5YFZ`**+5RUxtHlhT?3^(XhK^1_7$OYJ1|f*wk`m*#^7p^r}$poe3h_h+*A zbM%+|;6b5>==bY^_orK*%bx!s&t9PC-k>~PAa}g&cf6$eJVg7vZj-(5KHXNmOZD^$ zgKrdkg344-Ze70+3t3p8y}3p(_Lp6wOSG-~xEtg>BBUL#_&h`JaDPSGUHJF*Cefsh z_4UnJw&U&Y+~*~kA@W7v=k@g8+rxf`<_?eTj`|C7e20(jRZ)?KOIil}&!5?o4*x8M zF#xAeDkAak>G?z1$Cif#G@sMH2+>8mM_$YOn@;LbMrQ6w#|6);_E+Cw$*27(kZ=9L zw@2^)El_^wFr^G2r7Xx4crx@IsO2Fw7B3%p8t)Fu@-)nY7z*8&A*abY)*r7T4>^Wq zlPFyn6Ue+Kc&oaeCNf7Z-gQXET|)X3b&|B7cW}x^{$$;U zp>J%z!(e?VxiyEOt$4_e(ef@{D39nR7dPPWbAd+eKB#K(#vb>)b3sfA@7zyErSc}X z>7~SuxzwkTz72um(X}kjJyrX;a5_cH+WHdtw-9aDl9n~C0YqjJ_W=4~@(gcCaLxEVbUAnlDj69LE%Lq8n zWe`Tu_;&XjG>YHTL1a{p8BVc+ZdqNlYG~AtN7Nola(}A}+YD>t2hYOn|EMEtIh5_D ze;&jCwrk5!M@GfDb)3z&Y#pz%7RP54ZHCS{BsAaVaBReAe6Fd>lz0 zu;@J_?Z(;Mr0p`H>FS|4tFCj#ZiV0aG7|E=r-wDzRRJ4Ko^L-6x+yj(=ISBaI2SXk zq(AfL2hOrl*eV4%f|P?DWDZXWIa+Mx{W)j`Frz;aKM9Qm*`}w+RFD?gWt?ZYeEOwJ z>3%hIfpr_x=9mJv%C^=vnBVQKRJ_DT%;S=W*tbjgCE&2dZQyZ_H3kqz#!Xn{Gr&KQ z`XxKd-DOvlm;JD1pC{oecc5q#y=9EOO)xoU>I+=jZ=E+ilsHF=9&L4spr$FlqcM0} z;i?KZM_hNGN|n6(Hyouj<=;yyGe5K(k-?Cc7_TJ{zKPNsIb$u4=%JR8{7fIM(K89? zUeQ-wCFMa|{jlKXn&9fc4pMu$*7O|wN{K@ z!SHn77A%?ljWwz+gHm&YuF0$%Sl-rRYx~pKZ+}GB<0lA;OBMkA5bP>ral5s^8AXc2D#hk7cF)Tlu-u!ta(6k~hVU?iN!H?L0ypE|xGeAJ11#hSc&DV!bj)qGYbb&vjbM@d?%^Qm{*VO>6dHnmv_7%uoq;xLMoUhQgdFsw+5p;7J zPb5X$^bwR6QRg>m;PadhRQeW@Gi9qxj=Dzz6#RK}8?Ss=6oy7XiE>gvQ$qm@wQJHh z=CUixdo;1_ntj|#fyP{K{P>6I4$K^Aa?$y*?HIg7Gr&d`G*1G^?#49q7r3-uzl%ch zgZ6X6q6EiO51X~7$BJ@`AYHhGXk})?5VLumkAJ0{5E<%jHaRgshG;iC6j~w8o;;p@ z3v0DIMRlz8OUk3eCn*wXdo-`zB7twxxZu%;adwUX*IJf19sFhk1 zDv|MN3?8F`U9O-?JpnNN@~x-5hOcvdM!H3Ck@Qy;@&=+3Xu=D>2;(43=LP>mR*c2b z3X9AbKpksep)ylt9V+jk+02thMDub_rt|4`1VYi&=gL$&A?T3%${YT6yJ=bRVQ;yTaTrDmBsH4n_8RxXggPDdh#9LaG8Vi zoS@J*X50A-YRQ)wgj#!o$<&CbeiWMJ-|`n~w+Gj<=w{Ek+y^Xu9d zQJ%2AfT)wG06Rlu1R?s5%b3iL(Vyy0h86ulq$Tyb!lK#RwEPTtA~Tu4B#I8qBboQ& zv|O5|mY~6PjYZ|+T3I~y_pSxVDLs-T6d8`hrE_2}SA2?i#!rxIW4HS6rk6iX z#C_^i>#FVkWYi&Db*A^-@|M@6*k1-_s_>+OG;86TMSeJDO!&Y*BKE~Z7`JgccaJn* zH`9k>&Zr#(5pgJ7+VoXT@t=MEaC``n1h{=m?1XoOKFtI^1AOn$JFk;;3`XtPmPBAK zLcdZUSw|8^U$KjJBywH;2c#TzG4?I7&nNm3ti2(N`b^MWk|yI)@4B+ zv%H(5Sen!U|B*nkTpmSr>Zjw{7;8U+tow33KW^SsIthkIuTh@ze?E|L%@U3`8`?M^*<@qzXBv$kN>m(7ce%h#%b<;2R%159GjpA5obX7dD; zOA!Ot2_U62o~Od%TJ4l1;oWmf+a;y0Ou(Ig$9q)9rFoz-|CP41?I$KCH|=nkvyag$ z$`qoe?&=LDbR4jHY4N}^Oqk$-9cz5pCXq?nb7e?}-`$)6sb3oq3K+qXXBSvsN9<3A z2FD0;?OO7egB1t&W){Q}eYzH!4kVAMVhtA@tMN#-^1rHL+i)|a#wQxt)myXAL1vGL zp8P+?buPyj8h3m2D4|AwJ>%Td%Y(%MqGmO}86tZXVbqEy1iyIcbj2vNGAL>L$}dFE zDZ@Wd*7OXYtN=Q@{jth{NW`%~e*I2=_FGcF66!IRsXyv8Js{Eh$SXL~-AlHr%{RBX zsBsY78=T{}IC2{{gTN}F-{P{?gh~IYzSJ}Snp7rWp15_1UYs~ek2@pOkd;Yvtd?BLGGJ1sNHcJO(n{rdQ2l6LUx(w)GyOB;YqgCO?7CkhOk z@vzb0enDvq|0{{|O5&m*mN3zYL5pdwAQwMAYCr_+yxtlqXQqo?oZ#N7}%;E2>4>u-nTl843Y-V`?RlgQWGgP4_a=8L$4MS++Oh2jQ-{EvrzsBqzv z{h(IhP3QQiwghi}>uprd6=xK$jqqC*fR{COZ@8^xIu}35#_Hlv6pU`ou`M|EgR|1Q zX@B_`y8k`cElb#P!D(1CO+Mdgx6H{_1~Y{$4w&{e-^pI9qQ_x~p<>>k2>1lU zp*~vu0qBuw_@*?RgG2<)DfUXY97;O*_jp;7yE#!tcW20TH9+7}|5(q(@Ww`*_0?4N zmML27AmL6`mM>5sJYz-`{Gk~&5a@Q52!5~gV-sLAu} z5To|$!A02mx6=ar1H-~~O(SyFa`$bk6)QMP-Q>Dc4`0-{PtP-GFxaX6-!uT$w>tE= z0QUM&S;Dc3j=wzv@Ts*n^D4FwGZTdPp*6f#vbbw-cPF^(;_mM5{@>rdxw^W#s(Uj%Gqu${Jv+P8-Ou;)eNxoP54Ntg zSET5j_et4WS=?s$C*Z~gW7g`z{dH%?76)Rp?i`?52ymhr6_sspNRRpbQdhK@-(~dt zr%PYhOdhSk=XHJ<>nUQ_yOFti1tHsON*Hgg-uXBXG35S=1hwp1JRpaIgxPXxD>Y#_ zRrB~#cmfEa)x2|CG##?-VJi`EVL01AQSe>-YXl9oZf*nay~O9LaMR16x&|!sGIpr! zX?=4m+z(&Ql?SYkvk46y)I=@ysJ2~7;&iv-x80?C$t{j4gZ8CEoP=9T&d*Bo z@$TK$joH+sMbOuxE&7j2mfjHN9lStR9RNcjazh6e){j1NlxEzCD&6P0Wh(dUZxL~dj(%FXR^zY5r!Dd`c4O;Bcmq| zEGe$IUnjRR5<}Q=G4W_4nsXwA}HMz-(tSu-eRGJL1zRR zlK#Beijv+niPG`jFhK=jSQpFdJd8+?3_T|{niH8i-mEWWnl(SpS%Y)I9#@fFwiHp= zAX{Zq9y?k;f@^w(9r4LpspOlIM*oOz+9kP~rc41MO#zITn?*hV3MP?rbo#p^YMFg) zoqc||!C8OfR={1GR$}rrwZN@UtxxMiY4G6_d)}cb;MT$#S&o0eems%@Mum6V&!IN zs&0S=;*hRt$8;%-F(YkA^)*tHL;m)k@F;xo!Q}Id0Z7=-gt}}XmcA#W8(_Dt5@LY< z6U1z7UTI;x%)7&deYEO=sYkj)iTNN1C4A6YUsxI7ahBh$dy*JQ|Yk_vA5(GL0)oJ{w=OE9arV+L7W zj}ljE*hFk00iCzyOL6j5NPx%)mnU&bYBh1oP;V21l3| zbjijs!M){=IUF5#ifb*0AR>-{prpQXRvQyTB_!1D z!TRs3evIT_<~wbl$dBtkX3xGVgZRmzmW1FXo9oDQr@+;UVPB zcKvv`(!YEdL+JHS{cEr=o>Ehjdtnu1jr}@ub6|GmtOemTR&&icHWsf6Qh|dxoYbr^ z{{}SiYeVgmU+YkiH$D#OCsGS#jrivs&aRvi2jx$J7`6;@mQ{!*YGzYA5IclV8sy5l zahJm1vs0$AB%fRh-Wb@(Fh>cDUitF3h0pVcf8aFomQSTpK%Vdsonk)l@l$mc!R&cE zc2F}YZ(1uhc|_0J6Mq`-q|Jm8Br>S;)Ird@{`M#tDYW{ifb1qO&*8c4KBLuSudhQ6H2LtLtB?Fk7hBG{M5SQ zpA7xU&brDX2e4RP_<1e^yW<}M@8%XTX8R6H`bi&vQJE{vYOlb9RVVs}+`%xZdTqX- zT!i(=Vtx9%1!5#&eR9_IXA*8&Mvn6}oWfO3;EQ6u#{mo{t@zZe({*+tI8NYEtixf!M4I0S(hEFICYoD8o;y6}%uxiX+_7>%H#~Qqa=lT^IwfMl3k2 z4hSFD4jWi^QXIpQn!@?2zhn`+v<-D!5n5K+<9yWk&Bl8&wd5a>->SURz0`Sd?q=m* z1*NT1rcYBH{TUxL=1$!@#uN@Y%;7V}S zMq}~C5N4^&F{6syJ>Ao8g!os{x%cyl=;$^qUt@|DRozPzp9->(^F|F<<*)H`o_mAJ z;yJqDsdnG%38giVcZMh*$orJH0=kwa)F*yKLUJLBR!Q+_%ewZO0d>!5Gzt%_!Nf_pc2DPN{Gr`n)d(uF6LtO?DAS%$;mnw|Bf-A?gwYPD#oXHPTN_jIu+;ADnG6q&{0Iarqr z4^*o*!2C#nGyl)9hUL#cJes8@QkPp?>NOsIZL1K(Ty94(ObUk3)~h^Xub)Jcd1&>) zY0}9PZB}qgR^lv^(;k=eU=aUKVtc~`q1ZTI*(y?ff>DOFscO-`vuXC^U7UinW}C2H zfY@0!8ZQpT&;4ItW@1qjz$d^QR&RScw*uh+RtCX5Y3m*r>-Vf&+0YE>0*~$v>vOJM z|lS2{ZF+bK#U&9`nF(ZdrFCiDG^5->G0^q;&`zG$0CK zySn$bg2M$S#WH1;y)^g=l7=WC|30UnLE#v}GSiw*MQ9l0s)9g1FiYpmGWTHzyR?6b z>-1C@8X;gHRJ{o&k5c#ICilU8tyC*d+cIgOU|g*ml1Wu;$^nEwfdz!n%T;^CLrAG`N1z2c_;cfk2OZx=+S? zl9*C3+cDIxmWHLOCN#8-mR0zIjVH6;5~lnIm>jXgYIgG5jynPq4i5Ql&uh1$<9k|YmdiwR@GRADjf89F7}#euKgXyJSJv(D#HWcxhzAw?3Tr-l zfVD3%KSTm3Z2iTsI-+Tr%ttAEP{%f#EycoeHYW5TdbU|TG6c-P)9ZNoR%QYKwbyr$ zD9^?hboxFWE>e6HJP=a!>K=WQi6r`_%BT$!t`v!5YEXh1pTjdp*gD4u_gS?S=R9-He@C5nWZq&#f_jjY!f|EPck=)&61;N(zljf{&rXGWWI3TUprR?yGwJYr6Ag%_{(h zl6e4rKZ~Vrm_z+`lzozVGAkN_kjn6i^B1&ZP7CE)aM{YrY5l*fU@s`By7V{Pvk2{h zwW_IiWenz*+!)*>lD=|1@!_Xe0T`YhP6{XuW@wHVyOZsDacbKOxCM*&GeI) z(X8p8xd`67Cov{QGehRG@+ocO<`q!UwefBi`NxOlYptya*1*d6$lx@-fRPE+$ePpe1eNiYv0&Z$l{Jt0 zha`_uaIOTM$7x^a=P9V@R;i#Hx7*lJ3$})wt6EVi!F4adD@W)f6Y$=bA{X zY?@cQ(6;i4J;|7F`$}q8l=j3$8RvWf65*PldX7oD#v+CypK!rQi=2>Fg ziq|!$!C%c9cp@9Ux-L*>|j=yhfR*0fephlSzfhR`#Lz}=P2%ILP?%hbOk?Nrc>*QvPe z{4!RRA2adv$U}`Bs&H)OR;d`0=r+6V{j|Yvh=@*_jJ7|AT=i(~ZS!w!SNXNR9W~^h z>~*hChod0gwViSYe!^CTj0^vULRRG9Pvj{2u@s&T-#h}|d{EmzV)PWC{0I|UG|Xaj zUXO^MlB8;@xh|MSdZR@kZZGz~W&acy{C3Ui61$;?>vI3h2+E|ao}$A4c2ul1+LRon z{rP>l?NmaWUlG1AX`@_;TM1s?*}zD=v{n1|F@LJ%i#z!gWC0n0offpT9VXvhSZXy9 zv=2Y?>m%x>!o5dXA^q<9@5`B@!gz%n03QG8Z+KGor%?s0%JGLIkZW^_-xm0fuzwjf(1U8^OBhYN-r}%c7*Bmh3(%%O5)8? zCc7&^_(R_Vk5$MvpHBOJR4t^ahZOsKI%SY5zxD=%xmjQw@7v1m4Y>y~&WcP{Ofws?*d#1~!L+>EtXgB_eOHWjwdTC$=v zqB2gPaZp!^L*A2nPs1!aB0q7N~)|kB4i^tvXOscjCoAl zr<&30PE&p5*MKf6%^3{<$yu+dXl(9ODFByMt{4?3t#11Ra;sJs=6|rfJQfM}?x$Ke zrB!W$gW3+uS|%(D^1fd`7DjRMzuR$~eWmRlJ6N2CIc;0P+BKpWwMi!kd&4Xvvdv8HcBJGV4{$TjU$}qZ`gU<}y;@Jnc`m^IAzkYt62v@`d z%_cdpi`^dx9mI!qyD<9^wKch`w(v;tTm;oZfY)|Mr@d9eZ;hFD$YX8!3hEDk2c~e} zLGV6H3JC95F3Rx@HNb6`b@Rubn$l)Otz6gW9sG+Qohyh17_Wr`Ps>L1nT-Cc^tM4- z6Um}48yB<2FBsFcYS5T36bG=#o&Oj=zugr@je31&PTPjV)scOGGDu;ZJq37*UNm;f zTAz6_8bFnw0k?`mD%#P_J(<0>|IpYgi~((W=7HYX{7qoxvWbc}dZ->V!Nu`4<6D#_ z$C?YSIF6rPAk(~nwEzWhkO&G3vsY@XqK%;qB1|{oJ%(cOSJL59jf|_Svig~8HD*-sf zo(#qJk^9vgZxxqDc86M9gX^J9J<(oG%|KMEdpywRfAuO~=hx{6W*-0yL5hB6cL`B0 zEk!28i#*-7Pltqp_aiLLed~0a2Bu+ZW|W(W0uw>D1q!Mr^B^Of*Ufwh-nx5bq|~j6 z82U`$n)-zeLrFa}i&9w=>S6w`_DvL?t9$>IrpLRHm1z8158Tx?d4`Q%QT@z<-ZGj& zg-q7UISb=F1zq>Q%26g3MXx#?gB{+d3CXO?i=X--%o0ZJ#Lmh=7yZP-yYklPeHYYe zmhQNSXrS*iX($alI_CF76PiKd(qj`H%=>hBSGc3fo+Fur3z`f(5Ps2W;&VED=kLsJ zoT`lhg|J}R@dAedSIj?VXL56%W@m4e)k`udh-&Qz{&?W!;c~h-v11ftRj}DCLk`MV2hGbdOOE@;kJ{0bNhE56 zSggzo$z?loFg;?k11_|r7nGNXaLtXQUdT`^DHS89WD|2&qfbA90XSnZ?R72~O3AEx zj>u+1>sw#@XSrr=_4ass5C&IXP_XT}!@KCsU9GEDLJdWiX*66^+x+2%X16a3N!tvk8L$($jVU6JB=5B{Yc zQwPi?+jD8s1-OZI-xIGBq~3n=bH#xdsc?Sm#Yb^fSLXd*35U}019jdSNm%nCe zEN_sU(+SGh?OS^|M}`Rs*XWxeqUgWvTJ~({245)8MTl9^O9G|n(fu|<&2Ck2`uDkX z57V{zlD2w5+CqJ%IQ**`gnd6qkcu@Lb5r5S&T$7e{Xt^S5mWF&ljbq^Mzbk@xpxiU zIkC7$-dGoAIF{BKlmV%mmg2z`4EB(?_0y3BCeU?&#KxlW)p0&5%5kt;{_o+O<`|(7 zu0f5PC)g{oB{a*cB*3uXM zSWqX8kM&OR=LqMm1OgnPh^Pa$dDTUA6;{YF=85?rb@2cr=sQnxXI&@8Q>$tiuNL?c z+YC$ww7|F}#b5(-MBdkW$Z*-s$Mc~)N6Vh9Ksgp^7_51tG{2>MxYeB{ykbFrJ6l44 zOkOoV^3pgI4atEw$L`6#l@BCoCZ@G zvon2TsV5eO`*dY%6}e}8jBmD>Hl2Tb&BlHTc~-4cnBF6b`E+XXuAG$CAJp~Nd3}7E zNTJU<^~6@QYjl*LPp4OSCyR@x3{q9)@uLZQTxOxzg}ER5Y`VxAJ79p} z#**`=o&|AhO{Q5+!i%IW9o~H@SB#iBV2_3! zP&%wc^3*0@Ooylc?CA9auw>pbo1JP?VC~&Jxa3SHy^sW7c9YOvuu7B4BWJTy{LEtG zoK@TOIauL9`CEtyw%7p^?uroif=I@q(v%iUL|G?mZ4=H4Z$^rxp1^EU%a?Hudry9j zH@rr#Bo+sk&s!puuC{uLDZF*>tzKR7Ec}4zvScyt69txz$a&uH6!?S;GuP8YZqB(O zCQ15BkFhDY%`!~!M#=%%D(D2Z5Le3L>|oppEtr z&NDx?nk=h9M*Y2xrN!+>R5UHtlz1JHSuGh9RuNxC$zga(e;d8_MAaj2t1ngD(}Ry} zM^CF|cv^q+uuEm|e@3^l)l*W z6;kX(TCvU6Ui=W^a~65vqjHKKrT`HNkn;}2gMv`tdf;#q<-&<0oBe3i66WX9JW&*W z)do$qX+5(lH;SqcMS`X}htlZ{g%_;e6KHOsEoSPQH7&Ueg9*~Xv{dad)uOwct+>V- zL7Mo#$o}*)*?ZzE2W0GrgD$;$S@$D+=_X0cXO-g4YeD4zsnIx*8?-f9!eo9Cckpp)W)qh>%hCh8*&No9&zIQ&RjW@~`Rh z`^&Uz1GWMrDYm)%8}s9MAmy_iWZV-9{%hoJh>(oCJtyAQ@W)2dDBd_0ARoZyW`^IT zfGu-AR3v00H>iyOoc|l2&AJi3Io2G?@~5f{w~b8LJOQ8Qh{8KTivd74xc@b^o+ZiX z23jjm8W&T)MO75XlaUjyR}}kSn2L#~NYqUM;Xe7`)91zsH38#dYa*1MPja@SID#qP z|9mesRd=4#uW$12w#WBbHqZC^!h}DV&5jEXJ_#pS>y@g@yUrRU7^u~N1#uRSLe~892#tBFP!?K;FHCj*VoeH}A{1D9C@!{P(rI(#P&Y zyMxc;5~&f%3bu0VxN{AIe4#EfQ))27d8DJJUwlCxP1S9S)WUT2Bh zPq-Ot*x4k8R=>KYOJgSeB#54Kd7&bEa{eLP6wMr`=gC#1ND|F_61^gaj<7JsF9sgb z((;NtrS~BtjBlRObbWvBerCMC0SCh# zgf5*384M4J1$xaP-VVp;;Cw!Fc9_w5S+B;tLPsXW^%oodY8yl{i`0Vk8*P7+KS+Yp zWPIJgjF>5o4}PAug8A3|pRVAPe!?>z1!zOc?lt=Sd~kyF3%akjf-IayVw~Wih_-X0 z4T8xFH-}EJ{$Z)whmxYSrk6d0gQP94e%@R@Mk*Z-EpeK1Px;nAXQ*IJ5iYKl*L{r; z!8v{86(l;lNg(9q;+XOYn1lYlqLV$==HH3>qww4bx*cl zI*|xpbG-)#&o}$ofcEDVZ5n1F-UW}yJetM2)L+^z`X=@Q${_^5ojHdtssp?SXZ)-C z14zZ+GNjFJlT@N1w!4WZ?)4sZB%c*cASrVZJAw_(0wV-+q;~ zILT(aX?@*aHbNPFu)9zj`zahn!g)gDn$aDL3d}(=bA0rJ@3yB1$#(HPJ1ZMA75lz$ zD=nK1Z|1l!RNFf%y$041o;>v6TkvOrEbXdZlyh2S3pcWU0r`V!vUxLj%Ye9#T}n5* z@rz$|+i8aeB#^raU~v)hLhxOD`-l6z`q5r9GCa4eVVJP|#cW{YH8ZknriMcMo&R+~ zmZvRO86XIxMeXE*{~zG}ZoK4Ǒ=H=6uGM>Xvl&Q}RM7rRbY0n>~^JJJV%Y##*X zH9h;Zz9asn{I$Tma_ec^T=w<ZU;zhUh`^(0>;t1^A44F%|I=y2^~kJjCUkA@nrZJ50)B% zIDXYLFD~%oo*Eeu+i`SrXULAsK|DqFjLJOlRY2TT;bZq#f-56qm@DjACf`x|d|*J8 zVwrQC&xlKEKZ@&I#9-@>*91h~%zi+kQk}=4&!c5|rR!S}eA-!D#}mYRHux%dd6nlY zjTx~HxP}bt-d9+)^E*^bA(kDPw|~Z0&3Bh?GSe)wRg~+~OH!kN$B=^!6Q}SA zvdjaGg7C&f28#=7MN#3CsYDj=fRoMAPS!b{J@Rny}%Q!u6(uhjx;g`17h5MMlp)FYyG*_FXf{F{4$HDzKUclWv zsZHP7gxZj;G;}U2{xvz3&DhnCT=arTlm##?Umm;h8 zBZ1N<_R7)F+j|4zT-m}ngbN6R=P1{_SrjSHI5O;lg|LY4n2e(d#BLl3xV3zX88gKh z*VhdD?3^b=QCw^D+fj=d`A#Nn=tfi6rE zmx0qP0+d+*4uxKECwHH>dqZ0m-AtqQ!{h-g=i3*Yvrr1+vB=lQYA?+0%@ zE5<^A^sHrXjx5W`3N`oL#W2HRHf|xQh?VsxtMWnjkQ1+x zXNOoB6*x3dYfRa{bk(ncNm{XbWs>5Hg!0QCcY__A{H<4J2ajUeUl<7|aUdkD@h{~r zI`k@QloRgj`(4c6sGbMRs||DI8GJFaf4d@DvB)RvMAhF6G6FW)(_ z!Hli9!#Y-p0$|H3s{QTJb@zW877PXLOOIxFgM_&S`n_g`m+jgYR1;EkL;jX%wcHW! z86VfxzDVcxbX;kM8R3C|Y105cBHTNq{eNDe!b+1jWrTEA@=6+dS0FL|HF~&t<=*z$?I;#+* zgm|MAs<&TM*92QHHLm`*Dqa632vndHSsA#+6e>Y@)HR@M`Yo=^>lwBzTl`!0D34@) zaDBW{;@GMxO(Y*yn!7_)sSWm1TaL90+F10*++O2Nx>D7suVwj9o=#AUMMS69>K5+Cdj<@Kn*X1sY?`eL>u*ORs z;`t6x(72Z2aOhwP#uc}KQOM_PZpGC_2NCxU0Ya#@Ij=R)WUxdt9Dm0I6X1Y_F!&(R z#iZY!7(-zM7&AK{EKlw2$Om0;Gmu94nFMqTVQsC*$H zU%_qRfntd{1e+1#aQ4gNb#guX^wxN)%NX4FDQailMCi2IjuG_-r z3hYtm4t|7J$zufG)u@yf5hC7Il+oy-RU@v7TSmLeJ?aNl80|}{bLtG|*ECP&X4uyJ zScpJ{A%`Mskhz-v>9zT!lEJ3&)9rq)IN9(_`Rb-c;|!n=X%(Sq=EgiG`HOfJnOUVA zig*v@Ql6h102Zwy(z$XD}58k{$_s|36$Z0<01R0yPog!F_ z)PJihQO`KWKeGolJodK~u~=KZVIi^=H5q3kar#a)Iq!~HE>x%NcB3H}R(XVJE`bKR z+h~qII)_9DlEpT2m=6O-_4oLzwCWDtn*8(c@e#0mzV$<1!Zdte#b{<1j1qWRM?&6k z6^FSqw9D*p(TA7&zQ%XXNaXFFU)E2V%e*^YUA#(=QCE2!@qk7&wfeas-#I=sjw1xp z$N6pGqT^`cyK>Qil+%@)H~ojKUDDM7NseD$e~)uSseY4>XM(1o;+=D02F8*|9CLey z*9r(pI<$2vDpU4%kUvUFG6EfXcPb%K?BEn8;JSA>s(vu0LY?uUSd5reUbJT^d{OmB zCIBc8Y$Fv0r%0c2$i_K+ZkeRP?+cpz?!Xp*uoc2-QEW3B@_mitJ^B=_Hz=7m$y|IG zLU{bbi2z;i2$x|7gZb7h5eVp!pQ@6?RjFVE&+q?BFXi|x=)oYOvGUjkD zXW%ej4P-`J9~q?izGY2idFPPp2ViPe-C(sob?8ZfOY*`G3k+6?I;{a zTQx&rM}$_#ziPV=<(spLUc_gy-0<4Pt|tPg7i{rGa~TpU1)odcwm4P}y6D39Gfxt$ zXiwQK5(ws%S(kgvgS%4-2e7kJtVH~~+^IX*yeaS2y}Bu?1;moG+vFLdOar;`AUfzv zpN~*1;RmlGX!h4U1<8QF7wMf&iVRk$=YwC^SSwTh{Q5``w)gJv8J(sor10WP^Qtla zn4Mf4j5YXd9!~{ongYV?%T&B&p5QTyPaf*lMw@<21B3RfWmU@oA-w)B4xxRzSVHVv zPbdDP1nr9t&XS`c5#|}J+@hStEy}gsI^c;F#zu7y2a`~L>)#Ka%wM8*q=vPy%3~oD z2{yD_Z5lGDG%tj6?L&7m^9~a<`?Y^w_=%+Wvz*J*p9%|&<@mp=?3rQr=V+vUA)Ii= zJh|Uh8YGCT-n$95K5gMl{7Yb_TZG$tOI-w1rh$^lPU6D>4>aVtzuHgaPoiY0h`XrQ zNkA>StE%U`r%qu5Xie=_6o6@|pEt)7R1tguJ%so%fE9yh`%aAm3b`3d>~yjJ;S-_-)7mJO0HE%CUCyC3|OSGQ%vk>IyXMF7)RNsm~^(6OyVA(scC2`r)=k8FgG&sySUj!&&hV{D*xXk72V|w@cnnt=8Z*^2 zuyDRKSNtQQU=Uu9C^OCtBo=P%_@$ zpi`;np8udV?{JQgJbAZZ2G{p>0N7x zZGo4YS&lDOj&No?sOpbN7rMU4GGeZKpE@;4=Q_`f0E|FS**|3JJf zY;6B$65I-}lTwraAl^5OyU%#R|NxMuX?Q2*iyJe zo(>ao1h8p7Iezc25JLEo<6-nb&*vjq;I&_p@pfm~=k0Cd<53P=@S6SKZkEp-gO?$~ zm9GWTK!}{hQC~c^%q0<))P{lM9@Eo~C-=ZeP`3zfME7H-W6HHfe-pugC~W+1xo8SL z_}c_%x##%q-N)e?Pk?v$NBw@Dq#QNT^VTobcS=~Lh_3ke9{seJFGSJqe=@4(`MHwy zuh)Bvg4OBP1M;}&TC{*L$pRz;PLY_{kwskPErdqSC`tls{rXX&%=7KEKlDf@c^CF& zI}D%mH{%)#Nv2*Ch%_$(Hm;aW#SQOgyd{S;6JY9XbCe?!Ha_DNoPh z-!glP-_DdW{H7X^IrL%fu9hNH!pD!1o)a zlaTu22F)6-Ul?lRHFH%VqzyeGhIJYE$cfw3g+7khxZ%CkIcQ(<_N>Pp5su=8ZGLFw zJ00d17aHe_u&NEH?J=Dxc`=c*nP^rlk1H`l3T0VP8Zpip?^3^d!3?$hxQSd0VK5~R z>vMA0zUkL3MX3vVH~A9|_tYl%*k)I8L+MQkQaz;;Kt?DO!Fc1ecAe2IPP z)Vz=TZn?edqs_=mUi5P z0{qN~{^O2*r?-}xa^(A{XC^m84EdglH5G|Mtwk;Rn(9Gc$`@udw?V=Dr;tf;>Q)3VkOlfNgxE&b9 zimqcvZnpQF&>nU5ws)w0Arxd8rf?VN){OXFn<_o_W4yaJkH;VoZ)@0|Qea@4_KTa- z)n9Qpy6@|n;Q2II&6$`o>JXA{tRB^2S{rX*P$M3{9=K9j6}@^d}Yrq|KNB1Tw%#pzr-O= z3~b+^li?3p*1}9}S_beqe*+y|9q;~T4ZqmG?b>xY!gvB3gzg@faj6!V4BBM(d*oHj zw8;Vj^v0AiSk}l=SNDuFQ4{fN)}`uQ^hqnWiZ046*LUqcH0KPdSKk{v+zKSJRLuv$ zTSaQ&tg6mM1IBSuiq8)3v+~n;WF2W`NvuUOM!&EHSu}Wn<*$0WZNFg2+GMV0hI9B_ zN*)@iW0vtE3ip1eKN8>`BnE;l1iyL{#~;r6?*aAu{7xC6AN+>5k5gP!qrVb*gZ2%m z20f7a?Zl4gh9Pqrc)`@b#IWamNY{Syhkc&}JQcvK8ytsn0L0 z%Jv7s2>l|INZ>pvi5(SM*#ve8;|U&MWX8}?i@0>|lRF&j{VqBKEj~wYW+FkO{Zg_J z84{D-?+D%fBZA5?ja@eF(Xcq~K7i~^#Xe5Z9icw%9&BxoZJb=@tuLQKXpSso+p-UWyEKPsZSMtT$5q?k5iVhQb{42Kw@w8B1#L| z=V*PzT!qx-%|?6AoRF1j&?$r=3NDV)@{S7G2>w@{i^V;j3eMP+o9iJ%*hvylyP+E= z{2ukKyrGNU=Y%@lDAqhsT}CCD1aFRi1wiUy$=sv{W(^_B)t`j+$t91^I6gF{nfe(b zLI&frxkYG50h;vsg?jEFIDtYyi16Eb5SCi~7F}c&jz>fh*f?M37O1#Q&GR~+@;WA; zQqnyfcIBvRi#>4_Q!-`5DhrDHMs@X-@SJLEe=R#HxJ^042Y@T@Z>(Jkz~<+Z#~Ejf zHy#q@W}|5$mC8Fh0vCRD?A902|Lm9URyyjeP^=f+8E5w_KU~KCbS7-W^!nrFK8Fz7 z;DhX|WpflBp?7?QmO`-j;s=zG0TC}#&xF4ZMvgFY2k9$Yu=u0=&>6QylDncvwA>(m zIFOWkLX+V@y(dJ7K?Rj!Jm_`Mzqs75C@%G&Rr30b5J%eYmcnepad53|;zvK!Lm?7K zB2K6lGp^AI3UODv={xM2e)Ae(j6`C(B>0Tzs1BX9a(~TF{1I0oYKLjtGHU>rKf&+o zLm__f1Zw3h;Lp6`7fx^m;SJquhX=WqjoV^r^l72PxXlV)-(qppUhxRGgmer(LBld&1bjEzvh@zccVbH#YL6mOwQ5IOJyDc#+|z9zh$;-Ag zXO!C?D|>%7Wme}TJo;@=ZayrE;%%~YRBood$-=s-m!nj2Yn`RN>&>vBxe&;uQ(4ZS z*V}K%G>x2B6TD(wt=onlQK!rBu!4=4lMToN(80ad>o0%9`2&$Dgw;y?Y7JIjr1}UNH`R{dK5xf63F%q_fj<9TvP&@NC3$= z!Mj%$D8I9HIk&UHL{hf_K%o0psFz2PpV7m4BFPnKlQVv>i0hM=$5yBhw-416>h;xL z5ZV1vCTwu~{PW1S4_MBD-6snqUpYqn8oARDG`=AsJzY+20cuwJQ^D|kydB@(t2!2U zU>funl5|F3-G7a|5*yzH*#$wWaH3ALi-eA<_6ku2=Fos^?}CM8orsq{sifVVfG>_xkrpZN1MHq_kSIYS7!dN36|7A z)7~PisK=t8w;N9uG1-Ah?pslBEKCrtp ziYR_M!-G6g^daS=D;+1a|4sc4ZSc9wgBMKNJo zSWyfW02}-Ret-`9*J7zNrXDts%}3O3mJFVwmnHtshL%L<)w{N{#&2n(@uA2OPm19~ z{NMsY6HS+>!CxNCf9`Oo72PMkfxv&9y_E>+y2#EA^YS%T`zg|hqC$!&O*&~CW>^l+PRf#40q^q-)giIgGc)7iJ6ZuCk!4PohdKj8a>#CieVn-%*W)NN5umsG?65=ROgS_ue`*p6 zhU!8FFqNIF+Tg)WfZ&5foT$$6#C8|s|4mKHH={-l_4>&p^}j`u1JTM& z1s?Ya7&q-$VaWj2Ln+R}?ngIjTPqn%{`*o!iMBH`nv`zYV-~ioAv2C1`W@c{H=<3R zI_y|PFTZA{r^EKPv>2>s;;A^Y9|TwRV1dUk#>Vzm$4Z_IF7^#FDc@pT$=HeQNM-{T zeMtvl_wF0i)15ef*Kk;sk}Xwtz}Ug_dccV`#(rQxI0B5Qg92E;=sVTrkwG*R9O@tZ zN%lHJX$~ah06Imwju!y*b((UmTj2H99ZZXK0myU^>ZI_9 zl%d#RW2qXRRoS11dK{3%++2S08hF^0{w+N1Q?Fglso?hMZqaSfJRa@w>isq>U=g%? zm@xjJZE4GtJAKtU3)D9r?5(FEfFXPfj2x`q5FojcBaB9cdcJz>Y9owZsV^SPOW}TT zd#$i&i5h&JG20nv@?f_;smKK3g+g|ij+crjZUF4Zq|X^3)W`#`FQ38nPN6}(cwx@4 zWO%1rujLfOVf8DAD;6;_QH#9cfxOn@n%d&A9is|aSom59-sqC|PJ|qxuzbdH6Lmvf z!uR!~XDO!MIsU8D-@-YbdXTpz$TtqRW>y`*yQcJ5BR8J|z9(xDQp(=Ur^;C*dn4o;-#m>QGARepT%7>O?48a*+b_#3iY^aHDctFMH8rq)?QwxB$s z9mL2#VSMg{(kplSLshfU@1TQtRsS)Fe2Mwyvy7h7?D{w&lb5W5C&zi?E%n&00c2(Y z;mZ%qMLyH;S}({|>Q=s+?l*Qov*dc?qaN_?L`6<1oeEPPa0pI!OS^sj7%49y_;?W< z4NizPY~W2Aht3POqEDgKe(5UST1E!pRzgy^+MmJpRsn_rQvC=6oL2{35{_+t`Z((- zpk8;{MUG>>33{Fuc2^JDrnq{ZZfQ0RcJQ&x5WZxlww%^*u?bkm@y+yFRY^D`DST>{ zgtg;G7XRf9<)329d7s4O_sztVspre}6?6QfvKvC)n@ex++B2GO)BEt5^+2hPupCd?J})v`g^KB|uueG$oTwm2S4MX6)#6Z*TdBC$qSc*rR>mFZ<{ zxb4UGv8d0teIGc@ljE0VHb0{(~U%P zxw6@V51;W~5D4+w;dYns^8FNqo43A$;zvta zW*7LH>W`6-c{3v^QFq7T#X+}HDAv%|VqbT}=rmuj zD{z1U1|oH4eO6p!SQgIY?Ps}F6*q^|(ff*~4N&8z8D*I`>^s^v`F>V|I0>uzZurh3 z3om{L6$)DWZp|=6F9_PWx5O+)ebkLLZqmAcA>>g?K26>&6(C;>O22zWt+=z0tXzO!`b1r0T4m%lIV^lW;)IqRFBfz4y zA@Z?H>|;AH_!@iv_O-g#e&)M7I@IdKzQ-hujK*~7lPEL0Z=)u|aU6o0exo)ktbYkR zG~Ba#P8EA7+=(1qz zd~A0fb2gMS3M-x++n{d;_8~A9G_{Gi8(_MGO<$Ru?6e+Vb+SjUb5*$6r#?MR8DWw( zPjew>S=gaV9V>bO8L(%zm&lQ1Njy1HVOv>K@sBzomGkHlAX_(dxwM+2etKEqj=AC= zsZDbpFgTjf(!cxthp${&(P;d|YvY*hpL-T(Vle->tS?b6x?Sn_c>k9w5(6dtAEOpf0cY%1W5l`f<~E z*o(Oe3T

60T{U6Tlg?C=PMUPDdFTobF z;tSCfKc;(V5VmS6I8u@i=n8G#x(-R+ki}Rp5l&{;^+k5*oBo`EvZ_v{v29TwAI?U( z#s$pu`B@K}rclRDY;M_Ej&17IVP4pun@sm>=?`YD);K(u3SFzlN18>0euZY6A5$#tCaDZL$C8loX3`RSzCe|3xPtzoF+A zXt!TxW8{E{?h?oYqoA4MkQxljSI=V_S0*TTbwZtjEVet4m|9g>mQ5-C8|#%)TtDFZ z4+;$@B(Zgg-dwW+q(-Bx> zZz|nvhMYbcS(2lUWe4`=I!bCC%6PD;K35hsGL&c4dxJgZ@`$0+uD92SoJ6N+4!T!K zs+x8X8O)xv=wQINecKg^L=4FgWgjnyI?W=sXo)AC|4^?&WIjgX!XQj$`4J;ATL;n0 z)!wD8vE(ko4mVU-KOu|FzLTsv^$h92BQqt)D&DZ`G-Je#yuf{y4a#Q_b<~L@cOlRH zAI9D?Dvl?38%;ux-~?N2ad)=`5H#jN6D~TP9H*g=bAMN%5d`#;aiX zpPSCuIXA=9kGx!mD2kqL4-!6Oy<6OlEwW<)I#4Bw+MqOb^kxTHX_Ur(9(Tm_2pLEZ zgz-ic?QQlEWzD1S`HN9f=*hbLG8r%gKWt&11HfOW`;EYwNHtJj`>&}BF0;pW9Nk)N zN4JfOIIp!DOYxjDdf^t;m}2EkwsF(XMYQwfda4HuPWh(lgyCDVuGaQ*dE;_zxX~vm zk*Ks5Y#N^{34%8>EBmE>9?$_F)=7h8Dq_YY_OlxZ5D!5ObT(O&N3BoS3tRUczbT6y zw{Uj-iZjcLUMX6W<7au75szBi;(Vh?v9`Qcx7 zn+0xw_lI57UZ`DwEsnefyLNOuGAw&e!H8xC2WIpJXnO(q_)`mpw65bmKCzb}SnNLK zdg#O2o{T1ghR+ZZbyk8+a>u_eM_5_QEQVpe7HG}K4al)~aR$2R3VGRXyY51wzKV82 zJA=@Q!5j?1rmPA&<`B6n6Yt_FZtkzzJd^0d{?H3NycB|MP6|D*AS+HF!qjhz6arfE zxS{A&$-4p)t&u*>igICHh{rx~xqYjQQO*GO=fT$d7>B zFRs>E>5`dc`*bRqnJQ$$4jS-Cd&#^$^Aen6A*k3Q#JE(B_2Y@;y0;2*|rnb z>j|<_Y3IK*s^o4EE3$zSpJy!rYhHt=V?I|H+3Xb11T@eKg2QJk*J?AKIhM;Gbv6VqA8IY*# z%?+yrRLi!O%tR$^yEfL$Ko7+>YLY&C+Zs>4$|dE&A-92_g!{*gz!&*oKN<69=$W5t z%_Q}i!&kFROwbiHr=H2s^{vE$g?iUPcZPJQI#=`{&s9Sl;|R)8&*eb1XmEaBI&@$t zt5JV)3ctG=CIGar?!qRiQao&ro>0DrZjmX!(!d#8KWRk|?s<{@vN&@R`R}JaZW2Zp zYE^S^>xbLY0#3sT-biSOr?i4`z39w6g}wjYu)-6N1(T=AMbV)2r?yk!J9c2=%ItTs zx~HPFCyNyZwSCr3M;$zG0kf=onI1cp)73Os9QLyjxF_=CW#afNTekV{*IA_wO8aTq zrkRORSjU9lM?i#k8b0mOYzEMI-TGwGG@ONmA2`LkaFnQf<2;sCa^aomo)&HN=QO{3_vthJZsF%Do2jD55*seVvsSvA0(ZS> z2gPycPjfFUSTu9wLn1!1o4G}!$Sp?gvJhd@#QyhGS|oKs))LT<+R{r_1Dj^*e1Lj5 z@+mJY#U2N&U*$(6(rp(qy7Uo8861svg%R`v;jmSbUvudAqmKl&K9l0RNJpllL?DCv z!67)}<`bkMY2K`zli=(%oWGwqD_DMCGf%=;kCo`=kcjM~j8=-$^??9y*r!|8FyRd? zLh}gx%2p0NokH;K&3uE0IO0?B5s%^2EOmE|paB`7D@OE#^EO9b>EY3m*$Cyg`9A8& zzBDQ~){fF!Sa52zumf09WNNlSzDTgG%qCgsx>$zT$pryxYi2t8yA|+O_7;RxfF&V{ z0Hc1vo#|2h`HdXS>}X9?iOYKHss&j}rfEi6*c&NFdIk!?9H!2G$=l42`(I zNYvl195=9*^~sqlz)esueT@8GxneZJ{=IvqX?>0_knLlzqN`dKEP$|61)b9e7ox>>j0Ry)8)o8 z{MZs%yj{>aMe!%uflI^s%^9mN#g~i2 zxcbCp8e`kqH*wUooo1Qfd<}I_aAG>|99hIP{(|KO^phkT?p0weBjJbI>9G%Zhf4Om z&TEZHmKi4k#Id=7wngT0z0)1Al(gs5hDhVH@9U?+=zZsRWF#6?R;6%(nx@CBfgMMl zAaa_usl#dc%plMV{N6w|yNWuwE&nV{+9PE|I^;K+bMq1OMv=R;Ub?3mivlWw!%u<=csM+3E_Kmi>6BE8OfUYPWzz+sDZ|bES=A~oZ5sbdfAgg<=PDT7 z(dUi;(4-)bAP2Ll#amz%^f}YDa2kY)CL?c_xpz|a$)AnHl3oPijgZIzKZg#}v3m~# z3;Id>?%vA4U$NN~^o8*au-c9&)vKc`mUQGYkxr@^q09-N)W zi1cj?X_{Kyj!8a%`oxKeo)&;aB+sB)r$LVdy=iYl*%2_w-aN9z1~I+4$uQ=c}&H5VO=FxA8+;2j@Ul$N(J@G@kV(9R7w6j`UAhsY$188Chz z5qTN0(vb77M0d(6w~@~ThV=2cs`UyxQxO3-@1BnF+4U}?GGs-^sT^vDA93s-+-S;d ze@}k6$^bek6bhdeg(1C@;yx=<1ufM*z4-Tmd6)hbIzRrL1qxt#J=h~ez0EvhV|w}O z>Uj(FF>Ee7@zG`ECy9{VzL4)E2ROw9*H-RgfLz}ww|E;iQ)~?R4BZFG>_BE_yOwh3 zgAi zPNcxv+~<^UPJR%4T!a?z9G1z{(DX7PnM)-Iu-dZ0XDZ~0RMQ*FLb+M$nv+!rrYv|R7tkbS!-Q*0fI>QOc z;s(RosCORym+g4XKLrUMHN9E1NVh_(OxXz6c*j$%FjO{_Om4Hvce_n^>ld*Y6QnM9 z2q3YDXY~b~iza4Q?3rS;S^YMs6;`#pXhLLhI7ZP@QJSNZg)M=~Sbuw9br}Z%+h9qU zQ5^ZaeDz#j*T1My9_p_f`WYqQaQa4T9xe8ryn9zBg2#5+P)%X{a6ov1Ch`1M>!`o} za6`IRvMV%r#AjMk#?^Xa5?TTOJ90m~r94ygSnAAOB8unPIk}&nz01$4UvIBGv@TOEuwiXdwyO?(SY=^8 zwRTbIIZs+H;@j+5mL5KxKJ>E_8fYC`7@xii zINLvC{zUTR6{~ml+Opab8NOj+#PJU6uwcxfyV07;DvgQ_TTBkNL5zl^e?Ps`l zO0NQfo+hs)m5(%9ohCh;ZO1Zs(4pTsia;%%84wI@PdsjNJ$i4H8gG9YISG5^j8+2B zkLZ{uldw*@rQw#FAyJ=aT9uEv;{8!7y7u3WJrAGq+EtDI@-xD;ftQ4)!eay@&Y?-k z!RcSKTPXnv+cFqiJXw5(W5@L4wSb#7we@$7_n6d6oom2!R0N0eTf;0tOG5HOnY<ngaYl7>D z_=ek_Y2;1Xs1@76-^`N^wF$Q=VO1_vuRlA6S${9`23Gki8vKz5QR^yuA-wr_LRudU zq;u+0=m|kI`-_8YAR3-nLE5#SleP^v`~-IAMS2q;4yC!%p#u5|1}u_Dd6IfAn=tUmuB>R7)WboQUMt1qMWGP1Qm}x-@mpla86ge$9C|dBb23 z1r+s2XUWX_!-WhEV;0Fg^J=NY?wBR~m`P9as7T>?`GB%8fV_~HRU$QUyO>h=*pD^@ z#-p^Lq`CBMlF_Z#5aF1-sfatg>r5ga76Cryn7pgNy*_vrEk$f3dS6(D*c$$LXnSze z1Fu*q3{15|mqscdV7T9RPsW;?dKobcX9@i-9fw5;?i&XD?NV8o!}r>`)>*tZmMMnE(j)}Hc1j%P;#5JW7NMVstio^A*8P7)k*I};Q~yq0&Hl~ zq{mE5wQ%TIiueQT_6G-(mY6NEmtrz$n#tM2Xe|>c{bf;5++XR8&omH^2O1dp@?{7` zD51#*F;qXi_xUVqI5bZC_Q`H71sc#^v4u zCbjdrIcYEPQCL1;T0t6^sR!SNqJyXf0J&St0zjNlH;wEWm-{?Qydxm?rO}%PVO&-l z{@MS@_ffX*CFjBCK{iNT>atCkR)UW?NPesCH;=P(_WS$(uPK7NkJbU4BGlxp*}^iS zrQz)r>MXxwH^aC9snO~d-H8o&7qozg8~y<{kGr!xh_NmVOXot%1O@p?aV{=Z<1> zf4m%aZ@p&!kq3`Zr!)wbuW7cNBwSUA1!iptV4JD`WmNe&c-5`yTBHAU6DanmG z+nd5@0R-hqw>gWoiqUFuSjRv;$ACspG0Hl2iuwG$%(rP~daw%wt}o z32r<<8Mo!wqD@3kA;1JtSq$mvOHK5U?Q&I0+0cGvjVD5B!}^swULC@vSFbXBmLjHw zg6mjM=6wyK2PFo|JVcm%Qc{z_92&cquPVRn$$|=WS`yA@jb!3m5k704Am#`n1o{jbm9i8yiT!m)dNO0-(a8 zb{gGc4u)S`CR$B0s{&TEDE^HHxl|AnLzIUny$PXYq0#NS4Swa}#)%3LVJtPfuhF8h{Ns3^=VD zfvW?`Uj__KI}`rRXr_JP&cyYD4g{3QT6_zeQyQZoF=jE^ZBw=Y!vlVex`>RUKHeC& zw^OL`ZG2#1mrYayzw?7#kT@B<&uS_!1?CO@@OyD}IIH>?S_M!Plyvn6U+-y#g4ruH zkB}!nL|euskiQm+{Rtf;3@Xvj$htr|T`4Xj9y{ zrm--IvfMCkZzddJNtA9LG=&L^;bl@;B3(jTnM);SN?L-^zGdnX&D`#xCHqI>9=1!5J^)GoLUrHLDI7?N21Rx4opORgNB~B zEe>`-zJ2jp1EcQsD&=PYxs=m`1=3}|;N?@hzi!I>P32jT-$>ygb1UgOcZm#+B>Hl4D-LO8m=B#K`s5>%*2M-ea z`&li5Hrug?HnVUH68tAlR@jq=aMegMyH&_wysmV%u^n9)0QP)cl?e%5ZgqQMnQ9^06R&ga_?(?%|z?&d_UE&@pmXmoDgYT zh9YN=N%(dJ8^ptvfcV^B3p}7oriM;1TCJdr4Ha6dp>HsrNLX^LXDPAT4`;w`i+>Tpzdfz^P8z4BA2t`8uN(`(zOQWG$QcOrfco)PCKSb zXlM-McNi+>d7Twijz6`TsyAv&DKkgz8R~nwfmMdNxbE45UC_J0@L%39L~dY1fh<{? z@nVRQavmd#LQP08-fNL(A-IJ(>6zty&0cVM|M&3<7HU*L*)rVK?)MLCAJy<2Y48$G zz@1*2J8pzNM#{&#$d=V0?%bOvh;F3)BNbstTY^@WxGJkP{BtxER8=(a2%LXHc7IiS%ni}bj^amXLlFPH-`qCIw0@Z_PmTCg~Z;{*mb&FloJBi zhb?ECU?JME!-u>N-NNaXTt<_S{iv_gW+xu)o&3OPZdWuaJ~BnR@)~J|DtDJGoFWoW zpC`?OT|MS$x4dNjQRS1>IE)_Kcm32)b$y9d#nTb|Ptph8eP{epFA#*M%y7U1zU-Ef}vebcjDo_34E>_<|}^T+SkpquGR+00JXZxWUx>Lw+) zI&h|D7dz72L6yr&RFqJ4G(U$XNHubh{~@I6+eoe~7qRE{9D1I5kH(MWJaw99^Q<93 zSgHkwx7P-d3jk=%xN~c+QLYzO9d*8$z@fq-ZqjQj2MCiYMcdaLf%P$8<@0AzjxxL- z9PIu!aOf66td)jo(3!UXacJIxZVTx;aTmBAVCu&A_s^aQcAV4JO4g;~tT)Jo&%|b^ zmSs4OW$EDC$LiPp zpNswMe@`}lcmMo-Z4!H(eSNzBN5{h$a8vepHQc8A>*ySTzjH<)_MaYtJg35d{>rOr zu-~46WzPD)yXEY^j(@!W-ZuUDx3|uH`SwrL&867SXY#T88!>qFLowgy;dP#i1U^T? zqXwVT_l&G=C@rOG_iD%z;gt^8O zkoN!XAHa``t=^#u9n-&knCODVeY7&_h5$`8T)zZRJB`fNAgjm?{$yDL(xYz?Zv zJ-!$H|8OCLG(DB1Hb+#!l)yPtrH%>bUM$tB6!pWs85_ZucC5;mdPKz3sq)!(FnQXpGiBM$DJE9JchSblL4yx8e7l=aqj z7=#C~@k(5&UR zNdojG5Zc3X`H~9bnOZ%YNgM$UxY8s7l{#R2N@G0gBZX^&1*-4J==wFIpx?KYs|vRU z3pjMSYHmK!f8uMBbgDnNUGeoMAx!=dq%tC)391l+TxP*0tW#S%-Yd&^iI@hbI|n8} zLj#$iaRL?s=W=2| zEsxubMV;%|W?*l{2&{OmKC6OLJ1JgmN*!kl&MuWwcIHv9B=`|D3!M7G)nxyL2L!>n zqmj%{P~G{r!LcK81FW#3{YXy!8XCNu5lh;T)-G5j+4ogQExZfi(rQvSEaA`*omNvm z4Ry4m|LK>E$ly00Dg&_xX~QzM3wET&wGDN{GV9tp#KzJ_hsG}TSwJ&lqbtb6@Hl7^ zK}T(xNC*Bnyc4mJEv3s6xUXZ~-@Zd6+kF!3@|`!6^PP?=|2oH(w?qJS7SFZIulEfN zlmSdw5`Bwq-60!4!;84iJaywvXqPmD7yx(a=k_82A8S|2NLa347G9pn!6bm|);{tB3#f}&OD22l_EP#_|x%ja=!19gVFhUUdFQVKETgN( zeZy#}r{!n;a#}Jd4IBYORc-a;$rf58+kd}>l_rIU5=)irPkTlEjR(OH9K3&OKr6CESLn} z);pzKwiEEyH$Nv1Q|t1ux~iq19h#5Z;wW(u+`PwmODa^W=eU=a9Mokg%DBfZ6GPAK z>5=UZI`vZ_kC?9{PDsr-<`K6gM8DSrWqL+J2R?nJ?u^Qjn{VFRJW+<-1U*7vF-B=# zoc-3O3ir7S_JBs!gZ1^#_&zKEC^uk=cQHfyf-@lkKej@^gj;WU1q$mV6L*6-g5L6F z@s~hZ(TQS^iE)sqoc9p3hCVxs8Su84>~ZxVX#CT4wQoosrkH4vpts2Cn-IZkYCJaM zDCvf@>4~cE5eTBwAyI3hpn0*|b2XT{(_x;-fy8kulEo)e4hhT-LyM<*_)i zsko#Y6ng(Vr;sAn?$lRsp>HbP*gJEbtv8QKL#Uh3yd~*RTSCPzo5*{uWe zdQHIW%@J=!$r%3h5Py5O{_D))V4%V*PL#{}8WZ-4eU2s=*(~|TsJXBJmI12Xzyf9P zng&Q>cW!U$zp1JEZb@dsGNr&qYS~laY+dE;AA)9bWhZysKk|EhOXFyhF9Bt3y4;c9 z4qb{*7I>fA_jNShI+{LSYopua9+jHFmZPW%-+T58W@Y&gxXg&sUF;m=))VSLNqvTp zrDV;R*EoIG^V0%R$V~XcdnVgdRW9pbEE4E8BZbI;+`PAbJFuPN#zwT$f_Sf0IR5a7aOOXKo!<~N}1 zE?V@+HP2RT$MSy1HdBSYq|C9}z_U6|0xhO#-MM6DbM4Alh(&b~_9!*7vO%RRpREyi zhxLRT+!fw5cCp#93io1q#4d9RO*0qWxz4D6^(o}{+aE(W=#SAt`> zS}QB@nK}r*$MzEI@nKa>S#^Bl#j29Slyi8sVtM`8BNpr6 zCA(<>qSZPaqz=?(-Yk1TuCv8KEj``*#8IUW@Im_ zAnP5jrf~JZgPfy`#r?mKZZ29!+Yt*z7Q|=D|HA)*l4j?A882aT+wC9`UZCAJmbI$m zhy*0LIF(9+4bts7K!hUA_xAjTPd}OndF^>ogU&)LcrFdcv|bO-%IW_1FIgmm=lc6N_LkgjyAk-k z*apzWg#3Q%%O97)N7tpmoj=3G3#JZh8*=O~C>}?YfmE?LcRLId3+k_wDnF%BnhwJy zeXcU#R;Ajb@1ooD?3kyD<(Wp)+l`T69mg@|#VYFpu}XJIs=k_qtP?mrq6AWPN@pfO zjU0&Hi3uh*Xm*aJ;iDCKTWo9FCXPm#{ck8S^XDw-=l9dKUDQ&wp zH(rG5OC_RsQ#UdMJ=pTH}r~@V!TgKw?4o~*MrfTH{S^)?H%D1FgI;!6k*|qD6 zaNQx4UK5HvnpGK_Wsjy68f-g+Srk-9;&;%9Y~o=luU;2Gr!MOU5ZBP8I_X?a@Xb(h z$=t1+lW(9AYYJ$Z+f6Z~U2cD*BM^0kC}4KJ{XOxZu!{)VPFbTW=XEps7R4>8y!z3{ zLF!m1*maE8;2};;=tqmMV1!L{nud>cii)Sq3@lM%uf|WH4}T2BSbx6jJdYN_6Jc^{ z9L;Su3`qPbjK>e${F+mZ+*D680>iM~;lQE{;*vn;7a<>te&u*&yQtE+h#wWR?l$-i zpnE~Qw(VYTpHu{QVv;L7I9!^iUc?KE*}~ROJT0RofNP8eH`4qJG4}VK^iN6?{ml1*JuV#& z?LusE&P@kjLWraPijzbzH2iJ=yawLzDvDN#O2O5dD4i7!nLx0|>vz&x$+-$$gG{mJ z@{Guze4rKUF(cl;lk+06YqIFl5o(Uy(7R;WUUX*I=j6v3%kq&sihHIdGn`{k01+SS zSGC8ku=~qPL!pX?p*u`=YN`%SRn(i7d}BsufG7b=ajbjJH4&{mYrXL)RADW>sxsZB z$nC!OyO`;2%!j3{u|f>G3fs2`%8_Me(NLiF-H73Ppyv8+y-yM>Co8c<%nJLs{N<+z zx=eLuJF{p^mcOR6S9S)d^GH^c`2BSdPO@+apBWO&r1~wrBlqNl7VnuIKQ=HYN|r;} zH#nBjHL-`}iE^qh!+` zXD@LUqwc=eRB>I|F<{2WoM>W!x`U+O8Dp-}5Ve$uWKU6l#0~|1N6=_wo*!GL!Ue5fz8Qqj`wZ^(s5Z@@)dR1Z~tnQnJSPymgO&oRr17~EXYg~AYgqY@)1l9|DjyK zBWK^=Zu*NnSiNpOFAfkTDecn)6Sk!GFJvwj{-Uyfomo&^^$bt0={>L1{W}y@40hiw z7v^Pt);5nBBj;JPdYlo>J7C;Z+p_JVP=u@UYhreUhDzLK-wlfeXO!Ri@2<=7njs|p zH07whG}9Kr#_;me6v1}$s-+%xN?<-TG1EokbGLW7CbSAmEg=W8-#1%w@#1Abv>r`j zB!aY^inMf!ogb>ApAKon(LV zd9F5^rFTa`m4&~cqt~z`a>a?(#omLHy=Pj9FZ^#*B^nPGLag_>Es(*o_E3;G3qa^- zq^2mv|L%dBw6dy4_FMV>Bm2j{A6FKxUkL4}Wx3Lt>;kyhekzgp>Ez0Oi^xq!E#0^) z3Q+xg8UBHW%lrp5?(;x}DF0PjvJ?JSV4!w%AQ3?q+85Pt;9QoN1; z5blln;NayQRb$N;x>9ZIj|JX z+Bri!W$B5Hupp*!C5tc460nY>Sp9{xFG^^`Hj9O{VP!8j+7W8!KAx;XgO((m9RxnJ zll;bBtU?~|P_KjLjU(_^ZnXJ$R%$*!;v?Yv#fhBJtDo~05Q7m<5gH4K<%`=ua4Kkr ztOe{Hk?(=|;_~abZ|9DSCzdWkJP@|Rob17p4xzPMC4DRDIz(7-%gL~^N>PkHYd7}3 zkGmRx3h|Y(hPD7m={t$a??p5qq#5AZW+RjwyT-H1i=gb)W`opQ z{FZd^?eYksg2Aj|pJwqY$*ds~@+NTB#l>kH+sYxaM#l9Aj7(qP6k*H^&(i1DXL|l9 znP!{Ay)tjM2H$6>Fh3tfDh$(AnpRpFHcyDZ{;$hCw5j9uK zdei)jY|4dgjh8V$f*96J`r{5zf+ax?K!lb@wf*16@;5apQgTl<0ipzTvR^DHza^W; z#&yPqOW+VfjD+0qEICu6MJT`OP%IcOFRO@0*ZQAHFLtR6PRP7tewKDDZ{2+1VM4nI zP2_q*^gqaT3d-T5Byy(~ZniL`KX*QhUmOJ_}-ArXwDT zuHjHg3UYXj9#N6$i``l#SHg!umI3VKKG;7(C zA-oVt+E8u{`|mb~0b^M!^iM{DAmJ;$3zq}cO~wex9w)tvNp@O1++diN{`yCGs$gn7 zzgT))i<1nz0d+liQdsrh%hp=6d`a5szvYkg)PFnNm6gk_3wUTH!a1)}>>g3|;ZoN` zr(`uE4h^luO7%1N0HU?k&B}Y!)Xlnd&D7vdMQfS#?%^epL2Io<(HS*$+QtnccYv8% zE6ci>nhuZO%@cOlNxb&vOEm`f* zAKu#`+-SVCC$}CEKcg%Ee_!&Y&BA*MT!lAWyk@5Mlm<_bM&YCC)Q1nZQ~$q`F1%Eu z3ordo!dX#PK9sW$pVo0Ayjy=Q_^Wj7L;t%U8((&}`&!@+;RBovf)kKy&o@!+L+FCx|6PQ|^I9gKI{1nk*E#Gx zwv}7wYyDTvEm6(Rqy$a}5%?xTg720?8!!00kKrp#bOWc_Q~OyhQ>GaF;eX9e&A-Bz zq7P1*n+_LvHLjoVVSs1g6byzhYNa>4F4;JIJ*y7k2^_xhzv{Y*!n=I}whxy^wEVZ8 zznkH6Y4~r`fZ;0$5rMzzx6f3zbm9HG{7A=Qmuuem`Lj-iRD%`XS{&#N7%=;XdzZ1r#VqgPil69VUy=If0|+O#VEb=PS}Dsl@w+mRf8sp>-$m@ z{>KPH*;jCm9^swUepdQ}hY?!;?Zh)&y4Ru~L3_Pk3y@!EIj|VLKq0|M{kyFKXM=X; zpDjp$e@$l?=#0XGmuHHRHc%y86=bl^rWCwTQKhEhM;tiZI_jz?M}#xyLF-x*Qq)w{ zf@>f620mv_f-RsF$aXc?B^W(3!g;T~GsD!g_9^GB#f6hsO9|DI9|@GO7tuSaEIe;< zDI&RIait|Ua@nO_^$2GsIWo65`xqm3RZtksE5)YuI?E1{5Wn^P^PNCp)D91NM)HE- zlsuf19FRz#!>BI~kWE2BuTO8ak}~fg7CTabPSW}u;})iq+U&L#H2dqp+*dm7S{6H| zV_4+zLO;$-uChqPO`o_xwX%Gb-A{IBFDzt>UC<1II6pz+OG}hEzy2CmSO|$84FbJ9 zb&e$8h)Id%DpzCvanPpLamW@2fank!i>o*&D~kjiM>!;wEa=}=FHZ!-W;uD5*9lTG zyMmnlKrd>r4~;metlu4`%9My!HQT4${$3bk%iOu?H2T}>42-7sm491Nhc^D^-_GRk zq<8i*$!j!QqOIsD9#`L0hp#JlN-!eRK=7a3pehNv$GlbeSOJ>J5u(Eym3DpX%OF_= z@}q&6UBCa|1EU`}Mx>1%Itl$S_467h@p)qne5AU zaZu)U>FKv5?>m2?^{>Ur@MYqnOJ~qxKB_Uqf6Q2m(k31brXO53Tx(G!0XmUZtuXp% zPcNS=k}~CGzF9tMl_^!{#&U4M^RnmI>ggL?hxG1UiV{LRy|crkX`@c~!sQ}o#bc#K zo3dF>@~mz(749hhl#)J6^#H!k_f9&_V<@u14Tr1bycz8EH|3)OHu)OyzI-xC#^5@lUN}fb-S^gw_V+X>Yi|n-?p8%%|7Ng8idKPf zWn~g#L&ei^P_z|0FI=Ak$L0I)1=uhyo_Jt6G}z3|LEl(H(k4De|B50;-re|Nj`OE-7RbBlBV=>-C-PYo$J-@fG0 zfWz`>iS7Rij0Dr%$938aD{h0PonG=&WkulOV6I5Z-P#)7hVT@0a`D>(7D+g!zELt& zZ{m!;waUT4hxR&io3?$IG*(laizX2v3D9VJeit58fV|Z;LOy`xWw`NC zdpw(G%z^3M`1}W_-5-bHVU;f2>%;zs!(VaH95fFu+tAYpp0*~&(+>D4V>1;!l&1Ok zRoY_4GAS{RDcplNVA!JocFkjiRVOx+a^|t$@;7#Hr-|U|a*Q(R~?<|bv&_%iUr#T1L`3G}v zp81>Bfyrd!n*7`xKX z0KK*Twx)dc!pe--W*La47ryT`s{VxPLLBz_f+ z-Zl3SX&sU}IO^_z%FaD(XOn|6apKmDn%nS(>L2+4u>l0DH$m=z*n;^&{(6{W^oNR* z4UE$pP(!T=N1n2tblv%sqSwSQCU6=P_IvO3`8PN$aeD7?(BE3c* z+G1n{U0B6@o>2E(S{@>?v^}5A;~*^UxRQ4z`LWo{yaCvu zUiqOdpE*C?2Gy8H1WI4o_h!nYYBuUV#^}jL!Wjq1>Rku+4=!3ao;tKb!w7wMOIQ1Yh{fTtY!BF1;LNt0ZQP;QOG9)W(S9n7sEA25zT2_wa9c0x$j?*TzruAY~_BNYx zw2oW=cD;ZFqOm#Cs5;LK&buI`3^24Vp2TXe$sE~!%BO4=b89~c;}=KaQ@HvV?Kl>L zk>*cZ#=zu}ZMf0M4e&hovn6t{-rvkII`%wZ}Qn!TX0@)B{C=U z(Q0m>4QC}il}`S1ud3~$Hk<1++~3->19s$IS}1ZJ)CfH9sO!O*pE8cxq_^`Ub$ZCL z2_}f=vF^C6c3q+cr$?Gkn+rXf22N!A;4??jnx!^r5I?yjwIHTcqt^ zQiDjurf68>zUb>-(F=ChQ0nx%#S16CuoT#9N_712Imf%k*X!u?z#hw3lOF2D?0$d! zIyR|PeK!`%S#lYqG8Bw*mHzkF+UfMQiWK&vex~WHwi1`l7u;ZbR*SAU<+j(L&6}>W z&q)wof;n6*#6DxhDIDsV=5ak5=L+O77N9%7L?ppap_84B3On4)>{R5ww5Y8-tB%rI zkT*d9H3lUqJKm#n6fIi@o6S=Yj?~SgU2~at)WN!bwAp^uLL{+QkTjj*_GHci3!h8~ z(v%r1$PyCxq!%uN?|Gp>9x`&oMV&OK*=%~lsR@>jW=6U`4EeIJ^gM+VPwNUuFBTR+ z(jXad(hF}>h0D_a5*{{|jnX{h1(BoJzQPrH7^eI5cW_T@UnAIg`rC?jKJ|+J=hE%b zQ+!Kj@Bd-!t)t=!f<55`2<{TxAq2O3w7TgK$8eD?AOTNk5 z-S^&}-S3?Jr|(SNmg=s$-M8yj{aRmnxtQ;Z|BwFxbe1!EB?yItONakEZ;%1$o2oVl zpb_oCD6Z0HSN~d>Q5IHlA(y9m;GI12#RvvhZ z8#uZt;5{b4uk}>R`f+d>8pWk?cCC*U1Kpbg1-eMXXH{RB!l4t1kjk^&0TkN4^m3BT zaE0gBUiTWkair0!w7-<#619ubaFQ0#d6waWu?&;AdPLri31OLD{|!l+>t1JfcA!CT zvXEKq>pRz#+GdHX(H6>z+caDZ?fmBDkgs;!6269^C1ONFw?>ZpU#!~LcVeHksBc-a zOrxX(+qCGlb4e_(VkBK{I3EXy7^!^{Wx_mrrY1vi#R>aoEX97IUr#x5vZWRKUQAC} zMqrJsJ8Piny3+L^zNrz^364h9?uwJ2`guVnNE0W$ydW(2W${v~iXpq9_95${;bam{ zbIt^br>B-p&6MHvx9O+$)AIS#lj^DG6bsAm=Cs@jDt$@H0PHEAD52pD6N1)QM5LsV zOi8+#LSo?Lk9pIHvL=-pto770@H@-5_`Su)Vv>b)KLMO(sPDxe=2l50ZUA$xM^yaM zFdF+Ev1J$+KTMoEptzan#Ex`?KDgRD(6@v=v4w}WpUQ;@+B?~OkyT#+8UkRlWxV0> z?}Y-f^-5j7*+PbjUO->Amx>QZuj&#b*PmHXKp|MQRm~r~`%30-1Y(Eb5sw2`jau-& zukmkkdkZ%io(Mc>R@RK{lXLYsP)TG_q#HvNokFnOf|p4vupH~SYSqY(;pARE*@k~; z(h~nf;6U!*kGQHueOA(U(ry)rUiOR}vBeFx9i73;uU|Gc*KP}@&m;{$Gmu}CbTibw zo8|td4sg32v_9nt~)&{;mo@4PSQ(uYO;Dvu$TRw20BJYGGLa&EA4Sp{!(0piqQl zCx%A(Kx5XXvAHG?aGA(V4mbcNs^M-@W)r$4w4o&DI%jhGwrwk8yGyEj&0K~TLj0Z2paJdO;Pbp$%PbExgw@bKA zC*FyyiG>&e$jSLY@IAB>EMEm&`yue3`O3y=ij(@VT>yvHPQW92qxR3+F1bw|w@Pb# z!F||vcirM5-x;pR9{c?Gxs=T2D2_!abvM$~?`Vpcvjnm%OL9*p7hP#3`qYRR>zNtK z9G=VhHkh1;d}26d>?G5eirvk5>rl3Hw(uN!f(`vioq0jj9%| z~!Iu^uMR9 z($n`!2*fD*dKAsJ+3D44A;V}d8nh#C)@3ZAhxHvj!pmxy}*M)D>d5~ucp?lA5eWltnZuP`0H{QhkW{L@-`Mqql6_;EJWxW z3(1^qdvBNHIhY_nWY+)Y1G@y>taqIIoq%J@ExV#iQC2@Y>0pTL;9TNgJ~Ad@MMui*?X8 zF7P<&jrEXsDgV8Vp`&@{YzxIWyKY`|ogWyW0}mMW_e8}^F8!3@bP(2MyR zf|q~w`)#pF>BnYp3Y}yToIvziu2Ok^P|#c_u%n359w+kY!F!ivY6t4;=Xc7*abnL& zgk)s2-F5d`%vhs@ow@g)vV%RwOgNTRtNLVzRJTy}Rr`fIL|o1iuABb+g~`?MzWTqo z!gBsUy25gB0yvO`g(+M?F6PGe$euqc{Z7m5TjPHJ^^U%V=~&qy$Ar=6QGXT4qrkv) zAtvHaIJ=Kns1aIhD5S6KNFEPzRT?ihA8lg7(5~AJ;Cs5p$bKsmdU-O0I;{SEMKg@X z*y-!~d%p2@Q2jR~zB&Wp6nbhvzX8dgSJszD^1m;(Z#x@*_nHG&hY zQw`Vmf{WgSR|ZpwEO!DHhg~YneI(lmj4H(| za^U;Bg5uDimTrE6$`P!8mhL*Q8j()H*gr1?7fu0B=Qjt+VRP>W*#Dm0ehUb@`EXhX z8aWSmy^9ZMGZg;&bbQ|RC}-)FTJpRe@Otam^@@L>@>t#V_Oy8V_wMZ*Mz9E;{Us~?TGN_~0KD0%VpzB;$a564zw%=r=$d9CVr5|%@KVSeAo6`+AV%+-sQg(< zcbefAodjEdNW`!NbpaC%Q%zpP3DXEtYX|fc24lRrbR=4TSBv2C#j*Yeoa-5MbQqk< zDae{vfJ+90J6UGipEbr?or>Rt`j9H``h8ySqp}Z2}GdgFh`m`n_Eo*-AZxit_H~>i6kT#8)%Bbj?DT)i;&&c~TL;+=# zt~#fuTBv3XX&3#I;D&9xo#v(q=^>PevROaR;tUX z;Y4YHi$ZCWYMhcHvy*)GWTk%Qz2SsLOh+arUxeU!vuSpTKP4}wA7?1*|cdk4<*TkB+Bh+j?pE@NjHWF}3;fkk%e zlp#Mv21)#rFRR{@DDRxl8f1nH?#SNp7DE@ya-vZ^u9IwXcBX+oCi3c$d?)>`HsiYo zXsn<{P+tt#vgp&HuV2{L_QtajoXJAkn4Db!(TFuBe=CHpwwP%mpJ0-^de--B6cMll ze^#kP2Pa5B=b}oS&|nGS{gtK60kjh)Wlvc34^s*W0pwIvne$oSn-f)fmkyBdrHicJ zUTZxh&rYE(JQYxG0G6khkcvtDkcf0{H56#SJC9&_*L|mocwxw~3RERPOP!PFrByG& zVkoA}-BvIzLVG%j4QI73;v6YnsNDPY0dqW4mPzxTp_P;=V!}FdZxcc*Omo7TGdVsX zN$ru5J;5#Dh~P=rV8SfvyoEE5EZ+r=@%lF35}9Jlvy_gw)cy-kwXTOx4{lkE2$sb_krK~| zwn+2v(E5po=!?|GSS{UA%mUutBm1`Vey(oRLi3%CT`ezmkI+|V zcTKFwA`PhOCcCOQNk|`X1zo$o^>blf%rB!iJHdaCfyF;O-OR47vmd*}Gva7l@gZH?_qe*96$A?@|+xG0`*?E_#K!^gl~cw>uPg(Msr?3w?==< zOQODCTT;$u!oqTNSZ7nt3R#eH_<0$8GJBe6M;#9JdK7I3O(J)p6>`?5ct^`00Gz7Q zX<8^Chl1QGfn){C91yIU7=W&vc=0vi`X=`6USD#HJ>a%2_U9^A0z>sEub)=m>q7mq zI#qxA@|%V}NY`-f+lPBydW6w!Z05ekg%G~+1}`BytxV-Z^YPa~aQJN=I0GzYpfvQ5K+;4{Mt7Cr~>$t&b;!zxf$Tfk2A;8pv=!tl$3a&m2 z9dXQRsU><8^umS1j;FLawn|*H>Cl)>oWM92j9Ayry_xePeDE8jF&>4|WkZQb_(n{t z*eOFqcnxL$9Om0!O=*r`GbDtIIMgGMvQ%7^^?RZ56U9FKe~=t=)D&VS$7Q& z_{(8KpnbOpgcJtvy7QoDt2Y?4+iRn~k(%Vy?e{&w%hU+lbW{(9{TiW!>4oZ*7^<#7 z*6d9f0XGrZy3{8f<5RPBMb|pk3C^Bqd4$FhR}7piyYZSsqzP6~U2Pv2W^qxw8iO%` zQY*sJ0kUOm*ZI}()GFe5`OATn(W&QIEV`Gz8{tc6lVB=T-YR+55I0q1@%%%2_DrT^ zGD1-xaTHs6uLK`4(m4_bFU$N57ai>5H=y0r0(v}Mh4}ymAtb-g^Z~osqGd$fa_Hv# za68U+SMVhis0r$vbG~+;g*zMf@S~*kBbMSfR6+c$g3toY_Hca482V!flBcy5HNFn@ zE)(+t%L^78HH&vQqHYvbxtrb{1IGw7ouTQUm_S>G_5Jj~;~QNfKT+SZ=EdOt;?x-? zWl6PnmE}B-l=b}cDop;^#*6!_)A*XHQFi;#X-JdHgy@xJ{J;n(UVY z8iwlA8O+?Wj7;=G{}gCRhmS#UJx1ofwf|JpZ~} zJDVxVqpd^*M?UXgQMr%pzzt4&x)@)IGnCc2dsC_9C4v6 z6wO>VBLpg3PCG>4%GpnW9as;p-Dny!sy<*Yfgv+SiS&bL@o zs}_dSgDcNhn;SG#KQcUjkiBFAwXoI=Kk82pMqfA_JWa>du!*-Zwe{|s)<5-{PanR# zYr;5xf$B+!qP5orPj+iPVy>^}1zL8{uLy*VO+t;0i!w()XU3F?0p4I@u=~nt=*(hy zADQ(CiH{@WJC$=rUM-|5t!olNrX=?(YO8m@_3H-7tbV7?=T}KaI_WJ~j1Y`961G`2 z!0OtCk{n%g#Bq09|NWI-M`r-Wi5sRh7x92$%DEOXQu;0oof&20Y^m75i{!;M0W5z~U z0Me%~9NLP28@>F-j!lE3j&xOCCT2zIiqAyeE=AGNV6Airsg|nO`*3<> zdKa*^`X&^b6v3W zbRN$}A(7x>-FhL>X~N5(>%8P=${z-`%c@IA2d%#Hd4a!Zgra6^=t&H!&3IrVK2kqs zB`2V|<5vb@?9hdqfRQH{&cj(cSGU3?yq}pJh#Z!>zht~&xH+Q1j^y}=p^^?_6hDR# zi%2%~Z3Il$%19lwHW$dnXxJLCtO;_84p3{X$NA2D83v^3ONxoRwV9q(h5AYK5xxIK z46cdaMmFJI%A+`2i4>{}9sPu>ia*q9RtmwrmlgpVs1 z3Lyo4RQ7^?M?Z%X9anQrfARMSTp&*Z>{18!+ z*&_BP4!1EJ=p#&yp}mT<&;{nKE4If9yM5o9J*2*oPkl&Q2j(ysXR013^f@x2obK%Z zK*|Zo$cr$r8lB%BJpE z9>=UA64pY4$W(ZH*S)!E-FS6?zG>4?k3SMzl2;xH(+{d_;V5C#4}UFe)24EEr2}BX zUkdk-Pi|jl1$>qW9bGOBY-w6Zgx52H52z?A7VmS`07^(KD8*aR#Rf8bVE2Ufq}dDG z1b^DSHLK8ot6$}f(@p`oT~{{t*VMl8TQiE@hM@(=7p#Y-7}C-@?IY-<-PXG?t=6H$ ztc$mB=C)GO0=1mkb#`(L73adRytm&1jk5@mnma};EGn6dH>8P5VLZliP%ly8#^Mfl z=tUFc121&>25%DNb1`;gax$gU7)^XY(xM-(M%5W(~0MED_*a|H^(^PW7wnmT(=IWD%Zn~b@uyuVUP3-5jZ3$QCaEf?ut|MCaf{0png3{X+LHJ>Eu{F=@s?>eFr$k zl{afLv1t4C2v6o^|B_l-m%s^WmDx%M#R6$aDZ(G9PQI?2aeRg8yfctgMZ`Q#{-_E>;<1jqk`n#_-OS`O8{nh7sx~3b0|^k24HF&i`CgU`RDf8cUrDkg?5!y zzeEi(c^olVIHHl3cxhK02_yw&sB%1ubw*C5;YK&Bns=Zv*nDvfHH&SYgP&xtY@XAF zZU;Z{x2^?h9j(?OUJ^Q%f;**sWv>RX#c4v)p#N%j;7A<3V{0{o>vrrDrWIYK-NIsvlxB=TOe!E zYIEkWEailyCGkRqS7qEhAapp#xKd1P!YGBNdlZKTol4XKdu!V zOHGoYZDv)MH(Yb{UaJuS2o~>mHK6ptLefY*B5#vPG(DT^eWmVpm|7{P%U46>hjNSE zO?MTicd{nz)CjkI~(7dj0~0{aa1b_KXWL#$R>}owwh+Nm3m;8 z_L$7A&xe-?Ps~N@v6VzKL#1X%u$7{rggi#Qrja9&Rx>$BqPe4L-FwSZE|xkmH>|WZ z>Nzn>C{p*PK_9+cxwXQ$h7Xe9a#T>oL87A(1Z#3n(>@$b>Wtt$Z!u-b=l#xPk?Bx0 zyk>WYzVruaUHqbc+4`a%f;UFY<1Z{dKs^${>88JHzQrpV!WqwyE0I3(KWf2$%t-krn1VEoyR9_-x17u4BSuoKRiB@mJG=DABIHB(SoEy6oa++U6$%Fv==`+g};)JzS!|gd)aV1sWZFGwdz%Ez90YuE#g>S@UrMZ4LpmalRZ%R2u4t-`slwS zKBu%|ok{`+4&oQDOGhG`Vu>xfvu z#u-uG>*8q@83=tkZCd#V1$lE{7?OBXg(>j0QXyzKD2!#i^TZ6oCPyEVyzU*euI*N%KQtroSW1?mmPX{PaRLL%{#(2a}I0 zY6_^+6E)P%;m(NAf#W<*W+~^qc<9kBHmv9*5X7vofee`!b}X>*Ajry zf*t3|8K*Dk>vtwfBxEqa@Aeibz~nF9g2yAp&8rECq5euUKv<0hl0*nxuP^`pu*uPQ zU>EJ0N4q!FEzYtE$p7nxs?3oOqgPjeF#R_OiD0O$VEb66j)~>f;$BUh+?dJ9TD`AO zNQFI~KY1QeR*Dwv-PCt!)+#%qL7nl!K&v}nW3r(^zYGrT+jakz#I+>)-5f}rXKTQd zp&mcx?ze8`ppOKSi*(%$o0sD1eFx;Gf-usUkVP4nV>xAObb5v4FsgMM9gs5AOgkmW zw^@7&+E~J3dE>>Yb)bQ1ZcC4xC-79rr%Na0ssdfNc~XA&h1(?3f4>~}N5}N5U)a;p zOjxK8xi=AhZv~yU_jdH{ye1BxAAI=2frSK8EFzL$=7z`MlYh&3EDzDv@2+hNa2{-T zi|gz#Vj!c#0s%h@#*myjgmq0Q9)A!h(BebJ`i;k5Vm;@WY%wjz|Mw2j%e$2}+=^SG zDef$9son*~88Lh{38WjW+S=*JGOR_(s%dm8<-xF;DpzIg{CJgbPZ{}k60w(SdQ^Qj z>AWjchsWdBgK?k#xJhmv$pr-)R4^hzp5flwRMMMg#|5R&9(46F&~HYqA4)JyX3~WB z8ik!BUh}d}sxGjy4zM7^X=0cjODz~xMcq5uKA*rdA7BeVVEc}z6dbT&VkbeE70-qjww zq{1E0EN3;t$T+lg^ZN??91YF_RycCj;<(gI*y@GsX=$!#`N=422lr6*pdrP3$O}Zt z-8an)66sR{OSzz@j=2<16t|fC;jgGxiYW6F(C@l0t6ZmCd>LWZW^kCxFR^*4V=WQn zi8JLwC*mUhCGlW0EX?5=SLUJ)_Lnl52wR^=K-+=wNl7TNfqQ7_t{NH+d!03w-*m8p zY(-+e1pukfs)eS1=#TURf{C%_F<~7g^*-->m;>kJ$R+mH_)dB^;#p#o>+|9edOqnF!}NKe>vlOmcY=PA zdXA1H*T<(Tmi8);^6wf?YJLbC7%Mk<@g=&Jj>i2i5b&=P49{#B_-s= z+a{Lc9bvBm<9R$H+g9J^099@oi&I3*phC_Bk-80xRIq9+(xEc09tc+W0etoNs_0(YE@F47c94|~tH1tg6I=7#kCKOaA?~f=Drxr8XJ^V? z=p`Qh&N9WuPkJfW!+|2dXwS@q)QGOj8rurrpJLM-qi1hl@>rm}%g4StRc z`E3Q)AmK2u#FT|udq%Ayx?vdj-C2U*k8O}15}agvS}^sOxM6Y8id#bR zV+L}IySK+IA=PQ?DlL-;ELS8>52qjlL~4ILtEsh>fd<{K+lPli>58PZ4im~#!>dmS zq8OZELF|*%dt0ReW_9;921kU)aU9s-Lw`o9l#@~ zwj1Ub;+5nbKPqFyOb@R`na?9nxmi%I^*&=K5)Fr^6UJR}SFpxQrQ3g?!OQKuCUgM2 zL}ShC(e)~2B={Jznq@bPbHtX^vfftx8OurR(&b$H9A%V_7GruHTK3h6$L`c=ElatN zhxFwE!&M|2`}eM4#xqrE-xu44IMCb6)#sO&`&;l_$%Dql2J|a8-u?bTabWlKv`)ZZ zCtoqs6X;>5%P+ev-7{_eyu!+=(10wAd%DJMF1vieUNrGHNm&}4{=a{j{ZAwOrq-KU zceb@z#P1ZS!E=l-R<;qN>g`m4vF8(30{;cX?VJ_(RWg9E-x~h2Y z9m1y~QYH!itiTxw@+LNy%Yc<$tVVNTJ7crw=bA9An!na4vw4rsD1AP?%@^lXltu<4 zUd`^IZba3yvDMo^)xx$-4G-w|PQrR6)9QnkY?#bYSuEj?+e6Q;t^Fa(?dv%Nu1mkY zeM~RJ%d9&)rqywRb5i_YM7tYAHS zW8j3dJcwY$i$0-(vn<{^H8YzOu@2Sh7T8f`ABqE9x07OFOK8|aBv-9@ccl{P`wz9Q zXV+q!PWf1{&Sx#wVz@*)#$c_Ax6TlxEEs!Fz_k!;b+c2%ls}r6Mj<(gmShO&wP%oz zco3|!gi$=Rio+7ku)c|e$3#ovJ(DZn)?g-o7R{-Nc+a^uJy9RSwOfug!PhnXwJ`Cf z`QcT|#t>|{;V*T~eNcecFdX^EDD>NUmBxL-2M@EbrWK@ewi}9fTydI;pJgl$yHm@l;Zz$J`DFQe`A)fiMQpRdG zj{Zyen-Pcj$MNA_<3O~zBhqy7SXf#gk#gx%z#5OMdIg6TaG(rpyJH$|3yOo;zq*Ye zOcv@%j6mWmNZs$_7@=A5t6U<5J%X@tY>DtQ%yX*Tz!FX$+T5w1;Mx~9mWs|z_q$%Sty<=(9hb zz_tQ#CC)xMk>LfP%pwTFevNr(MXfxw z`{{l}D|`Iz-$2dy*2!G?lav|Rp?E7hNO^*Xgr+xA=bJYVk#U3GH*XIjV~Ili9WCML zJL>HXB3`2_fbC5-Ci~um8zxyKH#vsxbp%BVS)99ANE$y|V<)J#cKf}-G*QS1I&ur%!Z$WBBG5;ZX+)GgPx;qq+OD?2 zXu3;~O6Mr5&#cpK(8=KrxV9D>I?WtfjZ-i@_Hx>yk#c3b+DCM5zE^ip7w2B;S&hM3 zvXtMdsbQEEXpcmL`Ai6|;7hUCGdGQhbLol&emv5jPq=rN2*2enBRHo~YV?)ESWxNq z`7-M%f1P}vOQY%uY&AR|$CGSQ5ILptYos7VOymsps;|Bwl(;=N7*mLD|HGb5{IxUM z{W`3c{Z1MC0auKSU2u$XAyz(y%(BolCTq!}&{X`ZR454~asL~RI6juom+4u%IWHE+ z)CrB$>w7n!p0P*5j^vxI zFG6<3YJZez7uh&+S|?GCOS1rukO35-V;jHUa8VGzOQlb=6lK@YMD-nc@hXm|>q z&2z&cr+}U2P?`UYE@w|z`t&mA@-OX1HU%hIq}dj6kZ5RGAjV{3f|&LuQqm&pQCQ=$ zC3^4q8S%BpDaO#h#h|hU!$~v(RC|HY^MErNb)>0eVaUhbtt91~?phFQVr69qZNOK3 zu|O>wt=qvs0v@mq-YFtZc^BBi??nGPt8N)=W?Q zjrrCi1_aBW;UV=GdxYMzSKyV>X|5TEqm-cjJ?G(zBK3=!!>6p~{nnl?z*%lx(?P zW%PJ(o#Zo*r!YJ6-D-RnAy**P)W28UO>=!h z)ol@qZMIJ01Yv!;k>4eOC>M&olVdJVcZ9kuh_ zF}@5Y%)`-;l>gRzFVdt)FsMXUn?!ZSY^9Pbl_N56e#(FF;QfWJ8KVTc-&H8tN$=a>cl`{emWBCyQ{ znINenE)8J%XW&`$=AdX(kec$OB-)&$2*6eSu?#8UYi&=e|J?vMb7`wrdZ8>ebo0;z z`G{@BQ(#*`^;6)f!Ux5Onex44^TZH_+_Tx%lYBd7H@p+h)~QxC;YRWAZKkKv&03VN zr%IPw?yUWEsb-C^nc(u+Uos_(3#c&~zBS9)fExRHb$g_Z>RxxE#CyuA*0eZxvJMlR zgAVJ^Q-o?QB`L#RhbYI@A{$qwhtud;pIKl3i9NEVX1Y_#ejIZG6pn!Ygep3f9pWtZ zZQ?(HV`m3?P>^+dzv#19Pn3-*qQCBMH+irZf}NN)U51Y1O%qB)_!VwCVa1eDj@$D@ z&j0}erE#+G{w`7xzduH(?+nQX`-SQCY8DQye29`BK0*lgFRk4Jm6XJ{H{x=HL2U4W ztQe*yIk+rZ#}h*6GQO^gsMikwf$rjB%1g%~C31cO*zVK__F5%md_%vmNb$5cgj6w+ zt2UH#OsQ{irPLBe&GC#i&qZ4f(t0N2M#o^_F11*8uhY%#;caF-!s8GeI50zLcQA&n zOM$*}0;3(Nd`K-SpW`(8qQyG2o{08vSuM)vd-fvf9(NX1rMnL)9LEqcsr$~<8qp)2 zPZDHuBWrhHFwL}J@)4<2;mjyx@wr!*HU|s2100)aQ35m`C%55?j{pmRBatMzd4YWb zV(=EY2}##Y%T0<*A2yQUfg@TV&lT8lWj4^^Es!s<;EA>1w?91mCi2 zWHC#nj!K$$Hm&--h36bZ{A@QFm@#br-H~f4s6O$YmWe@0E_zH|!;o?eRm;ba3W2k% z$TRam+H^>uJBu71XUmRcGS7o3ah`h8a%`|JUgcBvEiF`k<1H_DVLuIPzeh)s`LNNx?<4S3pCd&j%+ED)CpF zY2AYVP?m_qIfi)w7yJkgXxqVEv22Ymi2)kUXu{Ldsq`q)|fTU2Vf zl~niBz9sj32Pu}_0Sju$Bxm#*=$zRrJf;NxQ$@a_i~;55(~?vcHn49fj-R%jG%9hH zknmYC;6N-0+w18{?;`3ZgwKpcJ$fzaIdg`b z?@U(yW9z$RhMKACbauM6N~Pqv+k!FIr`F9^s8#n*b4(|%3el9igvaqq%BE&RE6bQC z?w&qZ9uk>kA&81L?$^agJa(e+8@6i1Vj`jn&5;oCZG?G{e9q^IiKVV>=&9 z`kC6(tWKGsCLXYK-Ik`r56#(&1j&|mMtC&)y+i#=8)QPI(JiTuq2QbddDC`M!XRKS zZq(d;Cs{TO-ozxr;`izyfG=#+MB`u4&t=8CHkv6AP1O?$j_B&cp>{nXe~%NK)s$~l zCv-hvhRSz5aWV~_#b8+nYVrutkTPs}>^(?W-AkOV0L;6+^ch3hWq60+6ph5P;V5^r zqYWC5w0B<&q&ws-Z@M|lvywA*ACs3;+x`~8F%K;N*0r_3&d-_3MBMyaz~e}y?+A5Z zVRaI0OKc_VbFte1C^^y<2NIs{c7BAq-zxEqmbEp^Mn1n&BfmnJ|?gUg*OM(88AWL;lFt5G9+fPSL*gxvaTzp%_rPhv>1L0hST)X5!@4#qv=AR`yw&Zn+)_$DyjA-%b$^Jh}I;u4Kk9;S)Gk0Dr(5v z{EHKgM_urLi9g`_e~do>aPj_c;RlUcr)6v0n6F(*TlkSwU<)92irDO@$KNh_7+*8K zP=pnoV+hrGEG{f%cQm(}8*5v{AIM!hR{$h^*J*se0cXh#Spy8)Byh%q9Rm#fbKZ`6 zznTpi1hj8aaSAKEUat;WKVF_*@0||u`n@6804%d?X6PlStBz(Xe#*%jqy+GX&|7AN zDeOF1orHfUXG_sdS2Ov+;mq?}IfiM}0A4TSYW_=}m1My!!>`M`U8<0#EIZGe?k}_D z1-q*u^(Ds~`8ce=(~3Pd8pdN@!C#qdQqy)uA4$W^%yu9M@ZRt0cPWaLLHl9TajGK7 z)^xBFKlVTCpn|5Ar8T$(y^V~}bLG%Y{#A08Z+OXfQY+R`|NUHoD?y${G;?#ibT`bDe2`XILPX;8fus(dPo`|vPgBdC zoaZPqBw0y|kMTTM+fw|rb<7=&>}qB#DM_^`!_rsoFJ~*m)1N*P^_~!Sd@Ri4(P{-M z#tQp}ZSuu&Iof=4KAUd%S&63Tws>|G(_`eTE`Tl1=(?XEE5Hzm2H(_@;6X4J?e4g} z(kmUgd{a7Lg}z{}dZjm0#q4hLRYbXLwJ7#!iy)KO8$zM8Y%A(OB>TM<-MU{|;!okH z$&WUCm@(c>aI3+pWH*so^6}+KJRD)0^|9twQboCdVUu-Y)Z(mceLbc;S`i@_RiYso z%Q&@^V3WAEL*-C)bc?O%&o#Ra?|Znn1mm*xpKO{^$DoAr*p&(xV+@+N13e=|~O z5$0LyEhD{IB3&7Y9ndlfyeBTVltRVkCf=iT|4I3Qgu4vj2#}fJ>2>ukO2X1i+&*f> z`R1Bh+d#Fhjqzy~9#$3J^s)4_yfVE(>3Zoeudj^ieD^KxD}cN2VE{W5-27uF@zXi( zyYHolCe1(|ALU&I{tbrUGJ-kh_C~lo0&c0P7Y-{05 zd!6JuG?I8?@`>6 z4etwrjzSvk>eRwahI(CmV(M+$$#MME|~-8;TBM?jUgV4EfE{DcW}X z+INSSWp?xjK~9RHk8PHVVFH87cluV)7s;B_-rozSx*^zJSqw`=tqU$wrNrFRgj12r z%;M3S`*qXxecc-6Cj((E;#*qnoLSYteOt<*oC#hNvg>vTPS$UN*zvsH1z)$_KhvwX zGw`PCr-hv3IvK_)?Vsd-_7BA9t0LL3 zpOb$Lnm*wy`W--O<+>rq54d6dRsQ_v|D5{&TjZIVmy{WWawdF8`wmprZo_TFA`=^dBw$BSPyIGj=uqcZY&i)WzD^j#eh&cs2 z&p*-2X2}|H-+yjKsg9|El7E4wsCC znxmAAgo3ftfARVsWK}O$kh#5#gM}jnv}F}@OKWHVpBKevQRw)X(<8GgxtN)|{OjW9 z|J6k(QPs`K$-(^eH&Gp;QhI7Ito4 zeF^|Og)TQc8}tqzF9py4%EI-J%FWHf$H&0|?c0Cm=H{i)<>KOIVdvuDr1)RO^eK3t zSpXFOd4q@Rzv_hE}>lovLR|EhqKlaqywjfeL? z^XcpVcd7gv?*Bt7|0NG()_;Qxw6-(npkU|wAENn3Q8c!P>g)fmEI>PBOV@uYDf&-6 zKoopzTr3@e8Q3p#q^M9KEuURoMM^6e}W^N817CsI>9tvho9&Q!@H-MJ{T7r{> z8(KF7I~xEh94>b5|E-_@c>kaMls9*<1X-~|WyizL4XxMO4rJ~^!3t#`WG-R;PkW&c z+kfPN%9;zx$$t;}JniuEO_7}8HTXSwJ@!q8L*`TZ$bew78TPQKm^7jk4UTj^L6B2M zZX)VPDC#a1LgY`Fy3d#rm+x3pJSl^JB~XOR?f9{JZszCKF_f27v}}Jc-q-zoEsrkY zaqqXWA<~7xxu81#TmARE`uzNFPH{MxckgRZ;fAg3T)2fvY6VG*N{G9^>YkUlb{<}l ze6WgsADN|h(9)z#j_E=an5HaSWnTj74*o;;F1iSofV{NSwzbJ&`CGuiU80slb*VGI zm?5ecnwK&>b`8`@SoxkhVLgJ)#Nl-NXu{;xIZ(UnUD|dT@xg)+&C4tzTLJu+wgwmD zn3u^ukIoMX5j*8l8Vh_x2WLM#8NWOj2iE8Y9x84;y4%0g77*aQmr^KzUpmg4bTuXP ze@cpjwVb&5!HH|LPf`6b$O+!JY{>glk0n+qOMz z+qQQ8#nbLaROC4)D(*wwhl;vc_hgDuDvt_hJW@{)8w@P3vJ)6}A;K{2qcDQJ>C|sV zzru4*c*DP05c38z;u;YP6A!3CYEU!65fPZ_ORPrvPMLo>esWEz^und>Y6bv+`U4%y zSW*6(Z0J=gF@`N4XmXek+)3^PJ-t-54A@ET#9Wz0rwKa(F0N+20>@k=3pD}&aUBlI2!MJjo zj|714Q8ytZ%?fhCq^=O-1$bE@T_F7t4d6T+FG&l)al(!j(9htw^*rG9IthMv$)Sgx zDt-F`ay5S74!mU_==bZ$L9l@kWI=@-Z2wu!20IG1Iw z^5u^-D8=&|aa5B58~u_gh2>m1f%8?sBTT4;i1`c#-MR(4Tk>?ZHuPv2UFbH8}0S@ift}! z>H8i}iGBLO@7)7q+czd1_thKY&=AMk#qI$j%~9jj;Vc4{PJpJ#MM4b}SlVNjO=W|) zvXUQSoNKh0!1G2S20?&#{cJ|%2BUm$pwzwuqm%~>xl|O=PVB=*j4LO>9pW(nHNE;Q zl<0JFoYH&N^VJ^wT*MSV*0rh)&dv$)`oxV-^k7ItG_l1~K~hjZt!17p`x}m*K(e(i z#t8gt<9p-%2Wk$X*iJF#ab2-eQfI@EP>BaR5H2ULdCC zA#Hr`&iU4G^6Cb~hgx3`9=bT?_!;<@Peza4(y!=fdmp;NR;bSF{|fl05@^Mma3vmb zwY->@4APqEAqb5LN{;eBQ+b-!%^mT@*gVoKdvAB2^WNz1!QJRD z^KDO`ziqFm@^s@kZHsulEZG~-n_P_drZMx>`_A{qNk=Hw95#A$F~rWk%xCZ-1XwfG z_IffxeE3FPRoUV}<2Y><@YVMo?W32Y2?V*8djkMl8VvG5QnVD2FbN}GQ)ktDf|{cn zWsw4X!cLG2I*2OwhOqwNFOe3GGh3Xa8eBbbqbNyNkGVzlC!>&T%V7UK|0Msx>8*MT&wFE_7=cj|gapYGsB zR$(>Cu)2HiU;PIz&}MeA-@#WfK>pb6AyWB*yzFUf4__S&+N{1K^#nSilW_K)ee(W$ z=(T9BcpaRhs)Okoeox;K^6MFH#*9B&^qt))>xK3n-I$@Cw_*C=IbhAr6e5>6QN7(`z`5&*hqPH@W|gn&Zx z?8H~+K49~^N|32X>*HeopzZlO$Y+mfE@Iik?-e`sU-J)@>_8h&gm-9wvnINV5x@~S zGQ{BZyHR;T2>v9h&-ast5@}Mg2d8ofz8twrG7IUi32$Nc5?EugG_QqT&=;kkM1C~7 zw5mnmoj2MK0)6j9RT6G&|pLHZ2B)1oKe`ZIVmL@C3<*OOVt^LfRo2hrCn zuU0SRsAnu7sRQGNPCpC5yL7G@09TOX7kqMSuPMyFYh*@(ns5M#gXV!uf8YTzfJ%v+ z7Ow_L23gn!iZ7TQ`f3v_0*Qn4O!TQT6^c`TbAsgF;0*-wPJ2v0wE9gs;1{g$Y4;Tv z<67|kP52E0o?P#{*0hEF(gbua3D1#_7b*b{y!VMNvV^R zyB?Rc)n7s1Is8BHV?blO?2m!nKe%7tg1C1Cee-^r=9nuQ*c*92t=!+O0pEI&jAkJ0SG(ac_9I;B$>by@yx;KMAq(Sex{7YOus+AN?21bnRXkB~rTDs$(4 zx-9zV_uc5*+qaQd1Ow3yhKG4Bywb_R_N)5+a6o$Ae$>l0^Ca2m?7^81imwgG%EZmn z+Yd!w^8@%m6=dB%Ie7F*W?BhM&0G(JSsZ)bD*kBqDGnpLzIL@=7I_e-88M!a4$FKB z##^w(C`kTsh9le<#+xxS#>r2HtX|s^9ykSh(V;deE@q^uFun~Z>@;6%KKHWMu(@tC zvA5%SdJ&EU3;tq#lsq5A`#eZL;h*-&G$LJ~7pWvO1I|#wNU1yh_nM%QkCwqGkmVK# z_$FQpO5Z@iA(y<)f!?P-bl*82qyCY;c|d|(0xpF;Cp$#nCYLWKz5ORUY@gph?yLT3 zuW;&@0Tw+Rn2((wpL=+tar1e-)Q;GDJWkCoxVcozpSUp=T7JjoClX)}56?t?xug>o zk+_YGEIhpp{{iuK_^bHYuk%G%4S+ZQB8-TEB4H>4;Ql!!VVvF-s zgG&=W4PxBMxsX?NCO&Mk=t$SIT9=SxOy)@9aP*L(I+1M(+bE`yTsus=OsDRj{FHq} zkyB;cZrfbjZCg2~%w~LBa@*uSuG5SMwnKR9dY`zRS607yQ-&Q=3#d*d-DIjwRWn^< z4WOoB%eHgyIsXbuC(JI+F7JwLBdZ0Z1;Oq|a!Rg=B37navRbxUKKm!-G3BjlzU;KZ z$l^@v?J~Tv-C{I%T0qCq?S8GHaV1aB#qCSWVK&>v^W^*J?C#m4`x3>`@+8%UVq<*c z5LWvdt!dK;$cwHOn9e=w79cr9=oIM@($+S;VSO_`%iYoWHL zhKA9!Ss^qD-aP5WCCfg}tWA$amqrz*Ov`~l_rj=cfAMQlu2J7iZ^fn^u}Rl-lW^D> z%}H(3`Tj<;ws^()!uG1AraRRegw7q4j-zA6d1lDd0Lh-qS(;a7&UxHIqVVX_VlLjj`qAAh(DQ!h8FY>+gg8EYQfBi^A*{pTZ+=_zjO& z>T~s}*7m_U7W}OzSD%H1Z{9gUCz8?EP91m4{%W27j1`2~L>O)i54P`;M>6i)zrKvvf9%L>!fg6df<`Q_`{5c1=R`tU9Iuz@N5TNfLF^%c z6eqI#ke8{Vs{2w#c`r`xaME_;`Skh6GS2c##f{odQIF17$Gg;1(79Zje~w~I%3o z-9yOW`>G|ZXK|053d~Bln=%xiL}l0RI<=UJsJlj*KpfC3ec-PUo1m?I*lh&(VG_6D zbD`#6Fq=SSyWUd^?)}srK(Pbvx4&#d2zCt|vGM;N zkjpL+bs;(SZDP!!-~F$(;Pz9YLNab-!N)_Qd&W;tfh38sB2yO6Xj`JVq#9Ek&p*6U zxdYipC^7ZE-+k$;yF|s)f{*rP!4v z+==2K<++6dfdBtk|* z>;{qs+>F9*n*chi47?0ov|aRFF*Z+ld-1mgxAnK~du@)woK!v(J_@~MKZOEwi;oiT zC|?=h!u}-|r&{YIr@_C3;yPjarTB>XsqN7TP+G&|qI8Ds_EGke+Nx|Zw)WxcLEMjX z_80f*@31y4+@ic4^R_9jG4jyzP&$WChB1aPhSP@9hBbyXhF6AGhB<~fhFgbPhkGL- zhJyBa_B;2z_doWD?m+Gw+GKnw{YdeV;vxy6^1|~XM~8Z&@*?u048li;{lg6+4I(Ip z>V_SL9EOwj1%_{iRQ6x?dxr>y7Wd;0UG76(lU^HNqh6~##Cc(O@`G9uXg#pq`lCE} z&VzGqk!izD)=tz1OpPiPiDi+@Az4CbLX?d%8S(r`4HoR5xnMRyD4fjrp<9!9+fojb zLJrwC8LkPV@}oX8vPpkXF~a5c;fxy<$)}KOk!#T>hS7|x+9bP_bSbw<>CrVJtA>`1 zv)aVFw7S&0RCH;!iMJzp?(KXOdK5OpEixf)OA(Ob6skzDB;>B7J}e{5N0!3>3;RWI1}?p`lR7(PuEbu0LY2)X~yq zxh^jstxVNA_GM2{TRd%IuXMdNyFS6iz+`6XXk*ZKxZSz>zdt<5q|$d9y&0%)dMr+; zRc-v;?0QbsS=n;?z;mzNaa`GY+4SBS`5;(b`rZ{m5V$`*9&O?6am~6NN>Z?Mzd1eK ze0!|5=y1DP>vZ^OoOKoW8Hi%&{+y_)>;4|ea_IhA0fE@dao_R&nS=1>`P`F`i1qs% zFT9?D@caUEu(oiQ2p% zeyeW(+F@X}+hH|-+~V~?(~C25P&Q1vRfV%r*QH-4q5d^-V`(yRcMD;UtXtmUqo{n{ z!EeF^bsFfgkmF3?H96`-VfXog%$9*j;7fan0DGTLk1~2XG`CaWcv!&+yGVykAz2eB zit-A_M|kK=LDdO7Gy}!c*k}er>^3l+f}#mM=8`4=;kq@QjErm2UCN!yCZfO$UiN^N zAuqv54fc)tgOMpl7F>0Ld$C*V)}~ni7aG-jS9gUg$OY*{Zjrub!%^<&G>Ut#gdZgoN)_nNzqA707wx;?y zImdoHa~4!qii6+GtFXEXlY*Pg!RYWx({z>KifD7Y(Yf)zpN^A#TVF6m(I)H`We#et zyG~!3C4{EnNt)#JHnFVINFZ8@ORgD%Utv2*C^d;51HYgw9B|4wd5R*-p0!gMWA-Tw z1>26KvDm0$M!8~45l$p6t8gQ1U#Uqbk7|YE+BVnk(Ck9zt8>RY$#C$IHvEQ`^pX7U zHv-wgoM;&=%1Y=j3rY5I;FyV736k^e z@XU%9=RehHTy8Vzd8YdRc&{ZZ8)$`c&Epv=C{Vd9LjPU3a)8R~*G_%Y@fxPtXG(Ox zmoaG5Nm~6d01y#Z1I(MUL_dWZ7gzLo>3TdSs+HRTHNy3FcaK111|MVF>>dCRLdfx*e@2%Qr)6rD0|v`QPb{WVHuYR)K@j`)U) zInPm4ntsR0V`<{E}riQPFX>4;?&O?KMKciDzJ zY{(H5#Wk~`tH0u{A8j>~uj~DG!ICcwjg}Ak8t6uph1BfY8raGKH!1CwX)9ylq+QH) zK>RBL1q{~8h}3~Z$qL!puM8S_IgDpD$n4HisDhDfCHT!oNJ1-un2(nioOT5n?JsAc z33~C(fY(o?d#!+nrNv*kXgEVV{_ki3jFc_RHxvZMVEe=f=|r}#IplKt3vcYH$sHVr zIc?9wb1au4co5NcW;Dr-d>@zyx{yo9Mk3xJ`)_Rjc*YyIQJ3K{ZND&~DbZiZ^H|xa zE)76Yshg2n8GFE4YvmwfZaV=2f;A)Y3iBbe)}I#`Zb=D4mW=HRxGCMVx>31$yg*_N zKDYV^fh%EO2)Ez^G-xKoK)Z(!O-Kn{QrL6I*55a1*Sof@jrxvV^>~?ku&&_XW^N}o zA@!^0BP#wJz&Lh#7W-qeV+yF?BH+McMfCsdCO?q6=R#F+Ix67w7DH>TMCjH9LryN2 zR~xWB7eX%bmHxd9M0O%Nms?2)yp zocH{xt}kI0*?(@NJ~CzF*=qm|$zv6%D^>}Xz!CTwd^wBzH()T5bcAOZrU63x2u*vS ze?0Rf=pQf`#4Znf)4v5b^CDBdkH1o}{8aiKXkP31FRw}=c$`qZF?`eZOx~CtIo;T^ z{9v=<3x6~CA~7rOlB^FmF8DnKNPqFR+?6K6CSuxw$N6yN7|Q=Z zw-AX8Tj@V&7bPL+Mhlcu%@+le{twd11tXx7{{#B|$D03w(jR0p%Kt#O5R^<>@jqx2 zg_2Ir7lld#6b0jvT~ZpIAiE|1uUvN^_P^XE0ycR}@ITl^#-dCJ{15ucm=%dZ$e7!L zqmh}V$0&(hps>hHljD_ykN-~%B8{my5P%F$ZzKryKl45^*iC^NNZEZ%Xt?vQ5gpV`Tee$N!n@4#fPIyF|DpTT1)~C&*Wn^RfTI z4B5KpZVNGE#1B8^kIl*)j${Qmhek&PeAGb9?OoW@3tN#rt9hjVCO59R#H5h=m#|6q(22bb{xm&XK%*mw5r`0Xht@o9! zy-TH0SZUMW|3Re7Si;I_&if*Z^Ew$_ZO#LahKbJ8d{d#JcRJ;Q$O zsb+j3fpcI%?D=gD(HnZ-0m+TfR$MbSJ}5pSKGaYIaNA*e$k04WrFShNDGVSUvxN*d zBib0)N0d-YVWRKlZ=?#@Pe4-ihHuB2_l|rXCa}^X)ha~j)xhy<0!3dT* zJgPQ1G}J^5zeZ=%BO5EI=PB2z7?j|b;TOefj8;HYm@JFSm5<~@Eu6A($A}j#gkx6% z#nexFPvs8uurgP|!d8erqjK5hUyk<}3`~NNVMC_{@yJq^4lj-*7`wZZq0T033CKm= z5Oedo$=n@;^E6>bmc_Zm$x>Y_dzW>%C`WFcCss+jBRb1q{p~obqz7J<=?qU(+rpf_ zZsmd8Y-9K0Kt%RDSZ`ovl%SnrISGk{!%|aq43r4Dmp${&_(-b1h8td#-qJCd+{}o+ z9Z5y&bTDk)*k~_$LIvRu&)BwhV zjKZbSB`b}EK^2qMZY<&P~B=#&0EAovxxa2pBxG zk8wwDOB=!%;|cBd2t3BndH}rs!MioydtAU{SkD2&8@wEZKEMMr%kgY4c>@{I-r`ZW zx(4H(G|Ne8sK38wn9H8cd|1_dv#s`FSI^~(i&Hx_b}~@z5CJ)uxCNecqJWz-g4Gb! znH&+r#2#%3*ybCt)<^D^0BryRzU8iqBRGK&9+6uC9N7`a7q``s;4Gg3eeJpr&U@0CaN*n+QV0ZeYh#88!kS{V&8 zQHK=ggI#fVxmH+pR_e*EsTGv4P_s8030&{WAl>3iV7L}Pd@$=RVOf~Gq@~V0+7gU> zPz~b!ySBqEV1W?BYIy_jk-|l|wzyhdIIr7XSzU2aS8p|A(a>eBsjnyWXa%GuoA(mM z+fs>`6lvRbLGTzGzx7>3OG;76Sdnk)_p-6y;dHM6rlsqqyFSo+MSPy7cTJwzN}Jw8 zw9$~ZS0}{clb;1)!$t|kWrtu;J$i40MvuP!G&9Nejx$K%j+K%ekd}Yka}L zU51>j@6{a(^SOE0#+MAu8g^YUx>|I^xg?`@C|L;^&EvEzqS{92$`d9|?jO7bJx>7{`-goM+96(1nHVos-t*`13t5B~ z7Q9HFo>Qn{T=c!X{3Z6u4^*Sy-UIGra(ioEZQo=;EtmJ^Ya8R%`U`eG!S~RUN~Pmy z6K)%#TOknt2`dymBlut{u($8wmfHv6r%uIb8RrdSd6C*nuuWl=s*3^n&Z)_rcXdvj z?w0a$e`f)Y5cxSzNP3KS;UC%cl4P?a0~J&3TsMQvDcf*Dc)0}jV(d3DBb~7H=+QY6uc39GHmOV&1vAH z^C{UOrCsD!&c~O<=kAgF*mdAL@&P|0kNBJiDP}p3!$^%EaJ`6cp-$!~HO^c6?c zIL5{)&7viCO{$QX_K(4`$tJsX$HfzYaa;ml4|da*zsYysKZRHw#e5`rXZi~Lrj!)U zW)#*Wmt+;FB%fal#jxN5KLq_Nm!F=JFhj7Rj`-#LR2wFT1p)@9wDu1C=7L6Pi zbtY@iO88@7Ahrp86wM(IKAL6$FEk=-Y(SL>4RlJJq;r0OBjBU;}3 zAUfl!(D9NuPw-MzMJtI^DycfIXesAcsahggB0FC`e_0l=#Fm1U5}!W$Hb7mJ)C~$c zRkub>OH9&4iqeQQ%=UzSH^Q!sBp_6{%{Ta?$#ae_PcqC%UeOR`{~QRq2D5f(rde4C zoclgxGA9(fV6q?eh|DQUEm{1UuE@|WR0Du(oagR{yCyp@q?j&YITHsObK1O*HR&>d zaZh|J#%f7OGG%SgJ&Z58<&QxuO+bsk@C}a?l_8H;3Bc}yGe*;W7~QEzQgTF<_rOO4 z85S_{JO2ZBI#02ya?hDfl=z}EWe;pl0%4w!#kB1Jn))>1ltVZCgc*TVtnT{Dtzk#L z!bR0&_+l6J6Plu!4^!!YG0E@DDFPup&)cQTs%^kAr5|0_oz0P|HvIsT?4y24TH`=^ z#fW}IxI>W`Lx1q4=9Wk+=;C%k zAot{^n^$cP&H#m_K0#I&fuba2f!nI8NySUuONR}%rW^QrF#H>~NGpt`JlK8L<~74| zjw2x+TC4mD;s}t~>kTWS2d_>%_(j69z>_ zo~=X{Ti5RwIX}~w;LYN7%TSDBQ^{yUG04GrL&%Kmuu)++n#*qZ_gp+Dc0gcmUB6a? z`Zxrb8Rg?0t2US#*9bHj;2(0h6_R0%j0)*c(2WufG?Ku14>_T>3k~;26O+tWp0e1B z+*3k14M6aM$?&)J;2I4sEgq~~jB(}wH#liL{wgDq`&GaB2LkZ)17Cgz9B?83dbShv z8i}Af{^K&Yi)MfWou73G@>R_bX*#zEA;iWHnG0=*c@h*495C1aYftGHQqkJZWIH6r z9;vER^__qy2~&zEn<%fcBgq zI)}``6vQ=rt8(4GDHXcdjxu{BjK2_y?*lst2-Os^eI+Cmj-&nkvbrYFmN;zxj1$Xq zdj}dZRF8bQ_nY+8(wI?GFckJM7WIUlwahT>2xV4idnjp;In>0!9l$WkYe4-oq3I>R z9^`c{d2Fxfjj++oRn(vwbXQ<*&}I!`VNr4U+P7fj9l-{*amxgnIBaSKpw|;Xb}r`q z;!T&x5pGs3%BcE~hV0Q5D;Jjm%~7YRYz`jFX9q4<(^g6*OHoQx#m?cQcB_tsLqbH$ zDrr;_QjEAPlX@mh$&@aX6fEM%D9n%Iz63%a=q1$=2VD?DxzvtIHMLB$s&GS)`-Q~S z)F4_s_D>V{Ycb}t!l(c29_-j92|5StN6_UM+xfwnA{Q14FD2KTo{Y@Bgq79o=!u~Q z*VoOY!={ZN(bx zR<7FU49;r-~|;kGt{x$Nfq{5LWasZPMAb zIcSUupGh7$?PS&(bM%8%fpwVQ=C2Egq(kMVhOy0}F%5<{oMgyINu>*+>p{Ye!ci1q z35?!tTr|~Duq_l4#b7W2+cDWO%Tfg_6=JDi+0MG+xG_<()+-5yIbvxAD8Ka{kL52C z<$AOj(mfL8GUc`%-w%PW)y8<-xl2BylDm?Krbc0uo$I|c!IP8jofGZglp^k&pjdfm zSS^MewOFMh{*{%EBB!@gNL*{e_mr$cSV-?-sW&WNsUNiDz>p}WP{wDVrkB*F@gT$S3WLNCh z>@aE4rnQfOq9P);fkHxvc82)y;Bul1Zfr&})hajvQBm8rae#kBiTV)x2z^wFFb%}V=ap_Fp%HB5!W*XVy& zP%1l8~X}I~)7vRXX(Njx_us!!OVlaQrH_54!wjo;k=ywEt6bE5APCCN6dlozowDcg!J9nGf= z%L4G2e6S1gkhD-e&5T_SH;1y29pbu3tX-C_Pc@44qyAQ@2JRtMj)jvh$KN)Li9M2c zXgxKZqosVeS0li&vJ<&zN?IHK;WM*T|HEg`$WLt2h>h%pY$>cntCNEz)UTB`gdEN` z)M4TugJxo)GVTa4q1zCkoC2(y2;WyfTY~UUDQW$sU%ziE{|j-hthHo}GrBs(UE<@+LJCE4LM@=Y+%gh#Qpk}1lOca8|;veGP5ank}7%yEb z1y3@J#J!%-NOfNvN-D$1rnRu)9oBgNV;4!mbY{G2@-5G$mwONPVmKj{8Z&Ppj7ZL- z3w6rTf|)2DjXC2(fk`5({-~BHjYg1`p2KD=3Sav|+wuyfL%uf>zL%lNMx4djO?896 zx}ruay1{Dzfw?qb7MCp0;D7Af+Up}GC35%Mc{;of#aSArmJ-(tKdDf%D zzUMA7JzG*vNollF%O=Xi)_E#f~nu3+2X zau8VjNYnsZPQ;?p{M=l8iZyNcvs&FO$^>{c*h~T=ayW33y}z|1~tF%8ysl{5J@N;-wpQ|*Mj zLu*3YH^tergs)gr`BzOsHA|04*%K?d3?Kr$PR}~@%d)=smazy*j&;DIF@@CGa})-D zwtY+E3J_*4r>Xm$hUSvPA>}=j)3Nwaa-uOyH7lW!VplvKCvU&mWVaIZ(uijcKEsD_ zz>EBCpQntii>{5XgmMY(DyHk8kL!uiE%Ot(hxV`oOOUQmE^jFxoCEnIhQ`ipYx;Ur zkS2keG5JW}?e*JIUNlxqlavlxpK1%Wi`!Y|)C8$h#9hoX$m3JYg6Z>jITMYg2VqqN zZt}2`qon#SKJor|`7k~<@-H`2FrNGVTUX2wZ^+y#fE0TLR27-JB!a1EzN<4LRAi*O zPE3MQ3BO8%-bvYg1ci!xo~+RZ+%-Fu?TVDprA)u2ikXR#5gW0Du@Z!(RJ52=`bk#) z#CYvR+6+SlJjF=zqC?)Tj|j1Rk7Kr97MI<-oz9CxS0&Nc8iph>Qbgt|zKK&(T#JnG z1{P5$Zx;?MCWjbM(e?Z`PFp`f%uBXaqmu(bBVELlIC)x9NxXF^qyy|;;U?By=`HsY zcn$c*c$y6iWa?+ydp=#QcJx-1RCj!J*@GAeakS47^2CM6KwNBglbtRg6G9VW9mADg zjk_Q&Y8<;&Y$#I)36n6wg?MtAb5-bDlkW{fdaAy zBtAOkDJ?=w<1aB=85eQ5rXu#@)f>v*Z+*r|nx(%=dq}lhZ5`$q!hg5ogX&@NUrzIgb zX3Po!!~vX;CQJqD8K+pr_)f;1Cs{apL$BLX$d%j{7^yrX)6&SbIi9Ze^h7F)H@>%L z8~o(hiTj?*jX&>fX$k8Wsj&n*Q-9~LJVq!7agl-Wo|6rYiHgP?xi=YOqha2HKc$e@BJO zjA>igb1aHKP_cZ?M7JWF7>jbc)7`g(C8_L@@-8H&T};x*QBCZjROBTi3#0(xo)62F zh{$u3*k~zCq{(S3U<>!s^%`E0m6#H3b((kTot&#rWIQLx#GVHE+H7)pdd?a~)LL^p zW8l4Q)!q8fvu1pwl1B3PTk%NOrmg4g=L*d|i8^>YTSR5jthUzeuY|7LwZ{u<=IEDO zOhQP@mprbJ{{*h?5#oxBJ5Lo+OO>Fzs>WPI?>JXOX&D!bXK3v3-X;f7fck~AQ}W2D z{A+4kcaOREks=<}6McNl?$(v%{Yd!XA7RUG!6Ci$hFnyjPB>*Ar*oBoD7hgWCsvb` zDRHZgSH$W-7L>zxFbmWq?OMUcuQ znNiN6g61(wQeVF#x+>;N6CesYQYK;oZ9WNht&ZQc>tn>1kCLp!!}Df5i3j9n4|Mw& zf?~>J(<6d@MyI$db4h4Hv~6Nv!+_RLJ(vw7E1_sB&&4lfzD(TY*cJrkI>yp!&F|HR zN8q@})@j{Q9KTeggWL>ffbYXQr+qcljiqJg4f?@j7?c?7&2uY9E-f$*P60BI%#J%x zw<_{r{tgndnKQSl+xFnTEdCczVT|@xI;zOoKmeX7BO?-wYE-(RzT3Z0XZ8gep;`hg>4n5URiX z>DEYR;$I4zR4pyH^8X|40=$sBENDSX}}~jZ+h+&ZqNF4{u?i{mr(@ zp_Bre7>?mDGNG&_&2nn!HeN?(#{^g1Y;JlYDkh0MyXn};*jahSs^7}tf34wV7eSX%eC*H)sH8!wQcy@VW6aKxkO2wG!qm5hL!5)e@kxl@B;lql z9@53e^Sc3%JZ^YQo2H|Fa&DJPrn58S%=J9+EKu7kSL*?DUAz6Sf(SmHG>*yJ)xp^YyY3!zBOc94~4GW7qz88skpw7tOW*JJv+Vs>;4b56e8e1*u19IVTYUeW<|0<$pz!g2dn#%HxS4UMghe^i zg$JXDP^DvrXf2nyo42QM@nB2ck~T@&gYMn85hp#HWnY;Fn5#m zdA_yPf$Xm3A*3QdbAHGcFmT!`|8)h&5l_i`33m;H4}kLY{uq0*x85(vi~%C9mtLW62$E-MG=!rQa!Ys z5Q#7FcQ3pC#J7P|HhMp0chwwndPyCsHkf&eR>AUZBDF{XTYR~G>72GD%x2*n_Z=8N za}kK4yke>FMFIDzdUqDPbG;sq9VOnJzXj%ID9l1l6lVNGW0H`%hjRaB*m? zqGgwF=rL6j==Mqvc%--xg=9&ul;Zc2h%e`Eup>4cLb95cTNC|yfQL(qu^P^YcX{#MDRQ8qyq zlp1BLv-GBu)3Ck|u;wmeSFdDIqiKUSo=MBwp#8jC=K1?J&Gz!IpBNBaJ=DcLzpY4o zd|nj!em~9sb7lO0PfuFnwJztOX0BBRE&^#1SZmkK5yLXq95>}|6sr_u$l)LSM*J8g zEpB&En>zMCX-c>_P_rzZMaYI9^(x|0E)a{K=Km3A z^n0KT5%+V!E+gUkVGS|=OmpVW91dvLDqsSia^b?8e9EYvlO@Qjqs#Vh7H0Xr;L7PU z?(35JZ`$3FJ7aD1Gg`RFcWQ$(XXL|Er9ewymPvbY_B+$>dF3%Ht#qgB@$2>Jzwuf4 z@?pkJ8!GN7%*L6ITn`&ZA{&M+uMiAgE@7$EW{MeCLSDSK82Cg#Lu4XHH-R8l8L-0h zoyJ%4Vog5CYFlySA2vU_810IJ_%VV5UUZuSWCfivZu6j2h<>M{N zB?M<}FzWr=NCxJlcF%;Vs+E$2^#M!Wc@2GAf7aV5{cNev)sf1@>d==ZhPqvS+K2V; zz-txf^V!aXMA94NSF`iow+^5?Hx1*i$0m6`2igJTW|E_&w%DTGMwaK$U*{`&T$JyW#Z4ZJzv`nO7Z*iPqZxIU7mAk# z<0jhb``ZqU)cnu7`WHzC!F2S)VbNvizBf?`;65|MEg_N3zI=~0W)Lqa4BV#ngVJW- z^w-VWGmB#@84TDL?Zpn{Uqu_sI9kjt22~!Vu{MLoizlt-W@IbR=Srh8zVjQ-`aM?s z4t^;Um@Xa&{}!VLV!Q|i1K4)ZLf=T~W~n}|r+dI`OWkP67}bM$8QIjYW7+b($f+q= z+Fo9&7S%G~ziA{zcw{ z>_cNI*{i?c<(&~;zoqem8uvtQcC~W%nz3_n@?*9A@@8I}2$v%rdoiijqi?aBODF=B zmPV8i$AOfW=R1DzD_zCxCnU&bX^d|2nI6Ny;b;4#0XgdF>%QDyx8=R6 ztwit3t-92&taeHNy425U!(IN>M26|=+~eoF%^GuVhxq#EZ8PhuF+u?2#fhD;j$yXR zIjZs7#_Qx@vC9AV3f0<%h5t^awd#Hm!z4mkFy~`@`sej_f%l6C5iB6LIQ5Tb=#`DXIj=+2M zvO9O}q3^dF-Ur_?=j`9*pOh9~_3tZ4kXdvPHfL&zWAsg$FFt@CIf2b$z*Qm%5sb5= zMe-dNtc;b%$(ZoiFSprTpkZhW@D}*NRN9c4z`aWZ3gYMh+pauD!|s(^7b%k1wAdT5 zPrl@v8xRd2vRR+rRt3h&fVvs6Zl}EsFg@uOGj6#fto7C~dF2^xxCyG<8igJC^&uVI zO_0UgwN?i}xp6CvoAK+-qx{L|W!-t%L1TyoO9mr_Xqcn*k!!+1w92|Rfl(QaxrjnvFI8i-J6;^T&nbr zO`P9wD%=;cxqlTmCag1$pq?l>Y?sQTS$$05Qyy$=jw))~+32S4RV^ADtqK~geMEet z8e?4-nl$a*@)7@bRvuAXG}_qvR%Y)UIxkJ6j2$@|ndpfmsnlB4xGZ&M>KsmBOC%LF zfPrP|=%!3N%fuAvkF*Rsi!@&jC}C%1tIbGTxVVU@G+vh1CE~66+nB{%Dt9<9Ptw&q z$$KVLm4FZDXl^JD7h~yN^f4)J2TuHxph4%l(9xAYtQ9S6wz^Zy$SO7}XdDk*w6SvL z);Ab3D>7cK0f9U+-D$9u3 zx2<$pc8M(weOs*od8)G=4m@u+SEC=@1pnJpJ`%!G?ev$uPz?L9y*5OCrmy=l-ew-x zIS0i~z@pn0clKD@S+oz1(Hl`JsiMg^XF{tR`No6RlTDfslRfg--Doz&mIkty` z$X7l0I!Ch;T(eq{MIj>0+Ef7v%TYgyUWy#tl#mFg47N!IXU?L|;hEg&7q`Ts5BrHg zq;K}LF02TJIBp!ZX%3Sdy;IXz#QkH${o(m}fi9bcU89QltHDClQ*YYE!1j_2$1CcdvuxeewGVF-m&_9^!7Jo8)Cl!#9fUu*254 zn7nWni zG#Bi1<@NnMyLRE(#WTXqbPmnPoE!W)YhB{wgm}B+>xmVM|Gijs?C2)_C!*ZEG99;* zol5UNi8vR1;Z@GF!wK8gl$c%BHqHq8^6bo6Q{J^(LnT&o$DVWDr}BOFOJln)ON2kW zg~lhnnXA0&=;gvx@A>^VkH~agyDI)}+Uh&n3sSh3D$5+dXf(_4%#3OJ%TG4Dr#GkjJXP*fD%?55XR-AwGz%-$!${k^K1vwAG zC*QwZ*TA;ORYto{@OZcKk%^09_qgbYx${n#Q|A~hX6)0J`1e56efQGO?0t?-4~!M% z`lFcFUISip`RhMJAIpWkm`4PGkI{jiTY~SP5@Q3@yXj*mp3gdLz;o<<9hdxWXU{pu zV%Zfhq-sCeV6StVdC9g*l48d#-YMqWM`u~jP;8vP#DD&TP0MPt)^gfZf1mtH;lsg* zw~6!4#WC6+IvJtHwXT;-{Lp@>>f@U<-c7SSaN|~DQs~jjZ-SePKXAvt{aVsuJ#B9# z@4Xws@wKM0Pu{(`!{3v@d_FR^nAiEumrJ{^*Rjl7m~;2l4D-ZD_p|XwpFe64Khd-- zaB|ztuUs|x{s&gs?3)$d{v(Sm?EmR}&r+EyeKQ|4yJNtP2; zv?iRAy%Z)S+adL3Ps#M~^|v#QGS7T_GS&OmC$Ae#Yfig!nTgy<4Cs2zBC%wm>PgkJ z#rk@}6Vjs`tghtle)Xm{Mkh|#d{$Ye&K2@jmqJr#WC%QM5LwL7$ie`4Y!QkW z&@CoL=;j$)8kiYir~{q6gQC~a)X)TYND-Qtu@So849$V+(EJMwD>G9xJAh&q7U=#3 znvU)dkcDPw<{21RnpvQQJy6UX(>&nOOlbBR7#bLuqMK)EUO2*qzej;R^&Y#|VU z#W_$f@X#eRF(X4W4D*bP3^439v9Pd2_pgbiu@QP4nOd5o$2l;93@p*&7np{O(BlZ` zK41)^xzErL)4xW*gQn2UGXkbHbhnsTS{R`F*VMqk7+sy2fw2L)y=Dd`mPqPKiV}fm xO%;I-2@B4wN(E&>{h<8(65#nx;KQeYXK@vmBo>u`b0{zo0W%?&s;aBM8vrN_85{rr literal 0 HcmV?d00001 diff --git a/uncore/index.html b/uncore/index.html new file mode 100644 index 00000000..03f98016 --- /dev/null +++ b/uncore/index.html @@ -0,0 +1 @@ +My GitHub Page diff --git a/uncore/src/main/scala/bigmem.scala b/uncore/src/main/scala/bigmem.scala new file mode 100644 index 00000000..fbbc4c07 --- /dev/null +++ b/uncore/src/main/scala/bigmem.scala @@ -0,0 +1,80 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module +{ + class Inputs extends Bundle { + val addr = UInt(INPUT, log2Up(n)) + val rw = Bool(INPUT) + val wdata = gen.asInput + val wmask = gen.asInput + override def clone = new Inputs().asInstanceOf[this.type] + } + val io = new Bundle { + val in = Valid(new Inputs).flip + val rdata = gen.asOutput + } + val data = gen + val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 + val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 + val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 + if (nDeep > 1 || colMux > 1) + require(isPow2(n) && isPow2(leaf.n)) + + val rdataDeep = Vec.fill(nDeep){Bits()} + val rdataSel = Vec.fill(nDeep){Bool()} + for (i <- 0 until nDeep) { + val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) + val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) + val wdata = in.bits.wdata.toBits + val wmask = in.bits.wmask.toBits + val ren = in.valid && !in.bits.rw + val reg_ren = Reg(next=ren) + val rdata = Vec.fill(nWide){Bits()} + + val r = Pipe(ren, in.bits.addr, postLatency) + + for (j <- 0 until nWide) { + val mem = leaf.clone + var dout: Bits = null + val ridx = if (postLatency > 0) Reg(Bits()) else null + + var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + if (colMux > 1) + wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) + val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) + when (in.valid) { + when (in.bits.rw) { + if (noMask) + mem.write(idx, wdata0) + else + mem.write(idx, wdata0, wmask0) + } + .otherwise { if (postLatency > 0) ridx := idx } + } + + if (postLatency == 0) { + dout = mem(idx) + } else if (postLatency == 1) { + dout = mem(ridx) + } else + dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits + + rdata(j) := dout + } + val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) + + var colMuxOut = rdataWide + if (colMux > 1) { + val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) + colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) + } + + rdataDeep(i) := colMuxOut + rdataSel(i) := r.valid + } + + io.rdata := Mux1H(rdataSel, rdataDeep) +} diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala new file mode 100644 index 00000000..b554b231 --- /dev/null +++ b/uncore/src/main/scala/broadcast.scala @@ -0,0 +1,387 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object L2StoreDataQueueDepth extends Field[Int] + +trait BroadcastHubParameters extends CoherenceAgentParameters { + val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation extends Bundle with BroadcastHubParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt) = { + val d = new DataQueueLocation + d.idx := idx + d.loc := loc + d + } +} + +class L2BroadcastHub extends ManagerCoherenceAgent + with BroadcastHubParameters { + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + // Create SHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id), {case TLDataBits => internalDataBits})) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) + + // Propagate incoherence flags + trackerList.map(_.io.incoherent := io.incoherent.toBits) + + // Queue to store impending Put data + val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) } + val sdq_val = Reg(init=Bits(0, sdqDepth)) + val sdq_alloc_id = PriorityEncoder(~sdq_val) + val sdq_rdy = !sdq_val.andR + val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData() + when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } + + // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) + + val block_acquires = acquireConflicts.orR || !sdq_rdy + io.inner.acquire.ready := acquireReadys.orR && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.bits.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } + + // Queue to store impending Voluntary Release data + val voluntary = io.irel().isVoluntary() + val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() + val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 + when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } + + // Handle releases, which might be voluntary and might have data + val trackerReleaseIOs = trackerList.map(_.io.inner.release) + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) + tracker.bits := io.inner.release.bits + tracker.bits.data := DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits + } + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") + + // Wire probe requests and grant reply to clients, finish acks from clients + // Note that we bypass the Grant data subbundles + io.inner.grant.bits.data := io.outer.grant.bits.data + io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), + { case TLId => params(OuterTLId) + case TLDataBits => internalDataBits }) + outer_arb.io.in <> trackerList.map(_.io.outer) + // Get the pending data out of the store data queue + val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) + val is_in_sdq = outer_data_ptr.loc === inStoreQueue + val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.hasData() && + outer_data_ptr.loc === inStoreQueue + io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer <> outer_arb.io.out + + // Update SDQ valid bits + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } +} + +class BroadcastXactTracker extends XactTracker { + val io = new ManagerXactTrackerIO +} + +class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTracker { + val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val state = Reg(init=s_idle) + + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val coh = ManagerMetadata.onReset + + val collect_irel_data = Reg(init=Bool(false)) + val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + + io.has_acquire_conflict := Bool(false) + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + + io.outer.acquire.valid := Bool(false) + io.outer.grant.ready := Bool(false) + io.inner.acquire.ready := Bool(false) + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.grant.valid := Bool(false) + io.inner.finish.ready := Bool(false) + + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + + //TODO: Use io.outer.release instead? + io.outer.acquire.bits := Bundle( + PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + + when(collect_irel_data) { + io.inner.release.ready := Bool(true) + when(io.inner.release.valid) { + data_buffer(io.irel().addr_beat) := io.irel().data + irel_data_valid(io.irel().addr_beat) := Bool(true) + } + when(irel_data_done) { collect_irel_data := Bool(false) } + } + + switch (state) { + is(s_idle) { + io.inner.release.ready := Bool(true) + when( io.inner.release.valid ) { + xact := io.irel() + data_buffer(UInt(0)) := io.irel().data + collect_irel_data := io.irel().hasMultibeatData() + irel_data_valid := io.irel().hasData() << io.irel().addr_beat + state := Mux(io.irel().hasData(), s_outer, + Mux(io.irel().requiresAck(), s_ack, s_idle)) + } + } + is(s_outer) { + io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) + when(oacq_data_done) { + state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? + } + } + is(s_grant) { // Forward the Grant.voluntaryAck + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(io.inner.grant.fire()) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { + // TODO: This state is unnecessary if no client will ever issue the + // pending Acquire that caused this writeback until it receives the + // Grant.voluntaryAck for this writeback + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} + +class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { + val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) + val state = Reg(init=s_idle) + + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val coh = ManagerMetadata.onReset + + assert(!(state != s_idle && xact.isBuiltInType() && + Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType, + Acquire.prefetchType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO + + val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) + val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) + val curr_p_id = PriorityEncoder(pending_probes) + val full_sharers = coh.full() + val probe_self = io.inner.acquire.bits.requiresSelfProbe() + val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) + val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) + val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) + val mask_incoherent = mask_self & ~io.incoherent.toBits + + val collect_iacq_data = Reg(init=Bool(false)) + val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) + val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) + val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) + val pending_ognt_ack = Reg(init=Bool(false)) + val pending_outer_write = xact.hasData() + val pending_outer_write_ = io.iacq().hasData() + val pending_outer_read = io.ignt().hasData() + val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + + io.has_acquire_conflict := xact.conflicts(io.iacq()) && + (state != s_idle) && + !collect_iacq_data + io.has_acquire_match := xact.conflicts(io.iacq()) && + collect_iacq_data + io.has_release_match := xact.conflicts(io.irel()) && + !io.irel().isVoluntary() && + (state === s_probe) + + val outer_write_acq = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)))(outerTLParams) + val outer_write_rel = Bundle(PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = io.irel().addr_beat, + data = io.irel().data))(outerTLParams) + val outer_read = Bundle(GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block))(outerTLParams) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits := outer_read //default + io.outer.grant.ready := Bool(false) + + io.inner.probe.valid := Bool(false) + io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) + + io.inner.grant.valid := Bool(false) + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent + + io.inner.acquire.ready := Bool(false) + io.inner.release.ready := Bool(false) + io.inner.finish.ready := Bool(false) + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_id != xact.client_id), + "AcquireTracker accepted data beat from different network source than initial request.") + + assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id != xact.client_xact_id), + "AcquireTracker accepted data beat from different client transaction than initial request.") + + assert(!(state === s_idle && io.inner.acquire.fire() && + io.iacq().addr_beat != UInt(0)), + "AcquireTracker initialized with a tail data beat.") + + when(collect_iacq_data) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + data_buffer(io.iacq().addr_beat) := io.iacq().data + iacq_data_valid(io.iacq().addr_beat) := Bool(true) + } + when(iacq_data_done) { collect_iacq_data := Bool(false) } + } + + when(pending_ognt_ack) { + io.outer.grant.ready := Bool(true) + when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } + //TODO add finish queue if this isnt the last level manager + } + + switch (state) { + is(s_idle) { + io.inner.acquire.ready := Bool(true) + when(io.inner.acquire.valid) { + xact := io.iacq() + data_buffer(UInt(0)) := io.iacq().data + collect_iacq_data := io.iacq().hasMultibeatData() + iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat + val needs_probes = mask_incoherent.orR + when(needs_probes) { + pending_probes := mask_incoherent + release_count := PopCount(mask_incoherent) + } + state := Mux(needs_probes, s_probe, + Mux(pending_outer_write_, s_mem_write, + Mux(pending_outer_read_, s_mem_read, s_make_grant))) + } + } + is(s_probe) { + // Generate probes + io.inner.probe.valid := pending_probes.orR + when(io.inner.probe.ready) { + pending_probes := pending_probes & ~UIntToOH(curr_p_id) + } + + // Handle releases, which may have data to be written back + io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready + when(io.inner.release.valid) { + when(io.irel().hasData()) { + io.outer.acquire.valid := Bool(true) + io.outer.acquire.bits := outer_write_rel + when(io.outer.acquire.ready) { + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } .otherwise { + release_count := release_count - UInt(1) + when(release_count === UInt(1)) { + state := Mux(pending_outer_write, s_mem_write, + Mux(pending_outer_read, s_mem_read, s_make_grant)) + } + } + } + } + is(s_mem_write) { // Write data to outer memory + io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.bits := outer_write_acq + when(oacq_data_done) { + pending_ognt_ack := Bool(true) + state := Mux(pending_outer_read, s_mem_read, s_mem_resp) + } + } + is(s_mem_read) { // Read data from outer memory (possibly what was just written) + io.outer.acquire.valid := !pending_ognt_ack + io.outer.acquire.bits := outer_read + when(io.outer.acquire.fire()) { state := s_mem_resp } + } + is(s_mem_resp) { // Wait to forward grants from outer memory + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + when(ignt_data_done) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) + io.inner.grant.valid := Bool(true) + when(io.inner.grant.ready) { + state := Mux(io.ignt().requiresAck(), s_ack, s_idle) + } + } + is(s_ack) { // Wait for transaction to complete + io.inner.finish.ready := Bool(true) + when(io.inner.finish.valid) { state := s_idle } + } + } +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala new file mode 100644 index 00000000..7fcd1408 --- /dev/null +++ b/uncore/src/main/scala/cache.scala @@ -0,0 +1,1078 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import scala.reflect.ClassTag + +case object CacheName extends Field[String] +case object NSets extends Field[Int] +case object NWays extends Field[Int] +case object RowBits extends Field[Int] +case object Replacer extends Field[() => ReplacementPolicy] +case object AmoAluOperandBits extends Field[Int] +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] +case object NPrimaryMisses extends Field[Int] +case object NSecondaryMisses extends Field[Int] +case object CacheBlockBytes extends Field[Int] +case object CacheBlockOffsetBits extends Field[Int] +case object ECCCode extends Field[Option[Code]] + +abstract trait CacheParameters extends UsesParameters { + val nSets = params(NSets) + val blockOffBits = params(CacheBlockOffsetBits) + val idxBits = log2Up(nSets) + val untagBits = blockOffBits + idxBits + val tagBits = params(PAddrBits) - untagBits + val nWays = params(NWays) + val wayBits = log2Up(nWays) + val isDM = nWays == 1 + val rowBits = params(RowBits) + val rowBytes = rowBits/8 + val rowOffBits = log2Up(rowBytes) + val code = params(ECCCode).getOrElse(new IdentityCode) +} + +abstract class CacheBundle extends Bundle with CacheParameters +abstract class CacheModule extends Module with CacheParameters + +class StoreGen(typ: Bits, addr: Bits, dat: Bits) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + lazy val wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { + val t = new StoreGen(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class AMOALU extends CacheModule { + val operandBits = params(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = new StoreGen(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} + +abstract class ReplacementPolicy { + def way: UInt + def miss: Unit + def hit: Unit +} + +class RandomReplacement(ways: Int) extends ReplacementPolicy { + private val replace = Bool() + replace := Bool(false) + val lfsr = LFSR16(replace) + + def way = if(ways == 1) UInt(0) else lfsr(log2Up(ways)-1,0) + def miss = replace := Bool(true) + def hit = {} +} + +abstract class Metadata extends CacheBundle { + val tag = Bits(width = tagBits) + val coh: CoherenceMetadata +} + +class MetaReadReq extends CacheBundle { + val idx = Bits(width = idxBits) +} + +class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { + val way_en = Bits(width = nWays) + val data = gen.clone + override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] +} + +class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { + val rstVal = makeRstVal() + val io = new Bundle { + val read = Decoupled(new MetaReadReq).flip + val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip + val resp = Vec.fill(nWays){rstVal.clone.asOutput} + } + val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) + val rst = rst_cnt < UInt(nSets) + val waddr = Mux(rst, rst_cnt, io.write.bits.idx) + val wdata = Mux(rst, rstVal, io.write.bits.data).toBits + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt + when (rst) { rst_cnt := rst_cnt+UInt(1) } + + val metabits = rstVal.getWidth + val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) + when (rst || io.write.valid) { + tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) + } + + val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) + io.resp := io.resp.fromBits(tags) + io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM + io.write.ready := !rst +} + +abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { + val idxMSB = idxBits-1 + val idxLSB = 0 + val blockAddrBits = params(TLBlockAddrBits) + val refillCyclesPerBeat = outerDataBits/rowBits + val refillCycles = refillCyclesPerBeat*outerDataBeats + val internalDataBeats = params(CacheBlockBytes)*8/rowBits + require(refillCyclesPerBeat == 1) + val amoAluOperandBits = params(AmoAluOperandBits) + require(amoAluOperandBits <= innerDataBits) + require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states + val nSecondaryMisses = params(NSecondaryMisses) + val isLastLevelCache = true + val ignoresWriteMask = !params(ECCCode).isEmpty +} + +abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters +abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { + def doInternalOutputArbitration[T <: Data : ClassTag]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + out <> arb.io.out + arb.io.in <> ins + } + + def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && in.bits.id === UInt(i) } + } +} + +trait HasL2Id extends Bundle with CoherenceAgentParameters { + val id = UInt(width = log2Up(nTransactors + 1)) +} + +trait HasL2InternalRequestState extends L2HellaCacheBundle { + val tag_match = Bool() + val meta = new L2Metadata + val way_en = Bits(width = nWays) +} + +trait HasL2BeatAddr extends L2HellaCacheBundle { + val addr_beat = UInt(width = log2Up(refillCycles)) +} + +trait HasL2Data extends L2HellaCacheBundle + with HasL2BeatAddr { + val data = UInt(width = rowBits) + def hasData(dummy: Int = 0) = Bool(true) + def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) +} + +class L2Metadata extends Metadata with L2HellaCacheParameters { + val coh = new HierarchicalMetadata +} + +object L2Metadata { + def apply(tag: Bits, coh: HierarchicalMetadata) = { + val meta = new L2Metadata + meta.tag := tag + meta.coh := coh + meta + } +} + +class L2MetaReadReq extends MetaReadReq with HasL2Id { + val tag = Bits(width = tagBits) +} + +class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) + with HasL2Id { + override def clone = new L2MetaWriteReq().asInstanceOf[this.type] +} + +class L2MetaResp extends L2HellaCacheBundle + with HasL2Id + with HasL2InternalRequestState + +trait HasL2MetaReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2MetaReadReq) + val resp = Valid(new L2MetaResp).flip +} + +trait HasL2MetaWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2MetaWriteReq) +} + +class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaWriteIO + +class L2MetadataArray extends L2HellaCacheModule { + val io = new L2MetaRWIO().flip + + def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) + val meta = Module(new MetadataArray(onReset _)) + meta.io.read <> io.read + meta.io.write <> io.write + + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) + val s1_id = RegEnable(io.read.bits.id, io.read.valid) + def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) + val s1_clk_en = Reg(next = io.read.fire()) + val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits + val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) + val s2_tag_match = s2_tag_match_way.orR + val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) + + val replacer = params(Replacer)() + val s1_replaced_way_en = UIntToOH(replacer.way) + val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) + val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => + RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) + when(!s2_tag_match) { replacer.miss } + + io.resp.valid := Reg(next = s1_clk_en) + io.resp.bits.id := RegEnable(s1_id, s1_clk_en) + io.resp.bits.tag_match := s2_tag_match + io.resp.bits.meta := Mux(s2_tag_match, + L2Metadata(s2_repl_meta.tag, s2_hit_coh), + s2_repl_meta) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) +} + +class L2DataReadReq extends L2HellaCacheBundle + with HasL2BeatAddr + with HasL2Id { + val addr_idx = UInt(width = idxBits) + val way_en = Bits(width = nWays) +} + +class L2DataWriteReq extends L2DataReadReq + with HasL2Data { + val wmask = Bits(width = rowBits/8) +} + +class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data + +trait HasL2DataReadIO extends L2HellaCacheBundle { + val read = Decoupled(new L2DataReadReq) + val resp = Valid(new L2DataResp).flip +} + +trait HasL2DataWriteIO extends L2HellaCacheBundle { + val write = Decoupled(new L2DataWriteReq) +} + +class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO + +class L2DataArray(delay: Int) extends L2HellaCacheModule { + val io = new L2DataRWIO().flip + + val wmask = FillInterleaved(8, io.write.bits.wmask) + val reg_raddr = Reg(UInt()) + val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) + + when (io.write.bits.way_en.orR && io.write.valid) { + array.write(waddr, io.write.bits.data, wmask) + }.elsewhen (io.read.bits.way_en.orR && io.read.valid) { + reg_raddr := raddr + } + + io.resp.valid := ShiftRegister(io.read.fire(), delay+1) + io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) + io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) + io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) + io.read.ready := !io.write.valid + io.write.ready := Bool(true) +} + +class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { + require(isPow2(nSets)) + require(isPow2(nWays)) + + val meta = Module(new L2MetadataArray) // TODO: add delay knob + val data = Module(new L2DataArray(1)) + val tshrfile = Module(new TSHRFile) + tshrfile.io.inner <> io.inner + io.outer <> tshrfile.io.outer + io.incoherent <> tshrfile.io.incoherent + tshrfile.io.meta <> meta.io + tshrfile.io.data <> data.io +} + +class TSHRFileIO extends HierarchicalTLIO { + val meta = new L2MetaRWIO + val data = new L2DataRWIO +} + +class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { + val io = new TSHRFileIO + + // Create TSHRs for outstanding transactions + val trackerList = (0 until nReleaseTransactors).map(id => Module(new L2VoluntaryReleaseTracker(id))) ++ + (nReleaseTransactors until nTransactors).map(id => Module(new L2AcquireTracker(id))) + + // WritebackUnit evicts data from L2, including invalidating L1s + val wb = Module(new L2WritebackUnit(nTransactors)) + doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) + + // Propagate incoherence flags + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent.toBits } + + // Handle acquire transaction initiation + val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) + val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits + val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits + val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits + val acquire_idx = Mux(acquireMatches.orR, + PriorityEncoder(acquireMatches), + PriorityEncoder(acquireReadys)) + val block_acquires = acquireConflicts.orR + io.inner.acquire.ready := acquireReadys.orR && !block_acquires + trackerAcquireIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.acquire.bits + tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + } + + // Wire releases from clients + val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release + val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits + val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits + val release_idx = PriorityEncoder(releaseMatches) + io.inner.release.ready := releaseReadys(release_idx) + trackerReleaseIOs.zipWithIndex.foreach { + case(tracker, i) => + tracker.bits := io.inner.release.bits + tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) + } + assert(!(io.inner.release.valid && !releaseMatches.orR), + "Non-voluntary release should always have a Tracker waiting for it.") + + // Wire probe requests and grant reply to clients, finish acks from clients + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + // Create an arbiter for the one memory port + val outerList = trackerList.map(_.io.outer) :+ wb.io.outer + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size))(outerTLParams) + outer_arb.io.in <> outerList + io.outer <> outer_arb.io.out + + // Wire local memory arrays + doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doInternalOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) + doInternalOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) + doInternalOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) + doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInternalInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) +} + + +class L2XactTrackerIO extends HierarchicalXactTrackerIO { + val data = new L2DataRWIO + val meta = new L2MetaRWIO + val wb = new L2WritebackIO +} + +abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { + class CacheBlockBuffer { // TODO + val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) + + def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) + def inner = Vec.fill(innerDataBeats){ Bits(width = innerDataBits) }.fromBits(buffer) + def outer = Vec.fill(outerDataBeats){ Bits(width = outerDataBits) }.fromBits(buffer) + } + + def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { + if(data.refillCycles > 1) { + val (multi_cnt, multi_done) = Counter(full_block && inc, data.refillCycles) + (Mux(!full_block, beat, multi_cnt), Mux(!full_block, inc, multi_done)) + } else { (UInt(0), inc) } + } + + def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + in: DecoupledIO[T], + beat: UInt = UInt(0), + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.fire(), in.bits, beat, full_block) + } + + def connectInternalDataBeatCounter[T <: HasL2Data]( + in: ValidIO[T], + full_block: Bool = Bool(true)) = { + connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 + } + + def addPendingBitInternal[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) + + def addPendingBitInternal[T <: HasL2BeatAddr](in: ValidIO[T]) = + Fill(in.bits.refillCycles, in.valid) & UIntToOH(in.bits.addr_beat) + + def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = + ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) + + def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = + ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) + + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits + val isPartial = a.wmask() != Acquire.fullWriteMask + addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) + } + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[T] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[T] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } +} + +class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + pinAllReadyValidLow(io) + + val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_old_meta = Reg{ new L2Metadata } + val coh = xact_old_meta.coh + + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) + + val all_pending_done = + !(pending_writes.orR || + pending_ignt) + + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := state === s_idle || pending_irels.orR + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } + + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + + // Write the voluntarily written back data to this cache + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && pending_writes.orR + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.data := data_buffer(curr_write_beat) + + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels + io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + + // End a transaction by updating the block metadata + io.meta.write.valid := state === s_meta_write + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) + io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), + xact_old_meta.coh.outer.onHit(M_XWR), + xact_old_meta.coh.outer) + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.valid) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) + } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_meta_read + } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + state := s_busy + } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && io.meta.write.ready) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_release_match := io.irel().isVoluntary() + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // Checks for illegal behavior + assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), + "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") +} + + +class L2AcquireTracker(trackerId: Int) extends L2XactTracker { + val io = new L2XactTrackerIO + pinAllReadyValidLow(io) + + val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) + val state = Reg(init=s_idle) + + // State holding transaction metadata + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } + val xact_tag_match = Reg{ Bool() } + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_old_meta = Reg{ new L2Metadata } + val pending_coh = Reg{ xact_old_meta.coh.clone } + + // Secondary miss queue + val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) + + // State holding progress made on processing this transaction + val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + val pending_irels = connectTwoWayBeatCounter( + max = io.inner.tlNCachingClients, + up = io.inner.probe, + down = io.inner.release)._1 + val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = + connectTwoWayBeatCounter( + max = 1, + up = io.outer.acquire, + down = io.outer.grant, + beat = xact.addr_beat) + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + val pending_ifins = connectTwoWayBeatCounter( + max = nSecondaryMisses, + up = io.inner.grant, + down = io.inner.finish, + track = (g: Grant) => g.requiresAck())._1 + val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_meta_write = Reg{ Bool() } + + val all_pending_done = + !(pending_reads.orR || + pending_writes.orR || + pending_resps.orR || + pending_puts.orR || + pending_ognt || + ignt_q.io.count > UInt(0) || + //pending_meta_write || // Has own state: s_meta_write + pending_ifins) + + // Provide a single ALU per tracker to merge Puts and AMOs with data being + // refilled, written back, or extant in the cache + val amoalu = Module(new AMOALU) + amoalu.io.addr := xact.addr() + amoalu.io.cmd := xact.op_code() + amoalu.io.typ := xact.op_size() + amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData + val amo_result = xact.data // Reuse xact buffer space to store AMO result + + // Utility functions for updating the data and metadata that will be kept in + // the cache or granted to the original requestor after this transaction: + + def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { + when(flag && pending_coh != next) { + pending_meta_write := Bool(true) + pending_coh := next + } + } + + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact.amo_shift_bits() + amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val wmask = FillInterleaved(8, wmask_buffer(beat)) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), + amoalu.io.out << xact.amo_shift_bits(), + new_data) + wmask_buffer(beat) := SInt(-1) + when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } + } + def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } + } + def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + + // Actual transaction processing logic begins here: + // + // First, take care of accpeting new requires or secondary misses + // For now, the only allowed secondary miss types are Gets-under-Get + // and Puts-under-Put from the same client + val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && + io.iacq().isBuiltInType(Acquire.getType)) && + xact.client_id === io.iacq().client_id && //TODO remove + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready + + // This logic also allows the tail beats of a PutBlock to be merged in + val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && + io.iacq().isBuiltInType(Acquire.putType)) || + (xact.isBuiltInType(Acquire.putBlockType) && + io.iacq().isBuiltInType(Acquire.putBlockType))) && + xact.client_id === io.iacq().client_id && //TODO remove + xact.conflicts(io.iacq()) && + state != s_idle && state != s_meta_write && + !all_pending_done && + xact.allocate() && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready + + io.inner.acquire.ready := state === s_idle || + can_merge_iacq_put || + can_merge_iacq_get + + // Enqueue secondary miss information + ignt_q.io.enq.valid := iacq_data_done + ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id + ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat + // TODO add ignt.dst <- iacq.src + + // Track whether any beats are missing from a PutBlock + pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + + // Begin a transaction by getting the current block metadata + io.meta.read.valid := state === s_meta_read + io.meta.read.bits.id := UInt(trackerId) + io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + + // Issue a request to the writeback unit + io.wb.req.valid := state === s_wb_req + io.wb.req.bits.id := UInt(trackerId) + io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.tag := xact_old_meta.tag + io.wb.req.bits.coh := xact_old_meta.coh + io.wb.req.bits.way_en := xact_way_en + + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + io.inner.release.ready := state === s_inner_probe + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease(io.irel()), // Drop sharer + Mux(io.irel().hasData(), // Dirty writeback + pending_coh.outer.onHit(M_XWR), + pending_coh.outer)) + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + mergeDataInner(io.inner.release) + + // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: + // + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + io.outer.acquire.valid := state === s_outer_acquire + io.outer.acquire.bits := Mux( + xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( + client_xact_id = UInt(0), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + + // Handle the response from outer memory + io.outer.grant.ready := state === s_busy + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.outer.grant.bits, xact.op_code())) + updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) + mergeDataOuter(io.outer.grant) + + // Going back to the original inner transaction, we can issue a Grant as + // soon as the data is released, granted, put, or read from the cache + pending_ignt_data := pending_ignt_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) | + addPendingBitInternal(io.data.resp) + ignt_q.io.deq.ready := ignt_data_done + io.inner.grant.valid := state === s_busy && + ignt_q.io.deq.valid && + (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) + // Make the Grant message using the data stored in the secondary miss queue + io.inner.grant.bits := pending_coh.inner.makeGrant( + pri = xact, + sec = ignt_q.io.deq.bits, + manager_xact_id = UInt(trackerId), + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) + io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter + + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant(io.ignt()), + Mux(ognt_data_done, + pending_coh_on_ognt.outer, + pending_coh.outer)) + updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) + + // We must wait for as many Finishes as we sent Grants + io.inner.finish.ready := state === s_busy + + // We read from the the cache at this level if data wasn't written back or refilled. + // We may merge Gets, requiring further beats to be read. + // If ECC requires a full writemask, we'll read out data on partial writes as well. + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)) | + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.read.bits.addr_beat := curr_read_beat + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + mergeDataInternal(io.data.resp) + + // We write data to the cache at this level if it was Put here with allocate flag, + // written back dirty, or refilled from outer memory. + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasData(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && + pending_writes.orR && + !pending_ognt && + !pending_reads(curr_write_beat) && + !pending_resps(curr_write_beat) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := wmask_buffer(curr_write_beat) + io.data.write.bits.data := data_buffer(curr_write_beat) + + // End a transaction by updating the block metadata + io.meta.write.valid := state === s_meta_write + io.meta.write.bits.id := UInt(trackerId) + io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.way_en := xact_way_en + io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.coh := pending_coh + + // Handling of secondary misses (Gets and Puts only for now) + when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie + val beat = io.iacq().addr_beat + val wmask = io.iacq().wmask() + val full = FillInterleaved(8, wmask) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) + } + + // Defined here because of Chisel default wire demands, used in s_meta_resp + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.acquire.valid) { + xact := io.iacq() + xact.data := UInt(0) + pending_puts := Mux( // Make sure to collect all data from a PutBlock + io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenBeatHasData(io.inner.acquire), + UInt(0)) + pending_reads := Mux( // GetBlocks and custom types read all beats + io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), + SInt(-1, width = innerDataBeats), + (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) + pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) + pending_resps := UInt(0) + pending_ignt_data := UInt(0) + pending_meta_write := UInt(0) + state := s_meta_read + } + when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + when(state === s_meta_resp && io.meta.resp.valid) { + xact_tag_match := io.meta.resp.bits.tag_match + xact_old_meta := io.meta.resp.bits.meta + xact_way_en := io.meta.resp.bits.way_en + val coh = io.meta.resp.bits.meta.coh + val tag_match = io.meta.resp.bits.tag_match + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || + tag_match && coh.outer.isValid()) + val needs_writeback = !tag_match && + xact.allocate() && + (coh.outer.requiresVoluntaryWriteback() || + coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) + when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } + pending_coh := Mux(is_hit, pending_coh_on_hit, coh) + when(needs_inner_probes) { + val full_sharers = coh.inner.full() + val mask_self = Mux( + xact.requiresSelfProbe(), + coh.inner.full() | UIntToOH(xact.client_id), + coh.inner.full() & ~UIntToOH(xact.client_id)) + val mask_incoherent = mask_self & ~io.incoherent.toBits + pending_iprbs := mask_incoherent + } + state := Mux(needs_writeback, s_wb_req, + Mux(needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, s_busy))) + } + when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } + when(state === s_wb_resp && io.wb.resp.valid) { + // If we're overwriting the whole block in a last level cache we can + // just do it without fetching any data from memory + val skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + // Tag matches, so if this is the last level cache we can use the data without upgrading permissions + val skip_outer_acquire = + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact.op_code()) + else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) + state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) + } + when(state === s_outer_acquire && oacq_data_done) { state := s_busy } + when(state === s_busy && all_pending_done) { state := s_meta_write } + when(state === s_meta_write && (io.meta.write.ready || !pending_meta_write)) { + wmask_buffer.foreach { w => w := UInt(0) } + state := s_idle + } + + // These IOs are used for routing in the parent + val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) + io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && io.inner.release.ready + io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get + io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match + //TODO: relax from in_same_set to xact.conflicts(io.iacq())? + + // Checks for illegal behavior + assert(!(state != s_idle && io.inner.acquire.fire() && + io.inner.acquire.bits.client_id != xact.client_id), + "AcquireTracker accepted data beat from different network source than initial request.") +} + +class L2WritebackReq extends L2Metadata with HasL2Id { + val idx = Bits(width = idxBits) + val way_en = Bits(width = nWays) +} + +class L2WritebackResp extends L2HellaCacheBundle with HasL2Id + +class L2WritebackIO extends L2HellaCacheBundle { + val req = Decoupled(new L2WritebackReq) + val resp = Valid(new L2WritebackResp).flip +} + +class L2WritebackUnitIO extends HierarchicalXactTrackerIO { + val wb = new L2WritebackIO().flip + val data = new L2DataRWIO +} + +class L2WritebackUnit(trackerId: Int) extends L2XactTracker { + val io = new L2WritebackUnitIO + pinAllReadyValidLow(io) + + val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + + val xact = Reg(new L2WritebackReq) + val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val xact_addr_block = Cat(xact.tag, xact.idx) + + val pending_irels = + connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 + val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = + connectTwoWayBeatCounter(max = 1, up = io.outer.release, down = io.outer.grant) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) + val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + + // Start the writeback sub-transaction + io.wb.req.ready := state === s_idle + + // Track which clients yet need to be probed and make Probe message + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := xact.coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data + val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) + val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) + io.inner.release.ready := state === s_inner_probe || state === s_busy + when(io.inner.release.fire()) { + xact.coh.inner := inner_coh_on_irel + when(io.irel().hasData()) { xact.coh.outer := outer_coh_on_irel } // WB is a write + data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data + } + + // If a release didn't write back data, have to read it from data array + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release)) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact.way_en + io.data.read.bits.addr_idx := xact.idx + io.data.read.bits.addr_beat := curr_read_beat + io.data.write.valid := Bool(false) + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + when(io.data.resp.valid) { + data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data + } + + // Once the data is buffered we can write it back to outer memory + pending_orel_data := pending_orel_data | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitInternal(io.data.resp) + io.outer.release.valid := state === s_busy && + (!io.orel().hasData() || pending_orel_data(orel_data_idx)) + io.outer.release.bits := xact.coh.outer.makeVoluntaryWriteback( + client_xact_id = UInt(trackerId), + addr_block = xact_addr_block, + addr_beat = orel_data_idx, + data = data_buffer(orel_data_idx)) + + // Wait for an acknowledgement + io.outer.grant.ready := state === s_outer_grant + + // Respond to the initiating transaction handler signalling completion of the writeback + io.wb.resp.valid := state === s_wb_resp + io.wb.resp.bits.id := xact.id + + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.wb.req.valid) { + xact := io.wb.req.bits + val coh = io.wb.req.bits.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } + pending_reads := SInt(-1, width = innerDataBeats) + pending_resps := UInt(0) + pending_orel_data := UInt(0) + state := Mux(needs_inner_probes, s_inner_probe, s_busy) + } + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) + } + when(state === s_busy && orel_data_done) { + state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) + } + when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } + when(state === s_wb_resp ) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) +} diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala new file mode 100644 index 00000000..862eb484 --- /dev/null +++ b/uncore/src/main/scala/coherence.scala @@ -0,0 +1,688 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +/** The entire CoherencePolicy API consists of the following three traits: + * HasCustomTileLinkMessageTypes, used to define custom messages + * HasClientSideCoherencePolicy, for client coherence agents + * HasManagerSideCoherencePolicy, for manager coherence agents + */ +abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends + HasCustomTileLinkMessageTypes with + HasClientSideCoherencePolicy with + HasManagerSideCoherencePolicy + +/** This API defines the custom, coherence-policy-defined message types, + * as opposed to the built-in ones found in tilelink.scala. + * Policies must enumerate the custom messages to be sent over each + * channel, as well as which of them have associated data. + */ +trait HasCustomTileLinkMessageTypes { + val nAcquireTypes: Int + def acquireTypeWidth = log2Up(nAcquireTypes) + val nProbeTypes: Int + def probeTypeWidth = log2Up(nProbeTypes) + val nReleaseTypes: Int + def releaseTypeWidth = log2Up(nReleaseTypes) + val nGrantTypes: Int + def grantTypeWidth = log2Up(nGrantTypes) + + val acquireTypesWithData = Nil // Only built-in Acquire types have data for now + val releaseTypesWithData: Vec[UInt] + val grantTypesWithData: Vec[UInt] +} + +/** This API contains all functions required for client coherence agents. + * Policies must enumerate the number of client states and define their + * permissions with respect to memory operations. Policies must fill in functions + * to control which messages are sent and how metadata is updated in response + * to coherence events. These funtions are generally called from within the + * ClientMetadata class in metadata.scala + */ +trait HasClientSideCoherencePolicy { + // Client coherence states and their permissions + val nClientStates: Int + def clientStateWidth = log2Ceil(nClientStates) + val clientStatesWithReadPermission: Vec[UInt] + val clientStatesWithWritePermission: Vec[UInt] + val clientStatesWithDirtyData: Vec[UInt] + + // Transaction initiation logic + def isValid(meta: ClientMetadata): Bool + def isHit(cmd: UInt, meta: ClientMetadata): Bool = { + Mux(isWriteIntent(cmd), + clientStatesWithWritePermission.contains(meta.state), + clientStatesWithReadPermission.contains(meta.state)) + } + //TODO: Assumes all states with write permissions also have read permissions + def requiresAcquireOnSecondaryMiss( + first_cmd: UInt, + second_cmd: UInt, + meta: ClientMetadata): Bool = { + isWriteIntent(second_cmd) && !isWriteIntent(first_cmd) + } + //TODO: Assumes all cache ctrl ops writeback dirty data, and + // doesn't issue transaction when e.g. downgrading Exclusive to Shared: + def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = + clientStatesWithDirtyData.contains(meta.state) + + // Determine which custom message type to use + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt + def getReleaseType(p: Probe, meta: ClientMetadata): UInt + + // Mutate ClientMetadata based on messages or cmds + def clientMetadataOnReset: ClientMetadata + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata +} + +/** This API contains all functions required for manager coherence agents. + * Policies must enumerate the number of manager states. Policies must fill + * in functions to control which Probe and Grant messages are sent and how + * metadata should be updated in response to coherence events. These funtions + * are generally called from within the ManagerMetadata class in metadata.scala + */ +trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { + val nManagerStates: Int + def masterStateWidth = log2Ceil(nManagerStates) + + // Transaction probing logic + def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool + + // Determine which custom message type to use in response + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt + def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt + def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + + // Mutate ManagerMetadata based on messages or cmds + def managerMetadataOnReset: ManagerMetadata + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached + meta.sharers, + dir.push(meta.sharers, dst))) + //state = meta.state) TODO: Fix 0-width wires in Chisel +} + +/** The following concrete implementations of CoherencePolicy each provide the + * functionality of one particular protocol. + */ + +/** A simple protocol with only two Client states. + * Data is always assumed to be dirty. + * Only a single client may ever have a copy of a block at a time. + */ +class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 1 + val nProbeTypes = 2 + val nReleaseTypes = 4 + val nGrantTypes = 1 + + val acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) + + val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) + + // Client states and functions + val nClientStates = 2 + val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientValid) + val clientStatesWithWritePermission = Vec(clientValid) + val clientStatesWithDirtyData = Vec(clientValid) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) + + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata(Mux(incoming.p_type === probeInvalidate, + clientInvalid, meta.state)) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } +} + +/** A simple protocol with only three Client states. + * Data is marked as dirty when written. + * Only a single client may ever have a copy of a block at a time. + */ +class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 1 + + val acquireExclusiveClean :: acquireExclusiveDirty :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) + + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantExclusive) + + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata) = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) + + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientExclusiveClean, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // We don't actually need any states for this protocol + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + probeInvalidate) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } +} + +/** A protocol with only three Client states. + * Data is always assumed to be dirty. + * Multiple clients may share read permissions on a block at the same time. + */ +class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 + + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) + + // Client states and functions + val nClientStates = 3 + val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) + + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> clientExclusiveDirty, + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + // Also could avoid probes on outer WBs. + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } +} + +/** A protocol with four Client states. + * Data is marked as dirty when written. + * Multiple clients may share read permissions on a block at the same time. + */ +class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 2 + val nProbeTypes = 3 + val nReleaseTypes = 6 + val nGrantTypes = 3 + + val acquireShared :: acquireExclusive :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) + + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + val grantTypesWithData = Vec(grantShared, grantExclusive) + + // Client states and functions + val nClientStates = 4 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> getReleaseType(M_FLUSH, meta), + probeDowngrade -> getReleaseType(M_PRODUCE, meta), + probeCopy -> getReleaseType(M_CLEAN, meta))) + + def clientMetadataOnReset = ClientMetadata(clientInvalid) + + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty)))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeDowngrade -> clientShared, + probeCopy -> meta.state))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing + // only a single sharer (also would need + // notification msg to track clean drops) + // Also could avoid probes on outer WBs. + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive) + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped)) + } +} + +class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { + // Message types + val nAcquireTypes = 3 + val nProbeTypes = 4 + val nReleaseTypes = 10 + val nGrantTypes = 4 + + val acquireShared :: acquireExclusive :: acquireInvalidateOthers :: Nil = Enum(UInt(), nAcquireTypes) + val probeInvalidate :: probeDowngrade :: probeCopy :: probeInvalidateOthers :: Nil = Enum(UInt(), nProbeTypes) + val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) + val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) + + val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + + // Client states and functions + val nClientStates = 7 + val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) + + val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + + def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + + def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = + Mux(isWriteIntent(cmd), + Mux(meta.state === clientInvalid, acquireExclusive, acquireInvalidateOthers), + acquireShared) + + def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + MuxLookup(cmd, releaseCopyAck, Array( + M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), + M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), + M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) + } + + def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + val dirty = clientStatesWithDirtyData.contains(meta.state) + val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( + probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + releaseInvalidateDataMigratory, releaseInvalidateData), + probeDowngrade -> Mux(meta.state === clientMigratoryDirty, + releaseDowngradeDataMigratory, releaseDowngradeData), + probeCopy -> releaseCopyData)) + val without_data = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( + probeInvalidate -> Mux(clientExclusiveClean === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, + releaseInvalidateAckMigratory, releaseInvalidateAck), + probeDowngrade -> Mux(meta.state != clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> releaseCopyAck)) + Mux(dirty, with_data, without_data) + } + + def clientMetadataOnReset = ClientMetadata(clientInvalid) + + def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( + clientExclusiveClean -> clientExclusiveDirty, + clientMigratoryClean -> clientMigratoryDirty)), + meta.state)) + + def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(cmd, meta.state, Array( + M_FLUSH -> clientInvalid, + M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + clientShared, meta.state), + M_CLEAN -> MuxLookup(meta.state, meta.state, Array( + clientExclusiveDirty -> clientExclusiveClean, + clientMigratoryDirty -> clientMigratoryClean))))) + + def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + ClientMetadata( + Mux(incoming.isBuiltInType(), clientInvalid, + MuxLookup(incoming.g_type, clientInvalid, Array( + grantShared -> clientShared, + grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), + grantExclusiveAck -> clientExclusiveDirty, + grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + + def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + ClientMetadata( + MuxLookup(incoming.p_type, meta.state, Array( + probeInvalidate -> clientInvalid, + probeInvalidateOthers -> clientInvalid, + probeCopy -> meta.state, + probeDowngrade -> MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid))))) + + // Manager states and functions: + val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types + + def requiresProbes(a: Acquire, meta: ManagerMetadata) = + Mux(dir.none(meta.sharers), Bool(false), + Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive + Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + + def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) + + def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = + MuxLookup(cmd, probeCopy, Array( + M_FLUSH -> probeInvalidate, + M_PRODUCE -> probeDowngrade)) + + def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), + MuxLookup(a.a_type, probeCopy, Array( + Acquire.getBlockType -> probeCopy, + Acquire.putBlockType -> probeInvalidate, + Acquire.getType -> probeCopy, + Acquire.putType -> probeInvalidate, + Acquire.putAtomicType -> probeInvalidate)), + MuxLookup(a.a_type, probeCopy, Array( + acquireShared -> probeDowngrade, + acquireExclusive -> probeInvalidate, + acquireInvalidateOthers -> probeInvalidateOthers))) + + def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + MuxLookup(a.a_type, grantShared, Array( + acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + acquireExclusive -> grantExclusive, + acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? + + def managerMetadataOnReset = ManagerMetadata() + + def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + MuxBundle(meta, Array( + incoming.is(releaseInvalidateData) -> popped, + incoming.is(releaseInvalidateAck) -> popped, + incoming.is(releaseInvalidateDataMigratory) -> popped, + incoming.is(releaseInvalidateAckMigratory) -> popped)) + } +} diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala new file mode 100644 index 00000000..69cdc41f --- /dev/null +++ b/uncore/src/main/scala/consts.scala @@ -0,0 +1,48 @@ +// See LICENSE for license details. + +package uncore +package constants + +import Chisel._ + +object MemoryOpConstants extends MemoryOpConstants +trait MemoryOpConstants { + val MT_SZ = 3 + val MT_X = Bits("b???") + val MT_B = Bits("b000") + val MT_H = Bits("b001") + val MT_W = Bits("b010") + val MT_D = Bits("b011") + val MT_BU = Bits("b100") + val MT_HU = Bits("b101") + val MT_WU = Bits("b110") + + val NUM_XA_OPS = 9 + val M_SZ = 5 + val M_X = Bits("b?????"); + val M_XRD = Bits("b00000"); // int load + val M_XWR = Bits("b00001"); // int store + val M_PFR = Bits("b00010"); // prefetch with intent to read + val M_PFW = Bits("b00011"); // prefetch with intent to write + val M_XA_SWAP = Bits("b00100"); + val M_NOP = Bits("b00101"); + val M_XLR = Bits("b00110"); + val M_XSC = Bits("b00111"); + val M_XA_ADD = Bits("b01000"); + val M_XA_XOR = Bits("b01001"); + val M_XA_OR = Bits("b01010"); + val M_XA_AND = Bits("b01011"); + val M_XA_MIN = Bits("b01100"); + val M_XA_MAX = Bits("b01101"); + val M_XA_MINU = Bits("b01110"); + val M_XA_MAXU = Bits("b01111"); + val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions + val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions + + def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP + def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW + def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) + def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR +} diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/directory.scala new file mode 100644 index 00000000..db555ad3 --- /dev/null +++ b/uncore/src/main/scala/directory.scala @@ -0,0 +1,43 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +// This class encapsulates transformations on different directory information +// storage formats +abstract class DirectoryRepresentation(val width: Int) { + def pop(prev: UInt, id: UInt): UInt + def push(prev: UInt, id: UInt): UInt + def flush: UInt + def none(s: UInt): Bool + def one(s: UInt): Bool + def count(s: UInt): UInt + def next(s: UInt): UInt + def full(s: UInt): UInt +} + +abstract trait HasDirectoryRepresentation { + val dir: DirectoryRepresentation +} + +class NullRepresentation(nClients: Int) extends DirectoryRepresentation(1) { + def pop(prev: UInt, id: UInt) = UInt(0) + def push(prev: UInt, id: UInt) = UInt(0) + def flush = UInt(0) + def none(s: UInt) = Bool(false) + def one(s: UInt) = Bool(false) + def count(s: UInt) = UInt(nClients) + def next(s: UInt) = UInt(0) + def full(s: UInt) = SInt(-1, width = nClients).toUInt +} + +class FullRepresentation(nClients: Int) extends DirectoryRepresentation(nClients) { + def pop(prev: UInt, id: UInt) = prev & ~UIntToOH(id) + def push(prev: UInt, id: UInt) = prev | UIntToOH(id) + def flush = UInt(0, width = width) + def none(s: UInt) = s === UInt(0) + def one(s: UInt) = PopCount(s) === UInt(1) + def count(s: UInt) = PopCount(s) + def next(s: UInt) = PriorityEncoder(s) + def full(s: UInt) = s +} diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala new file mode 100644 index 00000000..b5864b2c --- /dev/null +++ b/uncore/src/main/scala/ecc.scala @@ -0,0 +1,146 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ + +abstract class Decoding +{ + def uncorrected: Bits + def corrected: Bits + def correctable: Bool + def uncorrectable: Bool + def error = correctable || uncorrectable +} + +abstract class Code +{ + def width(w0: Int): Int + def encode(x: Bits): Bits + def decode(x: Bits): Decoding +} + +class IdentityCode extends Code +{ + def width(w0: Int) = w0 + def encode(x: Bits) = x + def decode(y: Bits) = new Decoding { + def uncorrected = y + def corrected = y + def correctable = Bool(false) + def uncorrectable = Bool(false) + } +} + +class ParityCode extends Code +{ + def width(w0: Int) = w0+1 + def encode(x: Bits) = Cat(x.xorR, x) + def decode(y: Bits) = new Decoding { + def uncorrected = y(y.getWidth-2,0) + def corrected = uncorrected + def correctable = Bool(false) + def uncorrectable = y.xorR + } +} + +class SECCode extends Code +{ + def width(k: Int) = { + val m = new Unsigned(k).log2 + 1 + k + m + (if((1 << m) < m+k+1) 1 else 0) + } + def encode(x: Bits) = { + val k = x.getWidth + require(k > 0) + val n = width(k) + + val y = for (i <- 1 to n) yield { + if (isPow2(i)) { + val r = for (j <- 1 to n; if j != i && (j & i) != 0) + yield x(mapping(j)) + r reduce (_^_) + } else + x(mapping(i)) + } + Vec(y).toBits + } + def decode(y: Bits) = new Decoding { + val n = y.getWidth + require(n > 0 && !isPow2(n)) + + val p2 = for (i <- 0 until log2Up(n)) yield 1 << i + val syndrome = p2 map { i => + val r = for (j <- 1 to n; if (j & i) != 0) + yield y(j-1) + r reduce (_^_) + } + val s = Vec(syndrome).toBits + + private def swizzle(z: Bits) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits + def uncorrected = swizzle(y) + def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) + def correctable = s.orR + def uncorrectable = Bool(false) + } + private def mapping(i: Int) = i-1-log2Up(i) +} + +class SECDEDCode extends Code +{ + private val sec = new SECCode + private val par = new ParityCode + + def width(k: Int) = sec.width(k)+1 + def encode(x: Bits) = par.encode(sec.encode(x)) + def decode(x: Bits) = new Decoding { + val secdec = sec.decode(x(x.getWidth-2,0)) + val pardec = par.decode(x) + + def uncorrected = secdec.uncorrected + def corrected = secdec.corrected + def correctable = pardec.uncorrectable + def uncorrectable = !pardec.uncorrectable && secdec.correctable + } +} + +object ErrGen +{ + // generate a 1-bit error with approximate probability 2^-f + def apply(width: Int, f: Int): Bits = { + require(width > 0 && f >= 0 && log2Up(width) + f <= 16) + UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) + } + def apply(x: Bits, f: Int): Bits = x ^ apply(x.getWidth, f) +} + +class SECDEDTest extends Module +{ + val code = new SECDEDCode + val k = 4 + val n = code.width(k) + + val io = new Bundle { + val original = Bits(OUTPUT, k) + val encoded = Bits(OUTPUT, n) + val injected = Bits(OUTPUT, n) + val uncorrected = Bits(OUTPUT, k) + val corrected = Bits(OUTPUT, k) + val correctable = Bool(OUTPUT) + val uncorrectable = Bool(OUTPUT) + } + + val c = Counter(Bool(true), 1 << k) + val numErrors = Counter(c._2, 3)._1 + val e = code.encode(c._1) + val i = e ^ Mux(numErrors < UInt(1), UInt(0), ErrGen(n, 1)) ^ Mux(numErrors < UInt(2), UInt(0), ErrGen(n, 1)) + val d = code.decode(i) + + io.original := c._1 + io.encoded := e + io.injected := i + io.uncorrected := d.uncorrected + io.corrected := d.corrected + io.correctable := d.correctable + io.uncorrectable := d.uncorrectable +} diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala new file mode 100644 index 00000000..8a79cda1 --- /dev/null +++ b/uncore/src/main/scala/htif.scala @@ -0,0 +1,255 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import Node._ +import uncore._ + +case object HTIFWidth extends Field[Int] +case object HTIFNSCR extends Field[Int] +case object HTIFOffsetBits extends Field[Int] +case object HTIFNCores extends Field[Int] + +abstract trait HTIFParameters extends UsesParameters { + val dataBits = params(TLDataBits) + val dataBeats = params(TLDataBeats) + val w = params(HTIFWidth) + val nSCR = params(HTIFNSCR) + val offsetBits = params(HTIFOffsetBits) + val nCores = params(HTIFNCores) +} + +abstract class HTIFBundle extends Bundle with HTIFParameters + +class HostIO extends HTIFBundle +{ + val clk = Bool(OUTPUT) + val clk_edge = Bool(OUTPUT) + val in = Decoupled(Bits(width = w)).flip + val out = Decoupled(Bits(width = w)) + val debug_stats_pcr = Bool(OUTPUT) +} + +class PCRReq extends Bundle +{ + val rw = Bool() + val addr = Bits(width = 12) + val data = Bits(width = 64) +} + +class HTIFIO extends HTIFBundle { + val reset = Bool(INPUT) + val id = UInt(INPUT, log2Up(nCores)) + val pcr_req = Decoupled(new PCRReq).flip + val pcr_rep = Decoupled(Bits(width = 64)) + val ipi_req = Decoupled(Bits(width = log2Up(nCores))) + val ipi_rep = Decoupled(Bool()).flip + val debug_stats_pcr = Bool(OUTPUT) + // wired directly to stats register + // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work +} + +class SCRIO extends HTIFBundle { + val rdata = Vec.fill(nSCR){Bits(INPUT, 64)} + val wen = Bool(OUTPUT) + val waddr = UInt(OUTPUT, log2Up(nSCR)) + val wdata = Bits(OUTPUT, 64) +} + +class HTIFModuleIO extends HTIFBundle { + val host = new HostIO + val cpu = Vec.fill(nCores){new HTIFIO}.flip + val mem = new ClientUncachedTileLinkIO + val scr = new SCRIO +} + +class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { + val io = new HTIFModuleIO + + io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) + // system is 'interesting' if any tile is 'interesting' + + val short_request_bits = 64 + val long_request_bits = short_request_bits + dataBits*dataBeats + require(short_request_bits % w == 0) + + val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits + val rx_count = Reg(init=UInt(0,rx_count_w)) + val rx_shifter = Reg(Bits(width = short_request_bits)) + val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) + val next_cmd = rx_shifter_in(3,0) + val cmd = Reg(Bits()) + val size = Reg(Bits()) + val pos = Reg(Bits()) + val seqno = Reg(Bits()) + val addr = Reg(Bits()) + when (io.host.in.valid && io.host.in.ready) { + rx_shifter := rx_shifter_in + rx_count := rx_count + UInt(1) + when (rx_count === UInt(short_request_bits/w-1)) { + cmd := next_cmd + size := rx_shifter_in(15,4) + pos := rx_shifter_in(15,4+offsetBits-3) + seqno := rx_shifter_in(23,16) + addr := rx_shifter_in(63,24) + } + } + + val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) + val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR + val packet_ram_depth = long_request_bits/short_request_bits-1 + val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) + when (rx_word_done && io.host.in.ready) { + packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in + } + + val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) + + val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) + val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) + val pcr_wdata = packet_ram(0) + + val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR + val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), + Bool(true))) + + val tx_count = Reg(init=UInt(0, rx_count_w)) + val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) + val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) + val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) + when (io.host.out.valid && io.host.out.ready) { + tx_count := tx_count + UInt(1) + } + + val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd != cmd_writemem && next_cmd != cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) + val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) + val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) + + val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) + val state = Reg(init=state_rx) + + val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || + (state === state_mem_rresp && io.mem.grant.valid), dataBeats) + val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) + when (state === state_rx && rx_done) { + state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, + Mux(rx_cmd === cmd_writemem, state_mem_wreq, + Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, + state_tx))) + } + when (state === state_mem_wreq) { + when (cnt_done) { state := state_mem_wresp } + } + when (state === state_mem_rreq) { + when(io.mem.acquire.ready) { state := state_mem_rresp } + } + when (state === state_mem_wresp && io.mem.grant.valid) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << offsetBits-3) + } + when (state === state_mem_rresp && cnt_done) { + state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) + pos := pos - UInt(1) + addr := addr + UInt(1 << offsetBits-3) + } + when (state === state_tx && tx_done) { + when (tx_word_count === tx_size) { + rx_count := UInt(0) + tx_count := UInt(0) + } + state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) + } + + val n = dataBits/short_request_bits + val mem_req_data = (0 until n).map { i => + val ui = UInt(i, log2Up(n)) + when (state === state_mem_rresp && io.mem.grant.valid) { + packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := + io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) + } + packet_ram(Cat(cnt, ui)) + }.reverse.reduce(_##_) + + val init_addr = addr.toUInt >> UInt(offsetBits-3) + io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq + io.mem.acquire.bits := Mux(cmd === cmd_writemem, + PutBlock( + addr_block = init_addr, + addr_beat = cnt, + client_xact_id = UInt(0), + data = mem_req_data), + GetBlock(addr_block = init_addr)) + io.mem.grant.ready := Bool(true) + + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) + for (i <- 0 until nCores) { + val my_reset = Reg(init=Bool(true)) + val my_ipi = Reg(init=Bool(false)) + + val cpu = io.cpu(i) + val me = pcr_coreid === UInt(i) + cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr_req.bits.rw := cmd === cmd_writecr + cpu.pcr_req.bits.addr := pcr_addr + cpu.pcr_req.bits.data := pcr_wdata + cpu.reset := my_reset + + when (cpu.ipi_rep.ready) { + my_ipi := Bool(false) + } + cpu.ipi_rep.valid := my_ipi + cpu.ipi_req.ready := Bool(true) + for (j <- 0 until nCores) { + when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { + my_ipi := Bool(true) + } + } + + when (cpu.pcr_req.valid && cpu.pcr_req.ready) { + state := state_pcr_resp + } + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (cmd === cmd_writecr) { + my_reset := pcr_wdata(0) + } + pcrReadData := my_reset.toBits + state := state_tx + } + + cpu.pcr_rep.ready := Bool(true) + when (cpu.pcr_rep.valid) { + pcrReadData := cpu.pcr_rep.bits + state := state_tx + } + } + + val scr_addr = addr(log2Up(nSCR)-1, 0) + val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} + for (i <- 0 until scr_rdata.size) + scr_rdata(i) := io.scr.rdata(i) + scr_rdata(0) := UInt(nCores) + scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) + + io.scr.wen := Bool(false) + io.scr.wdata := pcr_wdata + io.scr.waddr := scr_addr.toUInt + when (state === state_pcr_req && pcr_coreid === SInt(-1)) { + io.scr.wen := cmd === cmd_writecr + pcrReadData := scr_rdata(scr_addr) + state := state_tx + } + + val tx_cmd = Mux(nack, cmd_nack, cmd_ack) + val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) + val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) + val tx_data = Mux(tx_word_count === UInt(0), tx_header, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, pcrReadData, + packet_ram(packet_ram_raddr))) + + io.host.in.ready := state === state_rx + io.host.out.valid := state === state_tx + io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) +} diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala new file mode 100644 index 00000000..7497e2e4 --- /dev/null +++ b/uncore/src/main/scala/memserdes.scala @@ -0,0 +1,584 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import scala.math._ + +case object PAddrBits extends Field[Int] +case object VAddrBits extends Field[Int] +case object PgIdxBits extends Field[Int] +case object PgLevels extends Field[Int] +case object PgLevelBits extends Field[Int] +case object ASIdBits extends Field[Int] +case object PPNBits extends Field[Int] +case object VPNBits extends Field[Int] + +case object MIFAddrBits extends Field[Int] +case object MIFDataBits extends Field[Int] +case object MIFTagBits extends Field[Int] +case object MIFDataBeats extends Field[Int] + +trait MIFParameters extends UsesParameters { + val mifTagBits = params(MIFTagBits) + val mifAddrBits = params(MIFAddrBits) + val mifDataBits = params(MIFDataBits) + val mifDataBeats = params(MIFDataBeats) +} + +abstract class MIFBundle extends Bundle with MIFParameters +abstract class MIFModule extends Module with MIFParameters + +trait HasMemData extends MIFBundle { + val data = Bits(width = mifDataBits) +} + +trait HasMemAddr extends MIFBundle { + val addr = UInt(width = mifAddrBits) +} + +trait HasMemTag extends MIFBundle { + val tag = UInt(width = mifTagBits) +} + +class MemReqCmd extends HasMemAddr with HasMemTag { + val rw = Bool() +} + +class MemTag extends HasMemTag +class MemData extends HasMemData +class MemResp extends HasMemData with HasMemTag + +class MemIO extends Bundle { + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Decoupled(new MemResp).flip +} + +class MemPipeIO extends Bundle { + val req_cmd = Decoupled(new MemReqCmd) + val req_data = Decoupled(new MemData) + val resp = Valid(new MemResp).flip +} + +class MemSerializedIO(w: Int) extends Bundle +{ + val req = Decoupled(Bits(width = w)) + val resp = Valid(Bits(width = w)).flip +} + +class MemSerdes(w: Int) extends MIFModule +{ + val io = new Bundle { + val wide = new MemIO().flip + val narrow = new MemSerializedIO(w) + } + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + + val out_buf = Reg(Bits()) + val in_buf = Reg(Bits()) + + val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_idle) + val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) + val data_send_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) + val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) + + when (io.narrow.req.valid && io.narrow.req.ready) { + send_cnt := send_cnt + UInt(1) + out_buf := out_buf >> UInt(w) + } + when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { + out_buf := io.wide.req_cmd.bits.toBits + } + when (io.wide.req_data.valid && io.wide.req_data.ready) { + out_buf := io.wide.req_data.bits.toBits + } + + io.wide.req_cmd.ready := state === s_idle + io.wide.req_data.ready := state === s_write_idle + io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data + io.narrow.req.bits := out_buf + + when (state === s_idle && io.wide.req_cmd.valid) { + state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) + } + when (state === s_read_addr && adone) { + state := s_idle + send_cnt := UInt(0) + } + when (state === s_write_addr && adone) { + state := s_write_idle + send_cnt := UInt(0) + } + when (state === s_write_idle && io.wide.req_data.valid) { + state := s_write_data + } + when (state === s_write_data && ddone) { + data_send_cnt := data_send_cnt + UInt(1) + state := Mux(data_send_cnt === UInt(mifDataBeats-1), s_idle, s_write_idle) + send_cnt := UInt(0) + } + + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) + val resp_val = Reg(init=Bool(false)) + + resp_val := Bool(false) + when (io.narrow.resp.valid) { + recv_cnt := recv_cnt + UInt(1) + when (recv_cnt === UInt((rbits-1)/w)) { + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) + resp_val := Bool(true) + } + in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) + } + + io.wide.resp.valid := resp_val + io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) +} + +class MemDesserIO(w: Int) extends Bundle { + val narrow = new MemSerializedIO(w).flip + val wide = new MemIO +} + +class MemDesser(w: Int) extends Module // test rig side +{ + val io = new MemDesserIO(w) + val abits = io.wide.req_cmd.bits.toBits.getWidth + val dbits = io.wide.req_data.bits.toBits.getWidth + val rbits = io.wide.resp.bits.getWidth + val mifDataBeats = params(MIFDataBeats) + + require(dbits >= abits && rbits >= dbits) + val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) + val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) + val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) + val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) + val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) + + val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(UInt(), 5) + val state = Reg(init=s_cmd_recv) + + val in_buf = Reg(Bits()) + when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { + recv_cnt := recv_cnt + UInt(1) + in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) + } + io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv + + when (state === s_cmd_recv && adone) { + state := s_cmd + recv_cnt := UInt(0) + } + when (state === s_cmd && io.wide.req_cmd.ready) { + state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) + } + when (state === s_data_recv && ddone) { + state := s_data + recv_cnt := UInt(0) + } + when (state === s_data && io.wide.req_data.ready) { + state := s_data_recv + when (data_recv_cnt === UInt(mifDataBeats-1)) { + state := s_cmd_recv + } + data_recv_cnt := data_recv_cnt + UInt(1) + } + when (rdone) { // state === s_reply + when (data_recv_cnt === UInt(mifDataBeats-1)) { + state := s_cmd_recv + } + recv_cnt := UInt(0) + data_recv_cnt := data_recv_cnt + UInt(1) + } + + val req_cmd = in_buf >> UInt(((rbits+w-1)/w - (abits+w-1)/w)*w) + io.wide.req_cmd.valid := state === s_cmd + io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) + + io.wide.req_data.valid := state === s_data + io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) + + val dataq = Module(new Queue(new MemResp, mifDataBeats)) + dataq.io.enq <> io.wide.resp + dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) + + io.narrow.resp.valid := dataq.io.deq.valid + io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) +} + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val mem = new MemIO + } + val dataBits = tlDataBits*tlDataBeats + val dstIdBits = params(LNHeaderBits) + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") + require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + val dst_off = dstIdBits + tlClientXactIdBits + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() + + // Decompose outgoing TL Acquires into MemIO cmd and data + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val tag_out = Reg(UInt(width = mifTagBits)) + val addr_out = Reg(UInt(width = mifAddrBits)) + val has_data = Reg(init=Bool(false)) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) + + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.data.clone) } + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + addr_out := io.tl.release.bits.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := io.tl.release.bits.requiresAck() + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + addr_out := io.tl.acquire.bits.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data + } + } + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.data, + io.tl.acquire.bits.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) + } + } + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.data, + io.tl.acquire.bits.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := io.tl.release.bits.requiresAck() + io.mem.req_data.bits.data := io.tl.release.bits.data + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := rel_has_data + tag_out := tag + addr_out := addr + has_data := rel_has_data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data // i.e. is it a Put + io.mem.req_data.bits.data := io.tl.acquire.bits.data + io.mem.req_cmd.bits.rw := acq_has_data + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := acq_has_data + tag_out := tag + addr_out := addr + has_data := acq_has_data + } + } + } + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } + } + } + + // Aggregate incoming MemIO responses into TL Grants + val active_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val tag_in = Reg(UInt(width = mifTagBits)) + + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data + } + } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) + } +} + +class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + require(entries > 1) + + val do_flow = Bool() + val do_enq = io.enq.fire() && !do_flow + val do_deq = io.deq.fire() && !do_flow + + val maybe_full = Reg(init=Bool(false)) + val enq_ptr = Counter(do_enq, entries)._1 + val (deq_ptr, deq_done) = Counter(do_deq, entries) + when (do_enq != do_deq) { maybe_full := do_enq } + + val ptr_match = enq_ptr === deq_ptr + val empty = ptr_match && !maybe_full + val full = ptr_match && maybe_full + val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) + do_flow := empty && io.deq.ready + + val ram = Mem(data, entries, seqRead = true) + val ram_addr = Reg(Bits()) + val ram_out_valid = Reg(Bool()) + ram_out_valid := Bool(false) + when (do_enq) { ram(enq_ptr) := io.enq.bits } + when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { + ram_out_valid := Bool(true) + ram_addr := Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) + } + + io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) + io.enq.ready := !full + io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) +} + +class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module +{ + val io = new QueueIO(data, entries) + + val fq = Module(new HellaFlowQueue(entries)(data)) + io.enq <> fq.io.enq + io.deq <> Queue(fq.io.deq, 1, pipe = true) +} + +object HellaQueue +{ + def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { + val q = Module((new HellaQueue(entries)) { enq.bits.clone }) + q.io.enq.valid := enq.valid // not using <> so that override is allowed + q.io.enq.bits := enq.bits + enq.ready := q.io.enq.ready + q.io.deq + } +} + +class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { + val io = new Bundle { + val cpu = new MemIO().flip + val mem = new MemPipeIO + } + + val numEntries = numRequests * mifDataBeats + val size = log2Down(numEntries) + 1 + + val inc = Bool() + val dec = Bool() + val count = Reg(init=UInt(numEntries, size)) + val watermark = count >= UInt(mifDataBeats) + + when (inc && !dec) { + count := count + UInt(1) + } + when (!inc && dec) { + count := count - UInt(mifDataBeats) + } + when (inc && dec) { + count := count - UInt(mifDataBeats-1) + } + + val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark + + io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask + io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask + io.mem.req_cmd.bits := io.cpu.req_cmd.bits + + io.mem.req_data <> io.cpu.req_data + + // Have separate queues to allow for different mem implementations + val resp_data_q = Module((new HellaQueue(numEntries)) { new MemData }) + resp_data_q.io.enq.valid := io.mem.resp.valid + resp_data_q.io.enq.bits.data := io.mem.resp.bits.data + + val resp_tag_q = Module((new HellaQueue(numEntries)) { new MemTag }) + resp_tag_q.io.enq.valid := io.mem.resp.valid + resp_tag_q.io.enq.bits.tag := io.mem.resp.bits.tag + + io.cpu.resp.valid := resp_data_q.io.deq.valid && resp_tag_q.io.deq.valid + io.cpu.resp.bits.data := resp_data_q.io.deq.bits.data + io.cpu.resp.bits.tag := resp_tag_q.io.deq.bits.tag + resp_data_q.io.deq.ready := io.cpu.resp.ready + resp_tag_q.io.deq.ready := io.cpu.resp.ready + + inc := resp_data_q.io.deq.fire() && resp_tag_q.io.deq.fire() + dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw +} + +class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val mem = new MemPipeIO + } + + val a = Module(new MemIOTileLinkIOConverter(1)) + val b = Module(new MemPipeIOMemIOConverter(outstanding)) + a.io.tl <> io.tl + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) + a.io.mem.resp <> b.io.cpu.resp + b.io.mem <> io.mem +} diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala new file mode 100644 index 00000000..b8d4446c --- /dev/null +++ b/uncore/src/main/scala/metadata.scala @@ -0,0 +1,315 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +/** Base class to represent coherence information in clients and managers */ +abstract class CoherenceMetadata extends Bundle { + val co = params(TLCoherencePolicy) + val id = params(TLId) +} + +/** Stores the client-side coherence information, + * such as permissions on the data and whether the data is dirty. + * Its API can be used to make TileLink messages in response to + * memory operations or [[uncore.Probe]] messages. + */ +class ClientMetadata extends CoherenceMetadata { + /** Actual state information stored in this bundle */ + val state = UInt(width = co.clientStateWidth) + + /** Metadata equality */ + def ===(rhs: ClientMetadata): Bool = this.state === rhs.state + def !=(rhs: ClientMetadata): Bool = !this.===(rhs) + + /** Is the block's data present in this cache */ + def isValid(dummy: Int = 0): Bool = co.isValid(this) + /** Does this cache have permissions on this block sufficient to perform op */ + def isHit(op_code: UInt): Bool = co.isHit(op_code, this) + /** Does this cache lack permissions on this block sufficient to perform op */ + def isMiss(op_code: UInt): Bool = !co.isHit(op_code, this) + /** Does a secondary miss on the block require another Acquire message */ + def requiresAcquireOnSecondaryMiss(first_op: UInt, second_op: UInt): Bool = + co.requiresAcquireOnSecondaryMiss(first_op, second_op, this) + /** Does op require a Release to be made to outer memory */ + def requiresReleaseOnCacheControl(op_code: UInt): Bool = + co.requiresReleaseOnCacheControl(op_code: UInt, this) + /** Does an eviction require a Release to be made to outer memory */ + def requiresVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresReleaseOnCacheControl(M_FLUSH, this) + + /** Constructs an Acquire message based on this metdata and a memory operation + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] + */ + def makeAcquire( + client_xact_id: UInt, + addr_block: UInt, + op_code: UInt): Acquire = { + Bundle(Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true))), + { case TLId => id }) + } + + /** Constructs a Release message based on this metadata on an eviction + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being written back + */ + def makeVoluntaryWriteback( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(M_FLUSH, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + /** Constructs a Release message based on this metadata and a [[uncore.Probe]] + * + * @param the incoming [[uncore.Probe]] + * @param addr_beat sub-block address (which beat) + * @param data data being released + */ + def makeRelease( + prb: Probe, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(false), + r_type = co.getReleaseType(prb, this), + client_xact_id = UInt(0), + addr_block = prb.addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + /** New metadata after receiving a [[uncore.Grant]] + * + * @param incoming the incoming [[uncore.Grant]] + * @param pending the mem op that triggered this transaction + */ + def onGrant(incoming: Grant, pending: UInt): ClientMetadata = + Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param incoming the incoming [[uncore.Probe]] + */ + def onProbe(incoming: Probe): ClientMetadata = + Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + + /** New metadata after a op_code hits this block + * + * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] + */ + def onHit(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) + + /** New metadata after receiving a [[uncore.Probe]] + * + * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] + */ + def onCacheControl(op_code: UInt): ClientMetadata = + Bundle(co.clientMetadataOnCacheControl(op_code, this), { case TLId => id }) +} + +/** Factories for ClientMetadata, including on reset */ +object ClientMetadata { + def apply(state: UInt) = { + val meta = new ClientMetadata + meta.state := state + meta + } + def onReset = new ClientMetadata().co.clientMetadataOnReset +} + +/** Stores manager-side information about the status + * of a cache block, including whether it has any known sharers. + * + * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. + */ +class ManagerMetadata extends CoherenceMetadata { + // Currently no coherence policies assume manager-side state information + // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel + + /** The directory information for this block */ + val sharers = UInt(width = co.dir.width) + + /** Metadata equality */ + def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel + this.sharers === rhs.sharers + def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) + + /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ + def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) + + /** Does this [[uncore.Acquire]] require [[uncore.Probe Probes]] to be sent */ + def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + /** Does this memory op require [[uncore.Probe Probes]] to be sent */ + def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) + /** Does an eviction require [[uncore.Probe Probes]] to be sent */ + def requiresProbesOnVoluntaryWriteback(dummy: Int = 0): Bool = + co.requiresProbes(M_FLUSH, this) + + /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] + * + * @param dst Destination client id for this Probe + * @param acq Acquire message triggering this Probe + */ + def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + + /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op + * + * @param dst Destination client id for this Probe + * @param op_code memory operation triggering this Probe + * @param addr_block address of the cache block being probed + */ + def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = + Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block), { case TLId => id }) + + /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction + * + * @param dst Destination client id for this Probe + * @param addr_block address of the cache block being probed prior to eviction + */ + def makeProbeForVoluntaryWriteback(dst: UInt, addr_block: UInt): ProbeToDst = + makeProbe(dst, M_FLUSH, addr_block) + + /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] + * + * @param rel Release message being acknowledged by this Grant + * @param manager_xact_id manager's transaction id + */ + def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { + Bundle(Grant( + dst = rel.client_id, + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, + client_xact_id = rel.client_xact_id, + manager_xact_id = manager_xact_id), { case TLId => id }) + } + + /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] + * + * May contain single or multiple beats of data, or just be a permissions upgrade. + * + * @param acq Acquire message being responded to by this Grant + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ + def makeGrant( + acq: AcquireFromSrc, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = { + Bundle(Grant( + dst = acq.client_id, + is_builtin_type = acq.isBuiltInType(), + g_type = Mux(acq.isBuiltInType(), + acq.getBuiltInGrantType(), + co.getGrantType(acq, this)), + client_xact_id = acq.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + + /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides + * + * Used to respond to secondary misses merged into this transaction. + * May contain single or multiple beats of data. + * + * @param pri Primary miss's Acquire message, used to get g_type and dst + * @param sec Secondary miss info, used to get beat and client_xact_id + * @param manager_xact_id manager's transaction id + * @param data data being refilled to the original requestor + */ + def makeGrant( + pri: AcquireFromSrc, + sec: SecondaryMissInfo, + manager_xact_id: UInt, + data: UInt): GrantToDst = { + val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) + g.client_xact_id := sec.client_xact_id + g + } + + /** New metadata after receiving a [[uncore.ReleaseFromSrc]] + * + * @param incoming the incoming [[uncore.ReleaseFromSrc]] + */ + def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = + Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) + + /** New metadata after sending a [[uncore.GrantToDst]] + * + * @param outgoing the outgoing [[uncore.GrantToDst]] + */ + def onGrant(outgoing: GrantToDst): ManagerMetadata = + Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) +} + +/** Factories for ManagerMetadata, including on reset */ +object ManagerMetadata { + def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + val meta = new ManagerMetadata + //meta.state := state TODO: Fix 0-width wires in Chisel + meta.sharers := sharers + meta + } + def apply() = { + val meta = new ManagerMetadata + //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel + meta.sharers := meta.co.dir.flush + meta + } + def onReset = new ManagerMetadata().co.managerMetadataOnReset +} + +/** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy + * that is a manager with respect to some inner caches and a client with + * respect to some outer cache. + * + * This class makes use of two different sets of TileLink parameters, which are + * applied by contextually mapping [[uncore.TLId]] to one of + * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. + */ +class HierarchicalMetadata extends CoherenceMetadata { + val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) + val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) + def ===(rhs: HierarchicalMetadata): Bool = + this.inner === rhs.inner && this.outer === rhs.outer + def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) +} + +/** Factories for HierarchicalMetadata, including on reset */ +object HierarchicalMetadata { + def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + val m = new HierarchicalMetadata + m.inner := inner + m.outer := outer + m + } + def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) +} + +/** Identifies the TLId of the inner network in a hierarchical cache controller */ +case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ +case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala new file mode 100644 index 00000000..4b00091d --- /dev/null +++ b/uncore/src/main/scala/network.scala @@ -0,0 +1,104 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] + +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) +} + +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) + val payload = dType.clone + override def clone = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] +} + +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))}.flip + val out = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))} +} + +abstract class PhysicalNetwork extends Module + +class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { + val io = new BasicCrossbarIO(n, dType) + + val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} + + io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) + (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { + arb.valid := in.valid && (in.bits.header.dst === UInt(i)) + arb.bits := in.bits + rdy := arb.ready && (in.bits.header.dst === UInt(i)) + }} + out <> rrarb.io.out + }} + for(i <- 0 until n) { + io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) + } +} + +abstract class LogicalNetwork extends Module + +class LogicalHeader extends Bundle { + val src = UInt(width = params(LNHeaderBits)) + val dst = UInt(width = params(LNHeaderBits)) +} + +class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { + val header = new LogicalHeader + val payload = dType.clone + override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } +} + +object DecoupledLogicalNetworkIOWrapper { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out + } +} + +object DecoupledLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { + val out = Decoupled(in.bits.payload.clone).asDirectionless + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out + } +} + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Decoupled(new LogicalNetworkIO(in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Decoupled(new PhysicalNetworkIO(n, in.bits.payload)).asDirectionless + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala new file mode 100644 index 00000000..2c6c4a5f --- /dev/null +++ b/uncore/src/main/scala/package.scala @@ -0,0 +1,6 @@ +// See LICENSE for license details. + +package object uncore extends uncore.constants.MemoryOpConstants +{ + implicit def toOption[A](a: A) = Option(a) +} diff --git a/uncore/src/main/scala/slowio.scala b/uncore/src/main/scala/slowio.scala new file mode 100644 index 00000000..95ca34e6 --- /dev/null +++ b/uncore/src/main/scala/slowio.scala @@ -0,0 +1,70 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module +{ + val io = new Bundle { + val out_fast = Decoupled(data).flip + val out_slow = Decoupled(data) + val in_fast = Decoupled(data) + val in_slow = Decoupled(data).flip + val clk_slow = Bool(OUTPUT) + val set_divisor = Valid(Bits(width = 32)).flip + val divisor = Bits(OUTPUT, 32) + } + + require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) + val divisor = Reg(init=UInt(divisor_max-1)) + val d_shadow = Reg(init=UInt(divisor_max-1)) + val hold = Reg(init=UInt(divisor_max/4-1)) + val h_shadow = Reg(init=UInt(divisor_max/4-1)) + when (io.set_divisor.valid) { + d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt + h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt + } + io.divisor := hold << UInt(16) | divisor + + val count = Reg{UInt(width = log2Up(divisor_max))} + val myclock = Reg{Bool()} + count := count + UInt(1) + + val rising = count === (divisor >> UInt(1)) + val falling = count === divisor + val held = count === (divisor >> UInt(1)) + hold + + when (falling) { + divisor := d_shadow + hold := h_shadow + count := UInt(0) + myclock := Bool(false) + } + when (rising) { + myclock := Bool(true) + } + + val in_slow_rdy = Reg(init=Bool(false)) + val out_slow_val = Reg(init=Bool(false)) + val out_slow_bits = Reg(data) + + val fromhost_q = Module(new Queue(data,1)) + fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.reset) + fromhost_q.io.enq.bits := io.in_slow.bits + fromhost_q.io.deq <> io.in_fast + + val tohost_q = Module(new Queue(data,1)) + tohost_q.io.enq <> io.out_fast + tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val + + when (held) { + in_slow_rdy := fromhost_q.io.enq.ready + out_slow_val := tohost_q.io.deq.valid + out_slow_bits := Mux(this.reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) + } + + io.in_slow.ready := in_slow_rdy + io.out_slow.valid := out_slow_val + io.out_slow.bits := out_slow_bits + io.clk_slow := myclock +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala new file mode 100644 index 00000000..306c8731 --- /dev/null +++ b/uncore/src/main/scala/tilelink.scala @@ -0,0 +1,1221 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import scala.math.max + +/** Parameters exposed to the top-level design, set based on + * external requirements or design space exploration + */ +/** Unique name per TileLink network*/ +case object TLId extends Field[String] +/** Coherency policy used to define custom mesage types */ +case object TLCoherencePolicy extends Field[CoherencePolicy] +/** Number of manager agents */ +case object TLNManagers extends Field[Int] +/** Number of client agents */ +case object TLNClients extends Field[Int] +/** Number of client agents that cache data and use custom [[uncore.Acquire]] types */ +case object TLNCachingClients extends Field[Int] +/** Number of client agents that do not cache data and use built-in [[uncore.Acquire]] types */ +case object TLNCachelessClients extends Field[Int] +/** Maximum number of unique outstanding transactions per client */ +case object TLMaxClientXacts extends Field[Int] +/** Maximum number of clients multiplexed onto a single port */ +case object TLMaxClientsPerPort extends Field[Int] +/** Maximum number of unique outstanding transactions per manager */ +case object TLMaxManagerXacts extends Field[Int] +/** Width of cache block addresses */ +case object TLBlockAddrBits extends Field[Int] +/** Width of data beats */ +case object TLDataBits extends Field[Int] +/** Number of data beats per cache block */ +case object TLDataBeats extends Field[Int] +/** Whether the underlying physical network preserved point-to-point ordering of messages */ +case object TLNetworkIsOrderedP2P extends Field[Boolean] + +/** Utility trait for building Modules and Bundles that use TileLink parameters */ +trait TileLinkParameters extends UsesParameters { + val tlCoh = params(TLCoherencePolicy) + val tlNManagers = params(TLNManagers) + val tlNClients = params(TLNClients) + val tlNCachingClients = params(TLNCachingClients) + val tlNCachelessClients = params(TLNCachelessClients) + val tlClientIdBits = log2Up(tlNClients) + val tlManagerIdBits = log2Up(tlNManagers) + val tlMaxClientXacts = params(TLMaxClientXacts) + val tlMaxClientsPerPort = params(TLMaxClientsPerPort) + val tlMaxManagerXacts = params(TLMaxManagerXacts) + val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) + val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) + val tlBlockAddrBits = params(TLBlockAddrBits) + val tlDataBits = params(TLDataBits) + val tlDataBytes = tlDataBits/8 + val tlDataBeats = params(TLDataBeats) + val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlBeatAddrBits = log2Up(tlDataBeats) + val tlByteAddrBits = log2Up(tlWriteMaskBits) + val tlMemoryOpcodeBits = M_SZ + val tlMemoryOperandSizeBits = MT_SZ + val tlAcquireTypeBits = max(log2Up(Acquire.nBuiltInTypes), + tlCoh.acquireTypeWidth) + val tlAcquireUnionBits = max(tlWriteMaskBits, + (tlByteAddrBits + + tlMemoryOperandSizeBits + + tlMemoryOpcodeBits)) + 1 + val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), + tlCoh.grantTypeWidth) + 1 + val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val tlNetworkDoesNotInterleaveBeats = true + val amoAluOperandBits = params(AmoAluOperandBits) +} + +abstract class TLBundle extends Bundle with TileLinkParameters +abstract class TLModule extends Module with TileLinkParameters + +/** Base trait for all TileLink channels */ +trait TileLinkChannel extends TLBundle { + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool +} +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ +trait ClientToManagerChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ +trait ManagerToClientChannel extends TileLinkChannel +/** Directionality of message channel. Used to hook up logical network ports to physical network ports */ +trait ClientToClientChannel extends TileLinkChannel // Unused for now + +/** Common signals that are used in multiple channels. + * These traits are useful for type parameterizing bundle wiring functions. + */ + +/** Address of a cache block. */ +trait HasCacheBlockAddress extends TLBundle { + val addr_block = UInt(width = tlBlockAddrBits) + + def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block + def conflicts(addr: UInt) = this.addr_block === addr +} + +/** Sub-block address or beat id of multi-beat data */ +trait HasTileLinkBeatId extends TLBundle { + val addr_beat = UInt(width = tlBeatAddrBits) +} + +/* Client-side transaction id. Usually Miss Status Handling Register File index */ +trait HasClientTransactionId extends TLBundle { + val client_xact_id = Bits(width = tlClientXactIdBits) +} + +/** Manager-side transaction id. Usually Transaction Status Handling Register File index. */ +trait HasManagerTransactionId extends TLBundle { + val manager_xact_id = Bits(width = tlManagerXactIdBits) +} + +/** A single beat of cache block data */ +trait HasTileLinkData extends HasTileLinkBeatId { + val data = UInt(width = tlDataBits) + + def hasData(dummy: Int = 0): Bool + def hasMultibeatData(dummy: Int = 0): Bool +} + +/** The id of a client source or destination. Used in managers. */ +trait HasClientId extends TLBundle { + val client_id = UInt(width = tlClientIdBits) +} + +/** TileLink channel bundle definitions */ + +/** The Acquire channel is used to intiate coherence protocol transactions in + * order to gain access to a cache block's data with certain permissions + * enabled. Messages sent over this channel may be custom types defined by + * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types + * used for uncached data accesses. Acquires may contain data for Put or + * PutAtomic built-in types. After sending an Acquire, clients must + * wait for a manager to send them a [[uncore.Grant]] message in response. + */ +class Acquire extends ClientToManagerChannel + with HasCacheBlockAddress + with HasClientTransactionId + with HasTileLinkData { + // Actual bundle fields: + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) + val union = Bits(width = tlAcquireUnionBits) + + // Utility funcs for accessing subblock union: + val opCodeOff = 1 + val opSizeOff = tlMemoryOpcodeBits + opCodeOff + val addrByteOff = tlMemoryOperandSizeBits + opSizeOff + val addrByteMSB = tlByteAddrBits + addrByteOff + /** Hint whether to allocate the block in any interveneing caches */ + def allocate(dummy: Int = 0) = union(0) + /** Op code for [[uncore.PutAtomic]] operations */ + def op_code(dummy: Int = 0) = Mux( + isBuiltInType(Acquire.putType) || isBuiltInType(Acquire.putBlockType), + M_XWR, union(opSizeOff-1, opCodeOff)) + /** Operand size for [[uncore.PutAtomic]] */ + def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) + /** Byte address for [[uncore.PutAtomic]] operand */ + def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) + private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + /** Bit offset of [[uncore.PutAtomic]] operand */ + def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ + def wmask(dummy: Int = 0) = + Mux(isBuiltInType(Acquire.putAtomicType), + FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), + Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), + union(tlWriteMaskBits, 1), + UInt(0, width = tlWriteMaskBits))) + /** Full, beat-sized writemask */ + def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) + /** Complete physical address for block, beat or operand */ + def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) + + // Other helper functions: + /** Message type equality */ + def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? + + /** Is this message a built-in or custom type */ + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + /** Is this message a particular built-in type */ + def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t + + /** Does this message refer to subblock operands using info in the Acquire.union subbundle */ + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + + /** Is this message a built-in prefetch message */ + def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) + + /** Does this message contain data? Assumes that no custom message types have data. */ + def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + + /** Does this message contain multiple beats of data? Assumes that no custom message types have data. */ + def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && + Acquire.typesWithMultibeatData.contains(a_type) + + /** Does this message require the manager to probe the client the very client that sent it? + * Needed if multiple caches are attached to the same port. + */ + def requiresSelfProbe(dummy: Int = 0) = Bool(false) + + /** Mapping between each built-in Acquire type (defined in companion object) + * and a built-in Grant type. + */ + def getBuiltInGrantType(dummy: Int = 0): UInt = { + MuxLookup(this.a_type, Grant.putAckType, Array( + Acquire.getType -> Grant.getDataBeatType, + Acquire.getBlockType -> Grant.getDataBlockType, + Acquire.putType -> Grant.putAckType, + Acquire.putBlockType -> Grant.putAckType, + Acquire.putAtomicType -> Grant.getDataBeatType, + Acquire.prefetchType -> Grant.prefetchAckType)) + } +} + +/** [[uncore.Acquire]] with an extra field stating its source id */ +class AcquireFromSrc extends Acquire with HasClientId + +/** Contains definitions of the the built-in Acquire types and a factory + * for [[uncore.Acquire]] + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeAcquire]] for custom cached Acquires and + * [[uncore.Get]], [[uncore.Put]], etc. for built-in uncached Acquires. + * + * @param is_builtin_type built-in or custom type message? + * @param a_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being put outwards + * @param union additional fields used for uncached types + */ +object Acquire { + val nBuiltInTypes = 5 + //TODO: Use Enum + def getType = UInt("b000") // Get a single beat of data + def getBlockType = UInt("b001") // Get a whole block of data + def putType = UInt("b010") // Put a single beat of data + def putBlockType = UInt("b011") // Put a whole block of data + def putAtomicType = UInt("b100") // Perform an atomic memory op + def prefetchType = UInt("b101") // Prefetch a whole block of data + def typesWithData = Vec(putType, putBlockType, putAtomicType) + def typesWithMultibeatData = Vec(putBlockType) + def typesOnSubBlocks = Vec(putType, getType, putAtomicType) + + def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + + // Most generic constructor + def apply( + is_builtin_type: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + union: UInt = UInt(0)): Acquire = { + val acq = new Acquire + acq.is_builtin_type := is_builtin_type + acq.a_type := a_type + acq.client_xact_id := client_xact_id + acq.addr_block := addr_block + acq.addr_beat := addr_beat + acq.data := data + acq.union := union + acq + } + // Copy constructor + def apply(a: Acquire): Acquire = { + val acq = new Acquire + acq := a + acq + } +} + +/** Get a single beat of data from the outer memory hierarchy + * + * The client can hint whether he block containing this beat should be + * allocated in the intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param alloc hint whether the block should be allocated in intervening caches + */ +object Get { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.getType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + union = Cat(M_XRD, alloc)) + } +} + +/** Get a whole cache block of data from the outer memory hierarchy + * + * The client can hint whether the block should be allocated in the + * intervening levels of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param alloc hint whether the block should be allocated in intervening caches + */ +object GetBlock { + def apply( + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.getBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(M_XRD, alloc)) + } +} + +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with read permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ +object GetPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XRD, Bool(true))) + } +} + +/** Put a single beat of data into the outer memory hierarchy + * + * The block will be allocated in the next-outermost level of the hierarchy. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + */ +object Put { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + wmask: UInt = Acquire.fullWriteMask): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putType, + addr_block = addr_block, + addr_beat = addr_beat, + client_xact_id = client_xact_id, + data = data, + union = Cat(wmask, Bool(true))) + } +} + +/** Put a whole cache block of data into the outer memory hierarchy + * + * If the write mask is not full, the block will be allocated in the + * next-outermost level of the hierarchy. If the write mask is full, the + * client can hint whether the block should be allocated or not. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat of several) + * @param data data being refilled to the original requestor + * @param wmask per-byte write mask for this beat + * @param alloc hint whether the block should be allocated in intervening caches + */ +object PutBlock { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + wmask: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(wmask, (wmask != Acquire.fullWriteMask))) + } + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putBlockType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(Acquire.fullWriteMask, alloc)) + } +} + +/** Prefetch a cache block into the next-outermost level of the memory hierarchy + * with write permissions. + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + */ +object PutPrefetch { + def apply( + client_xact_id: UInt, + addr_block: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.prefetchType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = UInt(0), + union = Cat(M_XWR, Bool(true))) + } +} + +/** Perform an atomic memory operation in the next-outermost level of the memory hierarchy + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (within which beat) + * @param addr_byte sub-block address (which byte) + * @param atomic_opcode {swap, add, xor, and, min, max, minu, maxu} from [[uncore.MemoryOpConstants]] + * @param operand_size {byte, half, word, double} from [[uncore.MemoryOpConstants]] + * @param data source operand data + */ +object PutAtomic { + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.putAtomicType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) + } +} + +/** The Probe channel is used to force clients to release data or cede permissions + * on a cache block. Clients respond to Probes with [[uncore.Release]] messages. + * The available types of Probes are customized by a particular + * [[uncore.CoherencePolicy]]. + */ +class Probe extends ManagerToClientChannel + with HasCacheBlockAddress { + val p_type = UInt(width = tlCoh.probeTypeWidth) + + def is(t: UInt) = p_type === t + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} + +/** [[uncore.Probe]] with an extra field stating its destination id */ +class ProbeToDst extends Probe with HasClientId + +/** Contains factories for [[uncore.Probe]] and [[uncore.ProbeToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeProbe(UInt,Acquire)*]] instead. + * + * @param dst id of client to which probe should be sent + * @param p_type custom probe type + * @param addr_block address of the cache block + */ +object Probe { + def apply(p_type: UInt, addr_block: UInt): Probe = { + val prb = new Probe + prb.p_type := p_type + prb.addr_block := addr_block + prb + } + def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { + val prb = new ProbeToDst + prb.client_id := dst + prb.p_type := p_type + prb.addr_block := addr_block + prb + } +} + +/** The Release channel is used to release data or permission back to the manager + * in response to [[uncore.Probe]] messages. It can also be used to voluntarily + * write back data, for example in the event that dirty data must be evicted on + * a cache miss. The available types of Release messages are always customized by + * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be + * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grant Grants]]. + */ +class Release extends ClientToManagerChannel + with HasCacheBlockAddress + with HasClientTransactionId + with HasTileLinkData { + val r_type = UInt(width = tlCoh.releaseTypeWidth) + val voluntary = Bool() + + // Helper funcs + def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) + //TODO: Assumes all releases write back full cache blocks: + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) +} + +/** [[uncore.Release]] with an extra field stating its source id */ +class ReleaseFromSrc extends Release with HasClientId + +/** Contains a [[uncore.Release]] factory + * + * In general you should avoid using this factory directly and use + * [[uncore.ClientMetadata.makeRelease]] instead. + * + * @param voluntary is this a voluntary writeback + * @param r_type type enum defined by coherence protocol + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat beat id of the data + * @param data data being written back + */ +object Release { + def apply( + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + val rel = new Release + rel.r_type := r_type + rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat + rel.data := data + rel.voluntary := voluntary + rel + } +} + +/** The Grant channel is used to refill data or grant permissions requested of the + * manager agent via an [[uncore.Acquire]] message. It is also used to acknowledge + * the receipt of voluntary writeback from clients in the form of [[uncore.Release]] + * messages. There are built-in Grant messages used for Gets and Puts, and + * coherence policies may also define custom Grant types. Grants may contain data + * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. + */ +class Grant extends ManagerToClientChannel + with HasTileLinkData + with HasClientTransactionId + with HasManagerTransactionId { + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) + + // Helper funcs + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), + Grant.typesWithData.contains(g_type), + tlCoh.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), + Grant.typesWithMultibeatData.contains(g_type), + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() + def makeFinish(dummy: Int = 0): Finish = { + val f = Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts }) + f.manager_xact_id := this.manager_xact_id + f + } +} + +/** [[uncore.Grant]] with an extra field stating its destination */ +class GrantToDst extends Grant with HasClientId + +/** Contains definitions of the the built-in grant types and factories + * for [[uncore.Grant]] and [[uncore.GrantToDst]] + * + * In general you should avoid using these factories directly and use + * [[uncore.ManagerMetadata.makeGrant(uncore.AcquireFromSrc* makeGrant]] instead. + * + * @param dst id of client to which grant should be sent + * @param is_builtin_type built-in or custom type message? + * @param g_type built-in type enum or custom type enum + * @param client_xact_id client's transaction id + * @param manager_xact_id manager's transaction id + * @param addr_beat beat id of the data + * @param data data being refilled to the original requestor + */ +object Grant { + val nBuiltInTypes = 5 + def voluntaryAckType = UInt("b000") // For acking Releases + def prefetchAckType = UInt("b001") // For acking any kind of Prefetch + def putAckType = UInt("b011") // For acking any kind of non-prfetch Put + def getDataBeatType = UInt("b100") // Supplying a single beat of Get + def getDataBlockType = UInt("b101") // Supplying all beats of a GetBlock + def typesWithData = Vec(getDataBlockType, getDataBeatType) + def typesWithMultibeatData= Vec(getDataBlockType) + + def apply( + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt, + data: UInt): Grant = { + val gnt = new Grant + gnt.is_builtin_type := is_builtin_type + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat + gnt.data := data + gnt + } + + def apply( + dst: UInt, + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = { + val gnt = new GrantToDst + gnt.client_id := dst + gnt.is_builtin_type := is_builtin_type + gnt.g_type := g_type + gnt.client_xact_id := client_xact_id + gnt.manager_xact_id := manager_xact_id + gnt.addr_beat := addr_beat + gnt.data := data + gnt + } +} + +/** The Finish channel is used to provide a global ordering of transactions + * in networks that do not guarantee point-to-point ordering of messages. + * A Finsish message is sent as acknowledgement of receipt of a [[uncore.Grant]]. + * When a Finish message is received, a manager knows it is safe to begin + * processing other transactions that touch the same cache block. + */ +class Finish extends ClientToManagerChannel with HasManagerTransactionId { + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} + +/** Complete IO definition for incoherent TileLink, including networking headers */ +class UncachedTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) + val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip + val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) +} + +/** Complete IO definition for coherent TileLink, including networking headers */ +class TileLinkIO extends UncachedTileLinkIO { + val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip + val release = new DecoupledIO(new LogicalNetworkIO(new Release)) +} + +/** This version of UncachedTileLinkIO does not contain network headers. + * It is intended for use within client agents. + * + * Headers are provided in the top-level that instantiates the clients and network, + * probably using a [[uncore.ClientTileLinkNetworkPort]] module. + * By eliding the header subbundles within the clients we can enable + * hierarchical P-and-R while minimizing unconnected port errors in GDS. + * + * Secondly, this version of the interface elides [[uncore.Finish]] messages, with the + * assumption that a [[uncore.FinishUnit]] has been coupled to the TileLinkIO port + * to deal with acking received [[uncore.Grant Grants]]. + */ +class ClientUncachedTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new Acquire) + val grant = new DecoupledIO(new Grant).flip +} + +/** This version of TileLinkIO does not contain network headers. + * It is intended for use within client agents. + */ +class ClientTileLinkIO extends ClientUncachedTileLinkIO { + val probe = new DecoupledIO(new Probe).flip + val release = new DecoupledIO(new Release) +} + +/** This version of TileLinkIO does not contain network headers, but + * every channel does include an extra client_id subbundle. + * It is intended for use within Management agents. + * + * Managers need to track where [[uncore.Acquire]] and [[uncore.Release]] messages + * originated so that they can send a [[uncore.Grant]] to the right place. + * Similarly they must be able to issues Probes to particular clients. + * However, we'd still prefer to have [[uncore.ManagerTileLinkNetworkPort]] fill in + * the header.src to enable hierarchical p-and-r of the managers. Additionally, + * coherent clients might be mapped to random network port ids, and we'll leave it to the + * [[uncore.ManagerTileLinkNetworkPort]] to apply the correct mapping. Managers do need to + * see Finished so they know when to allow new transactions on a cache + * block to proceed. + */ +class ManagerTileLinkIO extends TLBundle { + val acquire = new DecoupledIO(new AcquireFromSrc).flip + val grant = new DecoupledIO(new GrantToDst) + val finish = new DecoupledIO(new Finish).flip + val probe = new DecoupledIO(new ProbeToDst) + val release = new DecoupledIO(new ReleaseFromSrc).flip +} + +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ +object TileLinkIOWrapper { + def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: ClientUncachedTileLinkIO): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl + def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper)(p) + conv.io.in <> utl + conv.io.out + } + def apply(utl: UncachedTileLinkIO): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> utl + conv.io.out + } + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +class TileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOWrapper extends TLModule { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.out.grant <> io.in.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +/** Used to track metadata for transactions where multiple secondary misses have been merged + * and handled by a single transaction tracker. + */ +class SecondaryMissInfo extends TLBundle // TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId + with HasTileLinkBeatId + with HasClientTransactionId + +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ +class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + } + + val g = io.grant.bits.payload + + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounterWithHeader(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } +} + +class FinishQueueEntry extends TLBundle { + val fin = new Finish + val dst = UInt(width = log2Up(params(LNEndpoints))) +} + +class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) + +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probe Probes]]. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val client = new ClientTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.network.release <> rel_with_header + io.client.probe <> prb_without_header + io.client.grant <> gnt_without_header +} + +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip + } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) +} + +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** Struct for describing per-channel queue depths */ +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ +class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: TileLinkDepths)(p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths))(p) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth))(p) + } +} + +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends TileLinkParameters { + // Some shorthand type variables + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData + + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt + + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready + }} + arb.io.out <> mngr + } + + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + arb.io.out <> mngr + } + + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) + } + } + + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_&&_) + } + + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + arb.io.in <> clts + arb.io.out <> mngr + } +} + +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ +abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip + val out = new UncachedTileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ +abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec.fill(arbN){new TileLinkIO}.flip + val out = new TileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Appends the port index of the arbiter to the client_xact_id */ +trait AppendsArbiterId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = + in.client_xact_id >> UInt(log2Up(arbN)) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt +} + +/** Uses the client_xact_id as is (assumes it has been set to port index) */ +trait PassesId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +/** Overwrites some default client_xact_id with the port idx */ +trait UsesNewId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientUncachedTileLinkIO}.flip + val out = new ClientUncachedTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec.fill(arbN){new ClientTileLinkIO}.flip + val out = new ClientTileLinkIO + } + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. + * + * Won't count message types that don't have data. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(in.fire(), in.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param max max number of outstanding ups with no down + * @param up outgoing channel + * @param down incoming channel + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt = UInt(0), + track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val cnt = Reg(init = UInt(0, width = log2Up(max+1))) + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) + val do_inc = up_done && track(up.bits) + val do_dec = down_done + cnt := Mux(do_dec, + Mux(do_inc, cnt, cnt - UInt(1)), + Mux(do_inc, cnt + UInt(1), cnt)) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala new file mode 100644 index 00000000..d7573aec --- /dev/null +++ b/uncore/src/main/scala/uncore.scala @@ -0,0 +1,129 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object NReleaseTransactors extends Field[Int] +case object NProbeTransactors extends Field[Int] +case object NAcquireTransactors extends Field[Int] + +trait CoherenceAgentParameters extends UsesParameters { + val nReleaseTransactors = 1 + val nAcquireTransactors = params(NAcquireTransactors) + val nTransactors = nReleaseTransactors + nAcquireTransactors + def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerDataBeats = outerTLParams(TLDataBeats) + val outerDataBits = outerTLParams(TLDataBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerByteAddrBits = log2Up(outerDataBits/8) + def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerDataBeats = innerTLParams(TLDataBeats) + val innerDataBits = innerTLParams(TLDataBits) + val innerBeatAddrBits = log2Up(innerDataBeats) + val innerByteAddrBits = log2Up(innerDataBits/8) + require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement +} + +abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters +abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters + +trait HasCoherenceAgentWiringHelpers { + def doOutputArbitration[T <: TileLinkChannel]( + out: DecoupledIO[T], + ins: Seq[DecoupledIO[T]]) { + def lock(o: T) = o.hasMultibeatData() + val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, out.bits.tlDataBeats, lock _)) + out <> arb.io.out + arb.io.in <> ins + } + + def doInputRouting[T <: HasManagerTransactionId]( + in: DecoupledIO[T], + outs: Seq[DecoupledIO[T]]) { + val idx = in.bits.manager_xact_id + outs.map(_.bits := in.bits) + outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } + in.ready := Vec(outs.map(_.ready)).read(idx) + } +} + +trait HasInnerTLIO extends CoherenceAgentBundle { + val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) + val incoherent = Vec.fill(inner.tlNCachingClients){Bool()}.asInput + def iacq(dummy: Int = 0) = inner.acquire.bits + def iprb(dummy: Int = 0) = inner.probe.bits + def irel(dummy: Int = 0) = inner.release.bits + def ignt(dummy: Int = 0) = inner.grant.bits + def ifin(dummy: Int = 0) = inner.finish.bits +} + +trait HasUncachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def ognt(dummy: Int = 0) = outer.grant.bits +} + +trait HasCachedOuterTLIO extends CoherenceAgentBundle { + val outer = Bundle(new ClientTileLinkIO)(outerTLParams) + def oacq(dummy: Int = 0) = outer.acquire.bits + def oprb(dummy: Int = 0) = outer.probe.bits + def orel(dummy: Int = 0) = outer.release.bits + def ognt(dummy: Int = 0) = outer.grant.bits +} + +class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO + +abstract class CoherenceAgent extends CoherenceAgentModule { + def innerTL: ManagerTileLinkIO + def outerTL: ClientTileLinkIO + def incoherent: Vec[Bool] +} + +abstract class ManagerCoherenceAgent extends CoherenceAgent + with HasCoherenceAgentWiringHelpers { + val io = new ManagerTLIO + def innerTL = io.inner + def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def incoherent = io.incoherent +} + +class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO + +abstract class HierarchicalCoherenceAgent extends CoherenceAgent { + val io = new HierarchicalTLIO + def innerTL = io.inner + def outerTL = io.outer + def incoherent = io.incoherent +} + +trait HasTrackerConflictIO extends Bundle { + val has_acquire_conflict = Bool(OUTPUT) + val has_acquire_match = Bool(OUTPUT) + val has_release_match = Bool(OUTPUT) +} + +class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO +class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO + +abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = + Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = + ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) + + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { + val a = in.bits + val isGetOrAtomic = a.isBuiltInType() && + (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) + addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) + } + + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) + + def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = + ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) +} diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala new file mode 100644 index 00000000..65c5d6cd --- /dev/null +++ b/uncore/src/main/scala/util.scala @@ -0,0 +1,106 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import scala.math._ + +class Unsigned(x: Int) { + require(x >= 0) + def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } + def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } + def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 + def nextPow2: Int = if (x == 0) 1 else 1 << clog2 +} + +object MuxBundle { + def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { + mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) + } +} + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { + val io = new Bundle { + val in = Decoupled(gen.clone).flip + val out = Decoupled(gen.clone) + val cnt = UInt(OUTPUT, log2Up(n)) + val done = Bool(OUTPUT) + } + val narrowWidth = io.in.bits.data.getWidth / n + require(io.in.bits.data.getWidth % narrowWidth == 0) + + if(n == 1) { + io.in <> io.out + io.cnt := UInt(width = 0) + io.done := Bool(true) + } else { + val cnt = Reg(init=UInt(0, width = log2Up(n))) + val wrap = cnt === UInt(n-1) + val rbits = Reg(io.in.bits.clone) + val active = Reg(init=Bool(false)) + + val shifter = Vec.fill(n){Bits(width = narrowWidth)} + (0 until n).foreach { + i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) + } + + io.done := Bool(false) + io.cnt := cnt + io.in.ready := !active + io.out.valid := active || io.in.valid + io.out.bits := io.in.bits + when(!active && io.in.valid) { + when(io.in.bits.hasData()) { + cnt := Mux(io.out.ready, UInt(1), UInt(0)) + rbits := io.in.bits + active := Bool(true) + } + io.done := !io.in.bits.hasData() + } + when(active) { + io.out.bits := rbits + io.out.bits.data := shifter(cnt) + when(io.out.ready) { + cnt := cnt + UInt(1) + when(wrap) { + cnt := UInt(0) + io.done := Bool(true) + active := Bool(false) + } + } + } + } +} + +object FlowThroughSerializer { + def apply[T <: HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { + val fs = Module(new FlowThroughSerializer(in.bits, n)) + fs.io.in.valid := in.valid + fs.io.in.bits := in.bits + in.ready := fs.io.in.ready + fs.io.out + } +} From 8832b454ce9f4d1d0f77d9737484f492ff7fe689 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Apr 2015 15:34:56 -0700 Subject: [PATCH 375/688] add plugins to make scala doc site and publish to ghpages --- uncore/build.sbt | 2 ++ uncore/project/plugins.sbt | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 uncore/project/plugins.sbt diff --git a/uncore/build.sbt b/uncore/build.sbt index d78d02ca..35b7dcac 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -8,6 +8,8 @@ scalaVersion := "2.10.2" site.settings +site.includeScaladoc() + ghpages.settings git.remoteRepo := "git@github.com:ucb-bar/uncore.git" diff --git a/uncore/project/plugins.sbt b/uncore/project/plugins.sbt new file mode 100644 index 00000000..4f4825c4 --- /dev/null +++ b/uncore/project/plugins.sbt @@ -0,0 +1,5 @@ +resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven" + +addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.5.3") + +addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "0.8.1") From 4cef8c9cd4b1334d9705932f3ba226960b0dabd3 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 10:55:38 -0700 Subject: [PATCH 376/688] Added MemIOArbiter --- uncore/src/main/scala/memserdes.scala | 65 +++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 7497e2e4..a4944e1e 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -60,14 +60,14 @@ class MemPipeIO extends Bundle { val resp = Valid(new MemResp).flip } -class MemSerializedIO(w: Int) extends Bundle -{ +class MemSerializedIO(w: Int) extends Bundle { val req = Decoupled(Bits(width = w)) val resp = Valid(Bits(width = w)).flip } -class MemSerdes(w: Int) extends MIFModule +class MemSerdes extends MIFModule { + val w = params(HTIFWidth) val io = new Bundle { val wide = new MemIO().flip val narrow = new MemSerializedIO(w) @@ -517,6 +517,65 @@ object HellaQueue } } +class MemIOArbiter(val arbN: Int) extends MIFModule { + val io = new Bundle { + val inner = Vec.fill(arbN){new MemIO}.flip + val outer = new MemIO + } + + if(arbN > 1) { + val cmd_arb = Module(new RRArbiter(new MemReqCmd, arbN)) + val choice_q = Module(new Queue(cmd_arb.io.chosen.clone, 4)) + val (data_cnt, data_done) = Counter(io.outer.req_data.fire(), mifDataBeats) + + io.inner.map(_.req_cmd).zipWithIndex.zip(cmd_arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.tag := Cat(req.bits.tag, UInt(id)) + req.ready := arb.ready + }} + io.outer.req_cmd.bits := cmd_arb.io.out.bits + io.outer.req_cmd.valid := cmd_arb.io.out.valid && choice_q.io.enq.ready + cmd_arb.io.out.ready := io.outer.req_cmd.ready && choice_q.io.enq.ready + choice_q.io.enq.bits := cmd_arb.io.chosen + choice_q.io.enq.valid := cmd_arb.io.out.fire() && cmd_arb.io.out.bits.rw + + io.outer.req_data.bits := io.inner(choice_q.io.deq.bits).req_data.bits + io.outer.req_data.valid := io.inner(choice_q.io.deq.bits).req_data.valid && choice_q.io.deq.valid + io.inner.map(_.req_data.ready).zipWithIndex.foreach { + case(r, i) => r := UInt(i) === choice_q.io.deq.bits && choice_q.io.deq.valid + } + choice_q.io.deq.ready := data_done + + io.outer.resp.ready := Bool(false) + for (i <- 0 until arbN) { + io.inner(i).resp.valid := Bool(false) + when(io.outer.resp.bits.tag(log2Up(arbN)-1,0).toUInt === UInt(i)) { + io.inner(i).resp.valid := io.outer.resp.valid + io.outer.resp.ready := io.inner(i).resp.ready + } + io.inner(i).resp.bits := io.outer.resp.bits + io.inner(i).resp.bits.tag := io.outer.resp.bits.tag >> UInt(log2Up(arbN)) + } + } else { io.inner.head <> io.outer } +} + +object MemIOMemPipeIOConverter { + def apply(in: MemPipeIO): MemIO = { + val out = new MemIO().asDirectionless + in.resp.valid := out.resp.valid + in.resp.bits := out.resp.bits + out.resp.ready := Bool(true) + out.req_cmd.valid := in.req_cmd.valid + out.req_cmd.bits := in.req_cmd.bits + in.req_cmd.ready := out.req_cmd.ready + out.req_data.valid := in.req_data.valid + out.req_data.bits := in.req_data.bits + in.req_data.ready := out.req_data.ready + out + } +} + class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val io = new Bundle { val cpu = new MemIO().flip From 62b6f24798ca846f2982b8b22cbb436725791440 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 15:43:06 -0700 Subject: [PATCH 377/688] Delete TileLink0.3.1Specification.pdf --- uncore/doc/TileLink0.3.1Specification.pdf | Bin 172795 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 uncore/doc/TileLink0.3.1Specification.pdf diff --git a/uncore/doc/TileLink0.3.1Specification.pdf b/uncore/doc/TileLink0.3.1Specification.pdf deleted file mode 100644 index 23666814d9e8738924e59e5f9f8872e7fa58d725..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 172795 zcmb5U1yEeU*6$lExJ!Z$?gI=uxVyWC4DLEO2@)*0J0Up1-6dFpdxAr7cZUFPIP%?d z>c02t-YTklX6?1Q|J`ftu08A@otl&k8z+z(jc#r4U>Xg?0pI{QncAWW39)N=J6izQ zB_U9Vy^|FhyBfsG!VSO)yRLyIB7$b&X#U~{dNKT0z>D9Dx2uIC^k44m8WwI&?yhFA zc-XjVu1;o}7Em{UAvFKrLQkWjO#K z(7z?H%Q(TNfLUc=R<3_o*=0BZAnt#C1OY%i|N6)U0P+4i3?`mk*}~igBH`o(Fnn1s zFDD4V2L{0=6L)lUdQtX2%m0S~ja}2-6#9?tP_l8fLt~e4ay7Sb{bw)-EKI|~3<@yh z<x??;0ATIfH| zyTnkuesDcybHpK|)O7z5%eo2M)pW0_Oh~DZ{ct)`-4vrh_f1PG-2{2xVS zI87fSa(VyU4nHsmvzPndp>UBtTQ{D5M`W#j`b2blca!sUYIv(~c5u`6m%wR%@1&os z^W*b_>NAPN@$>c7!|E549~|#oezXK8QH!nn6>4o@X&}jARizKjiGNH4JQ8lm2@c=@ zEZS{j1ykaXW#`3g@cp-momXzuPMdQq;D^JOmshXC^ zNJR2?xw!pn$$S2nRoxM#AH7)EXnp@gGl|Wm{4SCQsW>308z|oE$e#;_&n#$|(#oo?!egNpp?m=%Hh_wnVV>c9f`B#%A;<`!S{Z2s6} zJ$d2upP!yZX)o?N^fR_sPsrClUhbDWZ z;Tn@N=+WePf0?&XHyS0a%cO?2jD)ajru47_CaU-p3NjM=kcbLD;0yQ#{bj{`18v3( zW3g|Z8WCh0Y7`$;Wu0oIGxh&6N%ba_B@3|7nEWQb{Y37|kd}2WD73d#>;1&V%;riM zk|%z>WBp}xNV&4&f(Tn-fNsjfU}`T zT_2T*Ki0t8f{6E~ywaSDQ+1e928HtISB)ofr{G!k1&!BHRi7aYTVc2MZHYYWZI1zr z7m_zL-aa1?)cuMcYK|ER66G`Kw#cQtttM^|q$)tUCl$c^9@5%R1@+VO+=uTS-Av?GuuspJTM$X*@H*LhNJV@GFK0A6v0IdZ^W96H(j`4B z1p+A@+SIMT@H@>Tswg^tjhjiX?lTzEf6Md;5K6Z|whyzu9LAbE-Y!oi=Xh))0D&Wr z!&{w9X-5OUaR^TlEVhP`w~mkT zH{I5qE%)c4o3dp6+{%Gdzq$}@eV94N6~9h%BQ?+J8#F%au>Tx_v)KY2Hnz=ATg*8W z6RJ>gxjoqRghJY_g$(7!fRdINlKm!=G~moo zq8q%{y1gW3>0;!Rf~nvyy(1F|2TR^D#3%@4@I0;p_qWanzU28FS^&Z0b~3K+o^@W)~PJjh4b=3TsN$GwPS6r<8wcJA(L}^w1B0JoZbmj zkH_f09Xk&JC4%k!SAJIkT3b#dSAGqFC7;k$1FoCdE~(~zT>J`^Y8A<6<4P>V|7aYM z&TG<3`2DW)Ol1)J?DS}D3!A+=GHjlI>0d}|#;4l?Esc_u)B&+=wUJCZ+ ziQl2`U2eAMO6kIHLPGNkiz9ME^Jx=PC!Whg2qY{SxM%O1PsTS%m+g>4gmr*QohB!u z{TNT078Hg#h|PKC^3B{_o`qLv+~NTejbSONbsAuu>i{X1j7A(gmc;#M&CEYrqp zxs&RA?4ot;GhN0tQWY1@5kj6OhS}%FxyHIp3v)779w1E}1T|wkD}-^3OU{(*-5~>3 zYwBHYpLkD<^WjS9w#D&KaTwmV_2d3PxSt)~&Rm>*I?e0b+zT=Pw;tW1>!6Zbe;+3*NYZG`u}Yf%Z*#A*NKw823y z=`QS|e&hC3WmTbS#Y4tt^owVw%`BwdGIDxel^U6Ks|0MvK(NtycQVES_q(7jme;uk zf6ZOL4_WGnFUkTJR$pZr&=IE_(r^M5v|(xy!2Xd)^p(fSL;`y@Ov_lpXy4*vs8r{& z=Bs%ulx(NigI~+Ji1Hf7h7jjI~?TXPsoQR_TompGXY z1;=t1E@>Tze}a>;h6~W@MvhA)u#O1#K6(VpiBFcPu^zAy{46Uplaz3Avo(*ga=p${ zr^m~0Q=xCX+{==3^&a@qe@I1D$NBSZ4D+VK0Sb;ho3!<(fvamJ73D*)FS>g86k>Q1 zj(&)Gk8`<%Obzzr;b~zUv)cFUtPlmUbd{@_1ZYlFq1Cnmt*xDKRH5^UqQYij;a-Um z)iZBXMwaj!Yen?_{E+&>qRvDXhquNWQp2d!U|C%``pg4JuAn&z&Y7u!4{c&*#80DE zssU#&U3Dt=y}fo{I<`b20dWFP2DX_obmch3?i;(%?LCR;qvRo|l-6F5T%i;;sxb~B zk9A{10qHi+^r<6tYnF+0z(B~r(+vgr%5Q~(%?fuJ;#UV zriT>emd|}eP=}Zq)I1{KS=g)oZl`MW7t^Xw*++Zk!%1cvB71VeVSe9id>Cf1N0z$S z@0g;dhTURxh#cut9)7^&mA@r9`nouajFsSHV3;ArW$_vy7@}I z06;x336?s&mK;aEDKtTMqSXG@7yx2I-{cr!fh#RBu3LNi#kbWyM&e>PiZ{QBJ^xslQB6yX6){L* zq2!e%qo53wMT{|R1OUjdwY)Pok;XofFyeC6wNV<=LNEBBLUXhFu+rUe!4YjeCsPmx zbjioLAek36eqSu!!d`w#M5JoEMx0y`>1ka{Pm!2a3jaljOB};r%aW(RdtI@Ttc%dk zUg4_{76911U{pBar&ilPmiSDz=*5pdR5OZ_q~gE+230~34=ZG%M8WY-+74Wt3a@@l zl5tqj`kXFmu_P5iBJ}*1;-UrZhH)Pv7m?~zLt=sKrM>KtZnFhiE0*c75WDo?v;2yS zh@+77P&>_Aw{f;5+^E4Nq7#%Kmu!Y)h^`bednXJFb+2$U9f@ngJdH@#e;RZwqH)3* zf0>3TmaS}9^Wrl zee9c*{LSjRkqAj&XAmJK63w|a*P}`T=0qt{8mVuLRapf>ZOS^#5!44mYg-i?L$s91mhMtI$h%o8^1bUv zTnbT@Sa)sTh5`NdK}O>@Md3I(ae^SS==IwI4A_0imS}94_*k-~%z&p|jGn{o6_Gi| zpgcMm+$Ulaz9KQelT>dP)UK~~`n%_HCTAMAqpiI#(;{X4n`l2@DuZ>T_sW;A%INEf z-dQ>}XIw@qaMUKZYp!{3=DQ^nz?u+acQ82IkM83G-p!G3Y&~UAGoLEhS&wltn;Y4ZP3F_ZJGme0hcWIA`KA4?C6JD!Zn8Eg z7&qxoosN>vp`=19=23cjQ=*6sJlG|oT0A!Xj-oCj${J^|n;lo(FV{MuinDd08IH#f zJ`a$~5uJ*$^i)v~8BVwIxHdXd8~u!}IYE}2w8yAbXjssv)8QVMDWIAMz1_>(-FO-?9ZZI!%v620j-O?2}rzj3gP$I5Z`L;r*flT(aW=u`yx2cwuI46f| zD$ro!GmqPOpZh2vB2K?H@ub=4HkyfTkVOca1EK$*VmiRqX(ZhxZ-it#d4e+bMcUU!B9dn1Lg>e1>+ z_6suFuzC_QjBVGf@&LmSqb}P_!=|N|usMztQAbI9UuEdOw0!Oh0A;;mob{>}@F@nq7kIr;bsIHj0uPy`r zvKpIBy#YSf(si6ucjJpmBl-Xfdf*CFB(8t~83Wz)9$+hyeyv>E6L>LBZHnT_WWk`$ zEP!opXi1gq&fIKh$;qop`B;`{GsTPxYWTQj1yVXeuN$5lGr(>4c^*>H6*AM8Xeo7P;k-)>?zCdJ0Pv!@+v^ok%S7{X)Peuk` z=OLuAG|Q!^HBTZ8CSyt!O8?kTvaS7W5h?m-BGoJxyU=*8FV9zeh@`hBP|kqk1Fji= z7P7&oZN&x=BAu;iVHDb>%RyV5rg0<+KsXX^&FE$?tzaR^I?x%OVHtPl`d%fzei?%uXPn)yfRvNRU9v&`ADuIayeh!Q=pRZCGN4xc z_2ynwHH5O^V2I@|Dz&kD z%s}aW#DQEYT=U_Q??1(8!bX>a$Vf=Qhjy>s4Vj?h(4+_>X?N~YK!}HgoGFu*GJ&$U zoIq)!3QoG7xV%pGim;ZN4{n~PgR4A|U;#Z_)(O9Ol3}EkwOddA_MSD6L5Kr? z>~~9s+V`1KS@h-eZ!v3Rx?^5ng$Cio&qTJf7M8Y3D}J}rXzWcC6A3@=lvdzN6-!q( z;y*~uiqH~C1L;jd4w?8oT`$cf(M+c-pd@dS3B~cO2xO(CcDH6l0P7!uKVC31gqId6 zG$A-p?50#c95mF8Ec+~vfOs+DHo+XD ztf9U3RmtfbWh*&ZLHZi^n*@hnK6$EiYXuYh*f*N9R_}PL`}}D%H7UqIE?E-u*r@vI z)rVg~nG51UKlU{;o2b4-istqDUoUW~4_g9@eCeP#f_i^KLiSN`KM>Vj9+r^ZXy*VH zmOGhG`B-%h8kE^?w2LL(+^3;>ckpc{e~#N5wPQYbeB5g?q{Zn>URQVZeCH&mB@@`( z!~R%Q>&)^)NL$79+YMJ|bm6k<4sXlR#~*wRpqmHxWykyG$lOExb6{w#vc;FCO#4K^ zGoVQO0fbO88~xks26Cj6;IRyE=G3_PdKY&*HO&ul|N z0eSJ-z!&g=sDsjtU+*w}+KbVW;G`Z1iZJz&T2WY!Gn4Fl27O?|6 z4`&t0!`>AGx9^EQcThS0k(mx0Z6T>D`h+Rv(~N?Jn4*gHNwrEP9- zTaj&Uk{>Hzf3{iJKQpChP@9eX#;kriD)2V9#ZNr`BmYQb#Mv&$mLa?cbn?@B+M_-| z<@E3y0iS)Ap4R?4Ir&^R>e;aSAL0VS%x`&fwIHvyw+zuX^TLgw&9uNG&h z8O8GZL${`@3up4`aIg(+&5EV@#4fKY(lH~?Zx=J-+5!hAsExDRly$Zp)Vs4iIPP+G z{fh9VM6chp#KoXZ+2XEP$3J48eD8i29>#mMSA$i9L=SR6AN-E#u#z^gIe^lQ^oC@< z{!fc28FL{^#*ntPxhFl~gbVvgar_F4ObN9kGFeIa4-smNY8DU;!(BkUpzf(sr`8be zE32DN7^9?qXKxTDxgHdsZzyj)M*8kz_hOrS@RRiY#nD*7+-BZ*Ti-TY@Oe|bu-%|p zwua)DS2+AAID1WbvU$<_yYM%1e#jtNqQ~E>Re20;vg^i^=C(jpa{=-CVSnYm+EI8} zB%@1nKC_uJ_64dYWBeL2gB3z$71I8U@GZ5)s?T~@1I_fN^9!Rqs_>Efd2+T?pOM_) zRBW}E0uqYj$hTycID7;yO+ttmxm@{&^K<$4%=pNid8Xm=%&=LxI%vj!DRD{1>eVmJ z^x{u5h1ANpf$QJDp0k;=@IWAD#5yFyQQAJo@;iDvqFpP?>f^c%ogY`Xz96W~3pwbc z7G|tBE#GHtYodCme0}|eQ`nZS_fRAfs%tf0|8?v22SF^NzBCJgp{eE^dP`Y-gARP- zqHmkL=+Fs4z0;)$8MoWj=AG@nynEK3{a#OxK4Ct06S+oW?xfCNhs$1=4&c4h?agV~ z9Y(0%T)heY)%c|!w9W2MOcF0nx*+O@D|2X_AN!Bi*VCSLNgPG(B776DdQ7{wH9WCo zBW-fX0{HxPeMdDPDm`k@8T8GONR}G8x2~KQhsXXyWyxswRAp}6a`~BLdp}JP+3|Hv z=k$8DBj8U&e=n$)#Q324_*i~bEV$uM50l?e{|c^ZA9emlT%_u%&WyKv-9Um}8O~q# zx*%8O@BO-&%2(UD`e~0dqdh;gDqnkNx|1nq!=3>e7gim9ZGAky2=}vPkWKBVP`m9@ za2?l|5WzCO9*b$Ypy81fOY(cww0j0MZ#}|mtJ!`rXa&2{$W08en`pG$5IA>I&jvz&#dQ6L zSPK{_5i^{!?wgS;iLOfY_RBlhGX4cidvUiiucM#D>e1x1H0O@Sab?>hxk#u>_>l?( zUt5`rj~U6a+hKv*!)-Oa+sCk`JIK#a)FCY6v$ih$aJNR~;CJ4GII%S%pGEkNjjZT4 zJVkO>Dq{h6iy<<&_hhay)03$YE(0C3sxqEtVKlG!zoUp9E~0c$!k$GCzvCy4hnPdV zQJdz-;G8*vy$7y~4BNM<7Bd}p*(4Xn^cjr@MISUay!H?$D-JY)1?y=n1+`Tfo^pjz zP1)i$;+k7OkUSQ}P!0rlmdmqzXFc8PpaiSBYvbAObrX1>N^ae(mdnALOqvYgPnN1C zCf$3M#=EvVB4W_0J|h#w6H7u$k!fn8DHA~Cft%o1QI#dC)p&`Mu z!PH)!U$e9goZ`3>q5|Pb%wmLrJtEKUy-)KuE4c{a) zZNrxme5B1wG!m9TqXd;M$5VG&t9QYz`7NU6DOAP1CXI2uK>X$eDEgjj^#|{UG>DEA zb#|5<&|12`Pf**R6^~%HGi=GWexZZ?A;AS9BD)7d2sOyBz)qdpbTy#)vaQ8qJFYEa zb)ea8B@g>`a2Twl?|B_nF4K*O|4{BaMBU08rAc&E51HEdDaRipqO>ti{=1~Q&OP_C z3}t+t3uzZD>zNB@5JOYNFkX~gT%)hs%kzEsT}h~&b&9eh=IjKk;a*()pHkjM32h}( z%t2#D6UjPz;&{ZpP`P6XC)uSHXh*0e>>Nrvb6(DgXf&9~p9e`~A~2xa_ZuZz5gR{5&7;^ox!%K+A{1PH*SOq=b6^|kLL`ANDSTq0T@)-(b7;(cC{cWom%76K zcOyl3qdPx4s4X`FRD=ECBD9;k=OJu;flg(oqrunZdIl78Ra-(xY`tqq-x*zc0xcT{ z%#4nI5^~yhiaTjlpngYHArr4kX?ytL-S<{?^K#%PWT}?rny-5NM}X3P1ftif9oXUP zh&Ks+m{L3uN%hRTWClZC$)Nk!c{>uL2d)0q11j+pr^j{Ed7^${^19CNDO|OCFWZIY z!-%eGqEx=v&mny!bMiw)U@F0s`haSdHwpmzt+^<_jjx}~+a54A(maTXy3O2`%NN`< zWrcq*yp}j>)Wi-asqdREBpZW_;kf{9ZI8x{#!8WVBpDJ)ga|+N*@dSFfh&_fh-sN& z{&>!7v_eqtPi<11#=mIK-ftCp9gwDM z)mx77kuMvwPU-LRKMh!Hp40OPraUm){*9vmsipKu+n#+I(!eb%uRoo2TJqR!z0bqz z&8Ms#j)R~xhqH*22YY}m@AcIO%FIVpZ@bH+`FemhSOtf0YjyRe<}N`)KmU**#ZbIX zOU)4=Npkh5`*F{|{aHcmPVQMEDUR#W?vyY77xu{s4a?E&z@B5Oj)y{m`-#3m4sEUW z#Um$`EJ{Q9_^bJa%s4pNtD`vgKcoY{ly3s0esvSg;tzP7Zb_oepq-G$FvcJBq={xO zMPZX}VAa)_Us9QduD+&5ChD`B3mp}p@DAHsCuIY9%TnR{U;OxO@{w+EdJ!60@Dma| zBf*F(Y@^Q3s~umak&n6TKV8>uL><9hQGsDZ&0JlQi|551K276Ue}!siMOB|em1t+k zYpHsOht=l<2IQVoBs9vI4!h)Zxdwcpi*mT(= z%joVj=y2h)w6Ny;k$2JkxH{q?=OaMu;j2MCWW^2<;)6pJ8_)LAIa!E>*(Xm|#V(mA zaQ#_YcjW}*QqC0a&~YiP zC~Hb4K|!C?q=lU>)9v-o(*AMv!yyDj?WcghP~H!stNfXrIZj#Rxr9UwU$EG$LkRUm zJ|xtW=}3*5`mC zJ`L)cV#I2))|zl>-e!1iNo8%3?rKn2SGYn&e?8u4q%+!*q}7Rz_GIY&L^sPXVtwT2 z4j~I}4=f2o0LclhT4ZQ5P9z279oi7_LTJr|#i1-#dQKD5tQ~Wh0Kd1mKOvYMVltmQQcqc3W_A zQqx`RiSgN9nx#9z0(Z;VjE-Rv(aniiXir`d2=9PHZ+hD?^lKL!RU)8zmzcwyr+Wd^ zGwacU>VXj7Yst8>z6TzY*!-46Ce*p9gpP_s7#d4BBB;ms;}~A5e=fu*Z=IZG#7TT3 zyDoP{$8$ea>KA6}uWVaGP7I69CabWP0(8uyFV|miMP=cr!$1i@6xLRBHwD((+XSH$ zT^Vk>3nVxvaV7f`lsbJGp;+kE37QsLDbJ(t?wIJ)oUnOO3MY2JYD;mtL^P(ZI_$N# zA#GLFjG&&(X4Q-n4oIDo{*I@5Nrj4J58u$S7>BLuOO_3NjwVLgnq|hI6&iVJEYBzi zK`Xrm*3Y;OL6Ym{1$($y1=Yr4R&;|qXuR+l!as&)m*I`V*_dhPIRMUE_g#d)F0=+X zC7vRbve|S}p9bus1mqc;^EA27Bvn7)aULKze^%OjbtJsu7B)DIczJ*EbC^#-%U8iT zKf-}{J-snMY4$h661I`c7OU2MX>)BX@i8)mo~2vPNmwuAZV|4j2nnP8@x^0+E3={Q zDWV_}(baN%%>swL`}zZ%!`sGi1(Y((-ghXHzTPo&GL`pAkarGm$0Q2SmqKeG@l6;S z-{7Tw)lEw5&Nx@cZSW(8`{@)~bFJACG&*jG*RR_VjKwf69EK^!om2|PeGoWmZ08xg zy?Db$cAa>(OC;)nzRx5Y+1f8|E#Vue{_@&EV}fklM9j1LkC_{jU>|So5&kN$se;aS zo|;%7x}SL9A%-RB~tO-N99ASu%jiVKe@-m0fMK(}xHaRqQ zDGN6xB^eZxvks8wK~mnf+g>_>cWylW?*(|38=W5B%-FHihe-?Aw1pc`vNn|CJmWR|^YG z3s)P<|8pJxpzQuz$3JPi|IqQ0!2cH=FZq4RwyTXZ)X5dV^-nbK|EpU5hrN@OmwJhO zsVxw!ishxWoMh#tlp)UltB(F**7SCRS~$o%S~|U8cN!K}Hn5w+8^9%QQ9hCCcE>F0m}eVjlLFPO*pMXezZKN!f* z&GoW89xy);2L18CAVMx6h@X?o7{JX9t6e^RAQy<^B{MJ|4qhM^H{S~}_m8hIDv+0x z?xWPO?&X=J7vG8#7 z0J%9iKrlsMAn%J04@@MO2UeXt+^`AxdHDb!epnO_?~4o`4sO_xFrgqYHxLZs1OxcF zIDi~{{Qu$3!^I05i4$h|_q4E;^Ya6F`C#tcumSk^d0_K_d4b&guvPwh8e`-CxFi3O z&HpXw|0~^S|G+k3$e;ymq<>O{|FEb)9ALQVf7}-^M|+5s+skPr{&F%v0bnkEAefh% z|KDc?6u<@ofqX_~Zle zzYO~S!$5L!@Nja%w&;IBkVn4Wep=GMcfYmXTz7QPb`&H+QRE$qN8#aWaTCM>LTPDf z>4U4?1LA@U_?J$J;50f!?p2h z?_+Uko21y|%E_qjs_4m$`_7VE>Hbowrd63{#Vfd1!sQ5XpoN8HhhCX$X*ekgc$^%A z-_KI(>QG>D657t2tgaTf2An543e? z@BK){?yWY4RIW(8&s74b%3fI;ZcK&pD*yiJ@Jb-dWC2adBUoqL82KOy={Z0TX>R9R zXa^_8$=z(mucy+d=^>RnBKv)QQN)#ar^_t_Mw9!pzhz0;2sGW;?zI$tIsq7405~N4 zf?n1`L6%2+cWEiHY|gPdlnOF`>61_mPV@vE-f-FX?U#)9)O2bQX6(pR2tO-j{u1;& zqBy;OKtg2bp$c|EQN%UHaTH4<4kif({TN@7_F>5kgV-0_hDs*Rq36wgdaM#p_IiD$ z6>LAp+#r#x72szE$vNdUkGOg7|MprGNr#EZ1^>KCYAMKzK?vX%e2ZHxM?7%D8f>!? z$Wen2L6_j-Uh@gtQ}-8L2RA=C^|3z3ni74(k=^h$o}l@QPfrDWV+X-|M4qC)o)yPt z3t_%z)01MGx`32zLv|sLrr@0u$O(!hq>O`t9+l=jNsKYgUrdZC?DqsrZ-Y;8R|2$* z_=snHxdPTiI-OqI!~Rt`@um5%Qc(^Z(Cn!2`xZ+GYW>VJSq}?BtuYqk?0cLq=8e?f z7>9>Z4nmM2MC%`elkKr=`08D&dm;xs-6vLb?K+X_xKYvKDah+BUNwrLnJi+qSc|4M zBjWWpK}7cNSc=?&2d9h}XN|sh%=CC!`fvMo-zZCzcSRg**&IvG(g@L$%eLjlhC=%< zY5cK6YBRe`A~zZanh@3?_Wlvhaia`>U+pQKEM0G$F0;yguVKJhA*2iWf z4LT8G)&1_(iFMJ1^M&M8RN(iCIDa6Ns|oi)I4VD`L*l!A_omGIvonky!;-cmIN?NB z`!1rs*w2uUl*TCOBTkc^GF+4vAA>ezIkBQjex4aUc_f)J?I#}{yyK6pCUL#Hm-fg;MrLxARiDcNGZVr?EWT#@<`SSyFaQd^E(VwBeXqPem&Z>(4rBmuV*||^7M2mM%m4R7i z(u|UNh{{v^Ne#$XiP#g;t!Aqh4ZqGN|?u)p1HMedG!GmL!Vur#@w6owg6< zc|U~lWSVgtHEr-ZwA}=Sy;tr|TP}_zeuwq^FQNDGC+E(tRB>D<%S-qfsM z&TLGuhVV62wI>dP+^Qt~?DG4^!|p5p@80uS#p*?*>c7b#GZb|jwTIz#b?34=|PPg|%+5F7b@i z$eUA4P&6@E(l+=szK8Z~m(lh!zU-6>3Y=sU_yBMTR2 zS)h5I@@i)VI$``UbLF7mVvp?|<`i_gOAiA_ZUOEY@7>hi9FDyA&dDK7dys5T-I!<# zY(IX+wTlyKMLzG_ILtcD!9R3n5G+e#Un7dfBAIw+>`GV}+X70%enVX@<)!CGdHj{~ zdo)JeN=S2bHg5SPs*bXb4mL!c2T>g1O(bjJh0%StxOftGZQidvOvsu{;yfNCIx3>E zALBDQ$(x+73;5)^v~W_==8ve%=@CE zX*JMu)%W=mcI?N?vl>;A$HR*ah6x=zmtx&d_y<2~oY`7pmc;*F0zO24WHw^4CvYUB zAXp*5T<3P_)MHqo@Ynpis+~yk?MK;X+uDp=%QCZH@a{nd1@DSQ%8%7@>J_h1e3OJH zzt~e+n8eB9^L+jxrD$rt9@lr5<6Bb zB4rP(k9AS1$4{Hrv0abm^9(@V*h^~9PVIK!INGb?pYLbhUFRYyvfyF}6?YGjbyWbdOss(xr>|JU2q#`2O~4sXpt9tWc1nyIVU|HKjS%c| z-%tb}{4r?nH^LT`pcImgNr(;q@>cFGDg!+3;Lpvscfo!Nw0TrArqq_yiQ@x1oBUw$ zCy7n~5uSKCZ9*2UKN98bAd_oG7L`dt)xhe&Y(cD_WG9`~py#IVx#u~;9dgFD)+@2b>dRFu9{bHEDR%xf&)=KtWNo`EmjcQ*tEI z`QM^{e5aIqJxHrhORCasBY+swgUbIp=e?sK7G?L2IefZYmN{O!5*mboiMrn$jftwz zoSBJcYg0i(oOzI2L%Jx@7W8R^wr;?_IcJ0ls3EzX;B7AQjSgLo{2K$NIh75qa-T9$ zyyX=9q8y99Qe}|u0A6hpE6uPxslNDvJn|l`u{qryec6ERX-^&WEjgC&d-iq-h91XT z;dV5>@07X7_cpJWFjv3l{K38DYDZ+pXUjrvFBJSa!rGLo%*K|kVWh?Ho#3_}e1^2~ zV|!;SW>B7z$5qPKb&4>i4>9wXJt%a~-9Y-cT%H(e6r+dIH(lYuD+r_TVX3r_AMy&PTnwoHk``ND+bPQNNOCe!3KwF!m<%$w(RMG%ORHz z(lI2zrho3CsYDL8)VoHxBFH!O)E z3tZUom%k0$D3z6~G?|<0asK3+pI^fGwJ9AKTZe*bCFxw%-v*kB@5-g%n;e?qSl*KN ztoJUD8H9U%sFsdtVAXuf(mgBmTLOEA@Ac|=!J9XnEi>!USJH5B^F}dyl^^1cL_6*3 zNgnKZqe4mO!x&0&q7#hTCx6usB&ty^20yjPx%V#!kCAAZME7o0LUdfygnDB(Ts=Ce z{n2kch&{v8kBn3_wBAG$+NL#BOJ)aQ|J;NpFoWu&3t+eA`A*}JId^4a(|yYTKCTG(F&!JN+LtoaBXnj9DQOZ@%0D)V|>HMz50 zj=7RNVBTY9_SHEZqwy5?Tr}Uu@nyV&bXU9RpIWODKWgvDH8XhyESu3xTQqer5zE4e z#-K<@L+ApGl;1Hx-l?DL(>jOcc}LO?7OZ+)?Ru3`-(Xkt+z;}!RetB36(QsTofIN; zlo!?#CE+WfzY`jsQ#%x4_%0yAe2%r^cu4pCAxVug@W(rnU9K~czfpbKP%aa{V17zY zSrzZ*)VJ;WsMyyDa>aa9kh^ZB2T*V!iXu*>gkID{#6-|UR5eO>^hEGPcr{KgTrNc} z76eX$t;a9e8^sOR7RLz14absZL?Z1o2SP3hL_mVDhdWpy7%i9ucL;X~r-(XwLj6uu<(Avm9*zz!PvXlqt@D#o~ynhjS+p> zN=&lmV9GeUI56RSA`u$uP5M^h>Ni)?dW9L|d|N;+trMH1uGbFN=INLjKEws#G2KFj-?H_TVr39~|GQDA)th2o zD7Ai+Dzogk%BN_*zqF~eV08K<=V=5Aal3l_HLit@KdWM>SoV9S%EpkUUOS<2l-A7c zal3I8-k$n)XKAMKb5Bu}@|d!A=6+2sX9IZA`UAta6M%y>qK0OTT4NfaRi(>%rg~qq zFhsB+cd`Oy4F!Vo8c{e1OFZ%`U;u>#5mj6Y2Zs?}+_@6TkD`g`TD3gl@so4^)5kPZ z|M9PLtG`&TMGpub1P%J18uR8>69^wNN%}uZX&gF*M?AWV*H9KkJo4ntVF#Le2ESn- zi8yLC)n-zeAov;C?^5r@NPw+wuaIP_%^D$(s*4e!qD~M^&8!7i*sgo6o9dfX>Ro9Z zY+rh&HZw=tr#YVP@#pNdbNlErrWrEc7sz z!AO~;yUq>Noz$S%J=t>yM`mH0S0by@eLlv@I|Nf57?lo^ZN5;--HZ#f0m0%|ZDde}&u%9_6s>vOTh?oH!gi@Hq9@ z>iKL3XQd+MPqYJHu;ZtPniHI*2+tDV>St5zKgLHcD~>&+GnfN4gx(UD!g^uY`^CiO z7%O%Z^pxKTY zIi&Lr#LQQFkB=W{J1U9PEoFk|9aU;Gj}8!bO=P=~`H?O+z~8s=;rf=RaI=(U33i~E zqtnUzMy*84%-K?|4N{DgW8HmI6@-eYOu50$7^D~kB{~y0*@&9v;#Li|Lm^DmI^4be zHPp%pZsyE$<`Vt9^2b*>?nN1+8Ls-j4$kI_KBs8VGAm+iDbQ&&)k`k0WT(^vm^z*| z#^iorXiecdFcl^+nx)gKr}4Yw%_ronq~%x1ep#|Aj6)X zQ{rUw;=n0o{%K~Ky+*u!8P1TF9#iqRj%0QPK`YhKhXCI}@pCG$GPjrtDT?~{*Q8?a zr*Ga)5c@nj3Ff!(R=j5D4RwZ;Y76oh*%7Mcd?*r`j_3YcX0|X;TE=6AOwj8tsYF>& zc(s;ac`c>}NhkIbQ}LkHsQZ@}7_V&E0a* z{cPvV=^&o#0a|VP+Q4tf*?Y)Vw3_kdED7@5@>KAoQ%Q;Mp1rMzNU3mnH6v}3<TC@=ZOG7iM$8`-TNy-PAqG$ zx0auJ*!%NJj~G>LXC#R(ff+Y z!n72G#~ zX;C^e;)vzWp*;EBO8vE+;RNawcWX;}bnzhN+%#^VM-eMD!`KwPp9Vv$Zfp`NUM{!g z&f;;GLT z@?S~|bTmFk_gx>DPwgGZ73M#zzV}Gqs+%x-G})EI;}iq9hG4#J8Jsxyo9v1duZ);5 zWBRMY=c|WQIIn?0aH4jZ3XM zeja%{ig1Oc?RW!4s0enu7zaq#um6%lBrTchPgzTg`fjwDs|-VN0WHH$6h&e-Asd64 zpDlHTy87g#x_Z3MWIVr{?Mlp`ac0qSm#f`Py0Cv0Y3luHlC*K}u%NDk@zKMql|S<# zc}iPl&0*{L5)=ECkRNnm+Qm#@XKOSkF4g3G-#9i)!dQP|%rlR2kYgo1A&80z?@on! zMLt4<5pF0unI2@O*`meTVJ#*UlIC`u4azRXG#L3}7F(c7aUP9Vm0IAoLuYg2cv-;w zJli8*gJVkdnfBxp?K9ul*^)PP9u1?7PMfxZW^|zpnr}36G$n~DsWi84f%;HG^4#`Z zhQo3r#UgK}y<+KvwtYBl{K#WMw2ZQMLEtEUx@px&B8r?)qFlSG0tE{W6>9b~-OeS4 zIcDEFvk<{IHcRUU71S!7C|x2zTecx+bVFoF_{5n!)YKB1m5i@td3dx8O)s^a;^);vWh^_hb>7Gc?v56zsw)=V zOx_l$+D*qG;&U8G*5_57h*j(n3U$DRF07XTVP~}GEdn@$0#Wg z!K(y-GrrhAFE%{LEfpi_emVQC8}02fmU=RFcqo_Y+n*v_<*46jd9i|to&8p#@BTqn z$?v4zY1v`VJ#ld7w@I7l(g=0?&*uhyvJ(&b@J|%Q-v1AA_ZVc!vMvsPx_jC-roEce zwryj&r)}G|ZQHhO+n%;<>+N&S-uvDg_aFD`8&OfKR%TXZR%J#-tg7FWD>747%^>7M zfQxFzs_*^oW$Mfj%fO!@)kM(>k%AJHe#2Pa82%kS0oaBoeZ^P)f;`?$+}(x<6PJUz zk@9zNIb2l5?_h5As$pkaDVS2STp$d98qv|Q$Gq=fwJD?im|zi+M?x$`_l?m_pxHl-GFJDE@Y z0Hwr}FTt-L%*Nz%v=Mg@7PGr;b%AYhP-k5Rf`eMF08NVISkJ$Og`TjZoP;zg7V;pWd% z7CjHd3XV+ns&BTN_8%BQCLRZzkmx??b+YXgP#dSjKXr#KB8%&?o{w6Q&ma7Oi0r#j zy)#+^5g$hZhgufy*)c>}q`ZTYI*XK#kC0k5LqMd$-l zi;~1?>x}HBQ5_(^+$e@PxD8#bRg|e!VC3H=Vr~5bQvt}2bKJ!uiC#I?ZxhYQ(L&ez3$pI+6#PqwT$`Pd%>Y3Ob*?N5#qFAxX z-%5;#t_HPo9~E!SZ|UyJAIgi$;zs=KCq^(o1EtrDH>;G);zZ!QpykYnn)LpVh_z=_)w0R@cL@;H;{~%8DFa z+yHLsM3g5cBy_rs1{$H@K@Lbuz3*>4%jkRg0QH32%NvPx9sK-sW z%WV;(c`?w%`-|mfaO!ddW*+OkGG|Bb=L6`Ve(7OvVAe})Ijr4KlMoeD0Kd4ZC02pu ztWa@wYM?4(LWzlsNr6daePX?$$&cARMn@zn-QL21@Cs~pAnXL4JPNAaJjfLUJEif- zbuV&u%7gx6Br6JC#_vWLISHjAuGyv@f<jh?_0>a2k^0_!#oOb z;M!zwyS9NamO~CAl?BGOFTIg%Pg_$vV;smd%hKC4s;vIZf&dwB(}NzGQ+ z+tE2UTqS;J|+drALuQ(MB%ERyzXyl8?jIi+>T27EgxP9*|2m7dY5nAiL(1iY9G ztq0AZUKnzrCh(;Adqyt9sqQ}63KA_JzDg=LSMLS335bvp4^Pso^1uRmyJdD7c*k3A zn@C+Zd~3UcQ@OL^{KrMW^U~0T7KhCWpy;4XX3EPR-lN`YF&WzEsH$H*lWgN>v*#(o{ zeeGf}!UU`Qss~aZtcl79LmkoE6S1_|$4%MlEf0B_Bpw?(gO7kX-Iu{vcZOFrfdqfJ z{kh$d`Z>%RB|;*a)MSc0rj}GBsXF2L>NYl-;N=vaAatR?QQT!P4s9B2|G8{PjccKA zmgHMTJc3kc!Dvi8pfaEmCygBmYB1gcJ zD5X=c6S~uL^Hm=OXmd8t3qNgRH=)a_lNI>UvY%sRU3%`A6-iMg_cbXp@u=~*xN^xW z)CxPKD*f7Ag1s&!7&V_THCFcC3+vQND*I#VP3)a#9(55(_;7fFzF^?DJeolHf)3pR zJtR7#j;ji;J#2BCGqQ4z%w4yE_c~}Y8(^p!Rt-eShwMRucZ)=}-Rm>9P3Gn8%43F0 zE^$)5*7~q>e$s}!?XJ=+^a4W}+hyp`{a_0IC zkxYnxM`R9wOw!tPzPV{j)TXJN@bD+%y$f|N%XtQ=Ib|A}EDq;m?@R&798v);3a+Rm zQho?5=$jra5oshu(0D(kBJ+^L&`h&&aAm)(Nt#ZOPSrN!YXRN0B#c-H@!3#43P4N| z(gFS$iyxHD@T*hAsi-Bw3=;OEg=#TqK6raPwc~-xj6Q9O7Q&TOa#R~uF?1@JOI`EE zFQ1O0OsaHUo=vI7j&ib;1b9@jA>ND;DM@@jjbkzr7}yQs5ng%-Wn-_kD6Xng6`Z~N z4zGV4qONMS7L9Qj3c1NyECrga!t=Z)+c}L@p=+to>ZnRRyl5r49I!K_*^%sR)|ih8 zP$o(0sm7{fl$2%6B|xZsmx>@!c1>#}7o0(w)@B;JP=d1|(FK|0bIb=Xq$8&1Pz&0z|{$%>16;cLc7vx2VXg-l zXL8w+Vn%0%f`UkuyvM8?PUx)1^|GFDOL2=5o--ih8{j$P+s{d5&wAMvBnz<`2yMh? z4TU|Y8HgNq^|HRO29o4kKM_kZqjKZR)7`G|>QXB7DMpdVKnV+yU2b)2tMQNbld)Sa zFNC>Qqxa@~|4gpW$tu!DT+hf(j3=uyle?X+(DZLPpDuXN4(6c7?aTry(P-85X}I|V ze4I$LQhcg@D~%DK&|*NFOx;-0;2)s-ROyRXA8xEvw;wp^zgaty=2X*7s7SWHNp%ZD z`7s7q)6u)w2eR@VnYYFY=d+EcZBDG%a-;1AP!||yX!kscX#s3iP-28^aRLm3D#St-A<6hC7 zZsX^S1U92&{h=xywRfi&{u^wh942*>ws;dEzRTKW?c`zcAW@pB!6rF1C5uP6!q~!G zYoo?pu1K}Hy~D!AO~B~%#vpHAOd_#3y-vWw$*FlwsWnY_!rY1G%+%b*sT7r=bbUfY z0&A6~lZ%#hZGFS%S8+;SZ7mo(g&{97WssvX6zrRqx>iI*Ky|9W&kr=cR|)^HU50Gr zYkK=euro6xPl_`nO$1HBWl~O3PRM}awLrUi0SZRF$fNJ5@;Qis1e5;mZ;^bZz#>0l zjM(h)DReIey>2;Pt52)YWRFpIejv z_zk_eJvB0}I@WSVKwao#KD-53UtStrGGaI0;($_aOS!GwdB2c&Bi$PKsntOtO0&~; z*6;Z0eK!te$&EX$KzdX5hUZ9Il5{(xH5-NXM^AJKhc5!uv72=@TGii*=?2$uzkrXT zZUF?%c&83lv%^PX8dPc53G&ct`?Sd}Gcfy5>mm-|n`#TrMV>!`${LsUDlts5{s&s& zqdl~0d_$`LIq{ZNx2*i2ew1nx>&)y4)Jt?95a#Pbq(4&7)6qib`@`9v#dhV#(?%jb zlezZLc&Y{`wJO*HUhuTm4Ivu1kFv?0s)3y&%sBdv>E5N&RkO`41jaa*Zgf^Xz7dP0 zAeDOk*3`KJViw(=9=hJcVY~4*v7@#N<<^h{*vk)JyGA%?ecsD+zITT!V-vVYCK%;* zz3KK}eq~L!#+Q5C$q-7Kxiea+3D5i#S;t8gYiN5?{V6K;4!TpyqC-|JpOL+6G>-#v z>bDXxqzIs6Hi3?oUp}u0o;?oBnK59M^a*pnb8p;M%l79r8N%*>WQ{r{;?OPwUdxZ> zD5mQ=b0pjP+m5=X4|IyKm4W{UU;h{L{>za5U$F9vvi=WNsUbIJ+DD7feoFC>jO)88 z20b|43i~t-?(hN>SuYLRK&k({)m4@6pkB<2#L&i|dM@m6`-jYg;Dr`oHN#JnK+76Y zw+(^bE8u%{YW+ly-{RTPEwADFv7Qp0(Q?Kry*rx62u~&hvEYvjkpZy)WROp>b2WO>)M7P| z4_?a?%M753!HfZoaq+BsT=MxjtYt~=;|`@Shv0%&sn`0O*Xo7lb;kUCua2%VLB}&E zsMAe;n^~93wCd&+U2rt%{(oQXe+&2jC3gSgWdFE*TDbs%U=_^GB7Yu(|+;huSr<{#j*dL_fO(ip)VZ$H9_oO>GWR{#4>%U zXXY<0{jY7dFYx_O=9jPk65kBeZ2vOP|Ei0bnVFi6?Q8dcHdy|{kn#Wi{3nB%j`pij zY^?Y!f7SGNiN9<94+#IOUe>?c{CBCp_xk5}e<%E1;Gc~Ds_3t#{FB4R$nb^Lzx4Uv zDPP3=>m0ry_+L$8X2xg!!sCCRJj-9~{hx$?HvZn>pM-xl{$ki)*XCbf{_k&ppNefzL+wMZf=jWHv^8Hs*i#;=enCj)e`M zj`ho^|9#!)Sn0ny`#)FU3*ys$NqpunwoXgS!1h0p`u{UB;lBp^e+K_wNc}%CC;ttp z|1<9Y7f`2VqNZnI_!8^?I@G_&I@On^r)HyL{0g=Ck5Fg*8i-%8|Gx`$7CI(s)~{HB z{|l;rMFvnaG10Q(Q!&tgG5jy7PESk!SBL=}EyGuk04*Ei|Dfvs?H2ksRcB=W()0f} zRj2=oL!kd3_tdiv7x!Nai!7ZF*3t(xVsVWoag0(>RIP=voy)bi!^!4RS7 zS>>to4D7w3S-(XN)%2j6-V?OGVMT#O^Y;~Q9I9|% z_mzCJsOD*TuD%u^ja=Q(gYQyKxKMV_3YB7||H4?E-{ApYJH+(V#fr znl;se+H-Bop(d<&gFS(ru1B;*>=lXo7)`##N4wAlR|W-rSEm2|ES&6bu%y?{HM^cm zop>3=>PN;4!%H+c>xv&p_!7#RxK4bc`vX~s11bZ?Z{9K)A2xVi$7`&>NtGlFdeYW~tGxTJh|UY6kSnbI`>q)sC&npLQ~*<{P&7Un|KQ!= z@}%#w``k*}bf`?EyN@0D0P`j9_yBikjH9r9AH9tz2mV2flQy>t=?(S)AV6jLlQSmr z2qmvckHe0(3*odt#~m#Y*#?_l9=SKnSdfLEyP*d+UxoG4XT}Av;WMK<32WDTJv4Y% z6&zW!cHhmt$@^sNxyLC#;h@)sS!cy;5>~pgK zxt0~;jJDREJ=|w~6!gv_klHszLC=U@gYep_Fh$z)BYdjg;oQ$-0V<;B#RE~pTH`0; zWAL^1r;6{P?)r$gHFqLckO3I9cB=r;q{eW5G2lc`?IRhR`w=|guzH+u?^GopTL)dQ zIEc`QPJ3f*SsIvMm|0fP_fHOdnNp_y= z1IHdwPu3p6pez%rqmK^SLs|N@&pp3PHqr<(1$DQq$d`tH5hXKQy5G3R!Oo)c2tpog zD{SPY+*Nfd42&I1vzvKrzip7uV*?!u^)t+;=FL;KaK$6Pg+M4_^-Jx zzEJ8}k33SJn?PLD7QAUK1UE9r%0#}mF6u}ACmgfb+a*QGov3ml!?H%Ps3$i0stb7I z^cNPvkem<}`S^R+9?jg2^|!^)$s1ZPzz?-TS7bbB&*2LE#9Ux@ijElKNqJIzokupw z#Q3Ws?APpUt$YWTi3P5zG*v9WbZ+(N_l1AveRv@BwO~{ddh)TkW(vLzBR{CD;7tny z4utoaY%BP<;d{eG(dWkwrEJ+RNS^06DU(=!8CdbXYE~7oMv9ys(wya0+w=-Uagd%# zdOW8cJKUEYxyI}11i|-ZCu!RNRUGhbA$J8CX6`t_pinAAiH`GXP^TcZ{KBpz8-xuaL00C7>_o!UY`N` zxvPOZ$G1ls>4r@sl3C>&z#8?P8U7w|4$%)(CcfdDb*Km5S%^92jmnu&m^Kw}*-)sJRR!U{$Sb9@3X-FbW+-lk9#59r1?P*~enk z@hI;Jd9%l~ga<|tN42cfzrogPo{gitk6E+d^?g9OS~Is+i^h*buInJrA~lK`9OW4g zr9|h4HTT?1B|w?@#u$XPKzwhT)!-t-d46yb`K51BfOj~!9p=3HiZqcAb;_U61)H+B zUI=Mb?92(Ma6!Sm;B0ODp zP?#N1WiReOV?o{eji3gXdeb*C&5O!1s^PY=f%jg$j(Yf7J^0Mu9L%T#y`s!f_T#x5 zMZlo0)^bYZFWJfTDu$%=3fH5poVHSToQ0kOuXmJx465)9@?d7s%gj(OA{#vY@-|8w zCimwN@9Q`wT=rHLkl__s1i2tRo$ls@%@}d%pQAJPT|;dJo%!+jl(-R*<75U&ifC7T&JtsZd63@wVn2<^jg$R!^Udou4>cFO$6Sj|E#+y#VeM_(BvW-m z*KqC}G{4}FqeXv4x_Esqv3fmkY9r`ZD~=}f#?++k*wXxAKxBj+N-dsOA)7|v@of!9 zFi3a-QzptjqV}~6q63C~rke2tRPLHl^eCPE8mZFCdU0rR@vwI1`vB5KxIo3@wafR2 zX{;)Oy}_wNXBl?CEN*A%51?Z6`jx_CzXQL{O8Ah=yI`0E5w}y-Jop^=rZ#1?JlTIm8UAvCBh6EiBad5FuP6+ zbTYM2;G|rH>|;MV7!wI^Gu05c`WBU$CHX*|L_)Y=%*(pq&aFgUH zuHMpR{>DU)?)!Fhyz!tb(kHEFhBueah^|SV-A(bf{HA;_aV7nZbfUI>W?MX$bWY96 zXuO!iQHnkEJ*B;go7Wq<=f`+9@-8%uIAn z8aHNFVykqPjkFV3=QNjBm&+UK+e(Ro65CSpg;Y}+DM^PSX6XsZ4lW1d69~Nf6c0_F zrxLA8mRAeUqRoAoTbAp?7wEUgCyMvF_uF^qcbTW97nYZ%7pK?9C-|q$TfA$pgHLu@ z1PCzp@1r1nzyOe4YEkrb^i}o3B(?czEE%b?RNqubrgJm)g}YMn3YTU>zKPmY(erhO z00*7`GeAR3a5D(p*Hi=*1Zg8Wwy=cDIuw6a>@j!+?%B@)O)n5%FW}9VP zYOgxyh;p@#2Irxx5e;V7uY%{{t1As&C0{HW3(XGkMebAvaW9G<71$; zZjSZ=3;6tGrL+#-i*A2>nc*>O{arDxE^MltqwekQW^O7&^PBYxvD{<$uKj>X{diGY z7F^~eTZUulZAtDg(4rfe2KEHI(&v)FB6%6&jM@xBQZLr$^$}>&sjN5odG_arkq**N zMYiR$f53fC1lDG91mnW zqgIB~>Qh&Sf!@M&2G!(6&w*NDhtBjKy8=-9cJ;b1!xwEKaYCPN8$Cf*cH31WZUi*x zApe2w(Y6rjU(|H0D?&aBnzz$H-L3#TY)iZQO;O<}hLgfw~)vJsF zva8K0e%VnCM#2j!=`Ti$S=jACHaE8gEGcr_iHGYSBsZz$*FXz`>m!~F`RRYDgRUa| zAx@^_j|^#=omh@fG4k^UWu7m2SWsvlfj}nCXt++FX*IV&P@rB=aq?zg#xQ9=kk>{#n}MpPAJs*5jN8+JxQqDz9N z8&G!z8UkRxVP6qABg&*CvWo7C>Cne>iTnB&fF1t{1VTunn06t!Xrdj$5u$*%jCRPUQE=CfGBB%s9r44 z;T9(w*w8kNI~eH}QF@@vmYr>gItkB?<~{OriaLSPNYx(84M&SGy%=l>u>+B0>J~Rx zysS9pzCtd&q8!tLER~74gMtj~yr|T?B-On5ZgB)-o+Fk_t*G5ewsa~lvI&yIjD9Nr z`n&+TJdG4Fl_>Ll40fvUJ0tI$31BtZa9WZ*i~DB^F?2`2VXcZeT-p$>LD(|V!*{j8 z-?c{8hwd>Qz0B28<_v4Sj@24xun#1!92~=Nd6gf2IC@z%Q!D6aoUe%*`bo=CcOoyy z?Jv>;B9CpH2v|$dpTgdVKj^iFw6;&5!8&8CdNm9f;IFCN zDK5s>p5Z#9&^3Hez0^VUG7@lyuVR-dVgkgO>Ayo-uS&Cctv$h!Qa9^ zA$rDt`gslW?BLz%y|I3NcPg2Bg7l2d+R3_|dxPN_>)O(~sd{4b%=DV*+D3nBddqs_ z_~7aq_S#B%GWY;|hJ21!U1<#~?O}I~dhK^zZ$2|=f~aXzR1mgQgE;o8IkT^Vt!;mM z1KsdPDZy8TdMY ztdFG-7c?w%15&jW_aw=kl2FTcCe56jaJ}LVl{I*HRpL(Fl{K|{`<&MvO0xz1Z2v)| z)sKEl+zj9;JR6a@E}AKFLE}n8btm>pt~xM#QvFksWkjVy@xXv=B#4>)!AFCg|(R{tpV`y@z)al&QW9o<3 zTHEEsyA{sU>)K_z+tsw&CEmx@u-wy75T1wo*%N7&r{m6E$&^f1%Ra}(4UW}wWRv@l zTIVj+=5^ww^O#fnKF5bqr(U(ktF}(t8q{jzh*icCJqu}@&~3S)RGPjaqh&nqRnfD2 z2d=or)NX;$7q}OJaZ2TKdke728 zbYz2aqWf5Epl#{_;@<3`tL8b`%v;&~A)V4fIp4KN>3k7x(P43Le_m+F&y$awQdz7L z39`f8uHyh}1x1tpA_vAvu;jNLiWWZ}WDp()LvwCqTv2MMUWIW=GCB!*4TQ&0N75CL z5W-88JZt(jeBI4EhTKa71g3os-!M9tWQ(25Zy$2)~IamR7bD;rGQohAAmkUjOj zQ4tc-qY+aVt-M!qqO@}x2o>jEAi&YmxSU-&^}dGWo9X|a0zs=CL-oe|7B)_IfGRi*f6O3L{6+s!HBJ-FDW`# z(?0q~I6v%#U?hrLcY(%uZOj9`tBcOSm{RB=%?k(-#da3oG+7G|&cHB`p58yWn0+U8 z2V;%=I#DW3vZ9cCQ?L#kvKBrxrb$3zO`qVkL$jtEy425_MVLRKLn7A({!d0ds;^2) z0f8#D?Zd+1@@rW7=u3Lw$v$`d-uYSje%mD(jA zab|Am4Te?QeH#JVgw=X9I&{Dedtx;a7W52J;a;vRoFy40A@K{s<1UX1;>q^HPbTq# zSbok7frFd1ye0XHrB}S)rIG4r4JbDv4a>9w7W7u(O&tWDeOGsiFyP}L^=e&wii)d0 zZe-_kaU3HM|sE`aM?`0)Z7C% zEgiTzzaRq4ALj}FJAN2GO1^U8VR5z?dLTNG8gv_#5lH%>#UVT21yM9I{k3Jc+JvKN zV|(iqq;=+MMsc^h$XRKPyn$80u!i9X5|&ehf@7)N`ID(i6a};4jU{eejmJC)DCBGL zBOqEtU`yq`GX#EFk|BBD@m4CT=nhw!UGIx;fTP4#z$?azX|;aS(kw*B_utlYct-=q za2?GP#v_{RA8;?k=E7$}v@gV-UD;aQFs^7!-Tq7b{igkz{fPa<{*qHWk1(Dxb92fk zcEkKAgn{_K0bg%|2^8+^+Eb6ob7V#p{h<-#GkeQYGW$(3`pZHxyQd*tfrB1q7Te#> zG{o^4DA255^?*d=DYYY}Dhu%}-#cpBQau|c;*K@vI@#Va_#r|wJi;TqU~mQLa-Mpz+F6C-+{cq zu|}5!9k)SwWOY_dKOSwOx+0iB%?jL|SCCs8SM|Afvm53c=Bp&r%&;^SsI134ptYi} z><*ftrd0!`!3; zK~u(62EvG;+=-O8gQWb%8rV-v>qhXkzUr2Q36a>!EWtoZ>b|P^*E*ZvYt38biP0;; zmAy`C^RQR6-sA#Sq-Sbe_T*hKzm^ZQl%8+@|TIqr$!A;ZvPyQLZ z1e@gAMS=XF|I zh|#{~y8a$glp0rYp_g2%Q}n90ly~8$vS2E?f?P}@>gX1q+ooQzo(4xQWTZP-T5%!r zedXA5d0~KSvOqsP*zk*>>+!|4$!JHvI+|C-omx6s(>J!vX&kkVX&_K7$j`4FTSf{D zhl4R#S4U`w7j>c(^nAE1=-XF^n6{J(E=VkhCsdf`-jx0+)mMxiEo>~EFwIk~S}a{K zRhd{&fHhTf*fXd7_2*mTh#_A;2gvtkB<5c>sE?rm8URx$Ctk8hhrIO{NBnjE*Fe6e zHm;m9503@Guph;Hr@Z`&meUnlL9(agmJ6k#S+bgjP0^)Kld)9(l+*+B2pL*Z){kW* zh%t<$V$G#(-UwsqLSnwmsddHmDCyU~pfds%9TVRsGd}j@aoY?(u5C6Dq~WM87n_Lc zxmy_HseohlW-W6_{b|~ur>2s^*w1_O7-MH+HOg_GONygJjn3H<5*^D(K;mt;2zYea z;*;?bY)??Gf^E0tiUh|hLKAQGJSdY$AQI<{yQ|`F%^1bmqy<~3;^soxEyN^#jk0pm zs*p3RY`1id6YwNnzb4RMCs>P;+mUt3$3r&!3T;+bCK@O}Aqd%$4{a9BmDlga6ROn} z!nGi~DV{|Jc*WZYjbThKCefu2(5HMJ8wm!DOV(Cc99Q@3V9Q=mVEK^2Qzf&nVwSeA z?XJ+5Gdl7@#BXw$*Jlfe_=|f0*aHHe*wz!&lOrLBy21t`qQsgbf3c$Xq&@B=_uZZ`JJ|a? z@pBP!wmfBg=JvMC3GZCz4>&rslq;caP?sd1FXWagqTqL8NL_|OUJDi+#DWT~2aYM6 z>4*9_eUS;7iR6d=d3?GiN*nM8JWfJ|tU!$?v@lz1ylf-lB6b;2~t}m$$?uD)&?Y`WRpBCg?77=93 zc{He*4W1fwP|X>y^LIx9P8;FQ08H8-5C4V(Zy@;`E|k9S%7U#Th27e$eMWq^_Cans zgeBqef@mVRQSIModZ`~Q^Wp5HA^NV|Sa1C9O!iP12e8JpiPH(9%p7?U+)Damr@D!h z2S!;#!b8>j5S?|fB7FqlStTE4B+Wg(y9=c65iBCAxo{(eVt30^llje5LpjDJ=0o3$ z`vJHJg9vb6lOroC{VS@GyB4m%zUE>-w%rRDHW2kDnMOUV!{Nq%Rc^L4Y?h_PYMIBl zp)&%0gWd4S>lfL`M<(isrvfbiC&crERps@m^ex@adoC{VcUhJ;j$tyY=nTX zP&h%XkU%Eq1xVt$qIdTwR%76DWM*hq;@(g{a!ko4U#KzQ!?Z!(#LM%Uj-@V$@Fjws|T#dFbE7Q%kvciNR8qY_k;}2s$#i@3uK0gvFm(V3W|T%P?C((jJo8?v z^qS|JtW7BoKKCAA3?khM*$0U~B%n>d&n7Fe$4QaFBUv*^Wz?{gSc}cH#*BK9nQb~Y zbxmkA;1Xq-MIQ|9Qb5I98642^M89gj#01$yCMA(h{g>|(*Si(XPGbTTRqcB0ND`Nh-*FcyD0N> zgwXWqd5C=cP^Yg7ZKNj@x?I#@pVo*qHx!xxoC`#!Pm(MJmc-*IW^D#`zMeB`1YphQ zMyY?3A&#+^=*KvTfq8;^h$q;uEr_LFlSV`lFaOR_*Q+skJbEmqEuCa&)R+|OE`?qm zqCD>ypshornO3g;+7|R^iUTOoN!VzxZ`1*{#2EIx7FVlW7xLA zbqF)Dc?8xks2vJL@+V^o>q9+-UuXAODrs!jWYVu>-a4T&ax(7N6|QBnX;6gGgd=n8 zS@@ybkS>Hj@31Ftz|XcF8{utxxsDS>jhGP+-N0N$G$Wfs(1b-Ud^D@UH5J5Y6%sH{ zw=zG0I)|>qJK(f-elqgID8cQ00!!ybLs|Ah#Ww&gZO`M$>U9=^h)c4)fHwiSzfU`X z(Q?9Sj|-=NfD{^jGyF4AzVl_yF2ufZB!+wfb*qFl14knOi*f>OvHKq`94~F}l5tAu zl0^|Rp>_x9iMv5`M>&8jeVeVx7-Ss^M-sF^k?Q>>l?7t#h=PBL&1)w)Ht1-3@ojhk zxSn5$hz4T1m4lZ%vf*-HOboJ;6ojecSSQ$^(w!RMorI3!yWQ1cEfCz|L^ zG3iv5Rhpi2FUS_r07B7SRu4-;o9IBR8i+y%#x~_L*Rqgp*T39;?XUX?>g_OTxkF6e zZp>z07wk4yESWekjjegzMCf)QY*~iDk#AJg zTh;bfb$@^O=iBOpf4a&8UlgC4;YOBc8cm#jpxY%u7ve5F>4|A%;%_Ri_lRnavz8Js z55VfMB$FxwP?>NzZ(AxN9WZjap;Jn@C4kAY{=?IIRv6Wz7J;y{4Sc1z9c}Mw zVDXSd6z@d(aPL2=3x((~K9IB^i(9khGZh*>dCl^U`{~cRKA0<1l?P!5tzrGT6IL^v zMFx@UBxSQ|4Wnx=&Cc46o)nHx)&su0QT{njx?SgDOgIulyHG5UB$D~anCE=1+DY8m zA+v&%(cPrfwuTRkSriOjuqs^CjmB`OtRsni|3YW_Si>j*o)VL;PHNMTnD=g+{RqOyj%dIv2`pY%9H_${BYGiv@mnb znbyY1(xCcDKEk85*{IsS-2qn)Q7p45A7&bo`>**2K%Bs=0U-S_NK0Ua~X}M(eOLwuZzCu&ya3L=}-G zyj3h1aCDY>;S3ZSCju1b^2a@jd=Y6?<|C7`LzXmD!B(3D`) z_{{>>vH-Y&ZIXD!$E4RbqxDb+KF)_)|4;Q=;L}UDQZ3_K5UIF z!<6X>+8W;V6;#>;DAQaX$^K&`W@qP9g0xe z8|Gke7_FgVK&F+?uPwW1O|GaU=Jx`0w}h)b6^*32NMRjNbuEXAqRsLF%5#f0b_gu zr6(%oLW(IY27~LZ@z7XAeoVndCm^7>CiF}XsgYxgwrak~FL)YSTWr+NXYSp|W9SoT zis6^9rs70eB9#w9rV--E1_p>h!i{~&&M5h)#rU+tx`G40*|i()TZ1}js)Sge`eja4 zbo$To&y}k>xxV{9-4T*S3!>?#u)+=H^2c=LiT6Jljc^k-0wLq+;Fs2_{Yivk=gH~g z%8vW9!RBAA{C1`a9FcJla-mhfkW7xQN9Z%6XJB2y@VhC$Z>#P*_W~xDS}Cb3B1@Wr zh=P4!sl1S)Omd_bl19e&?;mzMd*WErY^6@*sj?l+1j?e{nxC*ryY00V$9JZJfH&*;1;v@N?^mtfuc(=8zG|>5D zKZWf+_$0VDWgq8Pr-q{_IEq?r^VgTyUV>(AtSFl_pcTh}n!9bN#3*zQ>nZ{ExW011SXZlrs(bHqGfGOER*^Xqy>H;+| z;6L7i--SrU{^ArgmQ*oZ>j{OFeG4z+LP7S{0b&FKC98Q>Npw%AmFwqA_Qeik;o#PU zC~n3oiE^&HRfXJ#f0`yTJZ#9Dxw6Gle%#oL$vdFLRDM3_y+Yn=OsV*KJ@8C36(|#X z?F7mWXF6xK(tAZ>;&fbz=+qG(*I8scpLl~-9!2b_A61ErwNx&5tQK|p=kNoKe^ErZ zGp2CoA`xbx=SIigoZi&EQ3vCs+hZhwmcxa-JeIi{9ky-Rv5zA#&zk!AP(N%Y)m%*4 z5YrYQ5jeJcp^KsY!x%4`RuwDhE!)Az0aeXJeP2~XGy9ZsDBMQdPq1zxeR!a;r9qAx zkweNg&%F*3LMBvVrY;|)k`7$biMLw-Ld>267?(+oG#};%rQ-vI0T(qP@|f%V5Bad( z+VSz)TB!Lvv3>$OEmJJpq142<37k@I+N62QI;h=|rpqp>x5ekMZJEqfN%jjKq15{I zw7Sm_rA{o)kJFYZFENua2_~ypHy%!z7b~_`%{ith9`w$;<LTXEk?q&bYYbK8bSScGd-Invf;c_UI649YZirLnI+l?4!nQVLmy* zw~e#ka+p0D{jZeI9(~y&;+JWEZheN}rXZ~do})UJbBl+xE{|Kt-#Y8pGXK0^T*ZLU z1oc(HOl-`T3e7apc)XhlherDz%A)skXtOsn7$+5=G^iE;4-N=6q@v&yE9y?D8=6&? z;+kwd%h)r;R$RzmfKip5y0U0@+RH#Ha#JTWRKcSBjufzw6gZa-19 zy|C64vlCYa>B3pP;`TTRY4;#=+n6bGq!*=bDFV$B+gcY#a!Kgq;hHYF!Cm??a)3s*d%K|KCsz8D5KL7cjb z5dmaD2wfpK6=y70-36xKuTrH&C)cNm%E2&EF&j`yrnIsEJb)?$IYu)k&ls+sI~Ppa zj2#AFMRQiP^l(`fSFr9tGsCyUK4;;;RejyTTA#xD6HMj;;YN`7%u?-i%@#E^?ni-C znutl>pxnu=h>5LuJUcQf);N8#9oMY9Nu7VKx7#Ta=dHiD5g$V6HjFyh$VpFs!ScG@ zn{$aS*7WPTY6yp!ipfB}Z)@#~1CnI}!aVla#=CA)Z%ru9PmrOXeA2QqWH-{l4OgC} z$76hXp|#1t!%LVMRb#oX=oX`9TAf**!4vQ8kkFwlW|GU9DUy(r-%RD#jAQ>l#_l;f zvZh@a{$yfJY}@Ww6JugKnb@{%Ol;fg*tYG7ZTrjpoO9NDo^{Up-uJ&=dso%2+Sl&g zYgJWWzckr}sfDg`6KpS4FI5za@%LczL1br&Z1WWB3=_H2l7dNgAoD}s24?Hx34ZVo z(Q}yAI6752)oKHVazh$ctf&bx^LWy-!Fq8Ai70`F&X`gJ=sc2K^sekY7R*RdmJ(Lr zx`tWd;)Ww0rg%hU9Hi_doxeth=qdDx$x)~^JDx9o%nt`s)>Z;+9y1Rql!<;VX~_12 z*wI4CFo^}n4Yr6speoDsq9?pte3iE!q(@Qj z+j{LFFXHeeBttWx+*MB)8-niR7J+Vpkv?ns^WpQ$oQJ!ao1O+J)%J@ksmV6ulZyV_ zY5L}8Jn`z`_%KQdRD%j8QrZ*v&5 z4`z^Js?=1g(o4$cCAe6zr9@ip)L4u;^NCYSO_c@`ArkAt^YPsE+tm{Ugxe?Eweij} zrfU2$GjA7q&P*A(braKaa6?_l^%%LaQA<0InA8u)RI#i=Igq;QSbhYI%;+&goD)B- zp?gHC_Igs;@X|yIA(0DahV37_U~at2SYfWP#83TPR9aNp`QU-0dpw*Savc@-$LhHe zMyXA;Y9tj?Kt43Kp5t_sctabK*L)(IyiG(S75U+9>JBia6;&Q=@yxDP~9rrdO5$E{X6yt^X5r|K`JHqnX z$B@PKwYIRU&3t6Dt=lIiPA{=4>L?pHB{Yw+?bQ4MVcXlSH-?mC$F@4LKH*&aEQ7Oj ztSATGvmJR6yXea`6EA+Hn>UA8j|9EMU4Uk!I&gK$ZZhCH$Rj5P!zD9++Mw6mQ=Jn< zaA`NSTgV@X5PTR9`N*TW*fDT+l|O5p@6LI#Q%$bgq6Z2ifu-K;mTOE+`wTw>qh-YPwag?EdgQYqTfp5IbncZ zzf2Uwga_aE8Y-4j9tnhsj%=X6go{q3Py()hLjZ*rD-jFOhx4;xmJHA-kLxb1tvJ9Q zWpx(qS+kddKusZcr+f_;PM)J5N8qBvQ=rv^1{>Qmufn8W3)M3MTX3s;AU*2ykb3y} zA>%rAfJntfxp%nPTD=st<*R?I{`4!((fsZ&w-}FN5+}oF5BFni4+S4dpGy~(_jP~t zRjP=(k!<{1rSPlpQ#>R351&7qY#^#8bgM`_*g2?$aTfyd$|j_XDpdj(u_xEo)|Jto zEfZ-e5qCF@^LJVip%lf3X$JGC`6-{rje9=<1Y_w2N_&tL2HiI8eNG|8)z&o@?G|yy zt=HajxaXd8y71L0WzratTPfqB9h(MBE41kG0|iTJ@g?j`Iy9Pe#sNC^=78(y{y@|a z(Du@BVeU-wc==3ZMn1Y^w^{Nc<)T`kl8IEEm(|ky_Q$pYX@&z#m*(k7-2~QQ zmhPo3pBU?k-x+oVR)M$P8=%3EQV8kA1=0^wM#`;X@XAFZ2b^a0Z!tJY=}BvfZ)Tza zGB-P=6cUwjocYYx#fAFVT!#|U$vIIimy3UbYZBCUA@gxK@C$ZZq_kAwCKXFr-+(;u$>zf zfP&lkLgE(!n8rbPo5}TcZGHB}LFnNRBi6t?aELBr%$tiol*LCn{B<@@*-NH;{O9A5 zL!WG#b2ck?G9X(Q>vKiHjb97vR6ELt8`*Qe=VhKB9=QuO_Pzt$#JGode;|8EjGIr9 zUC&Oc%fsjh)>6rNbg5#rROh8;hL8hrv@OvC(BTgeRz&FW^VttUnl-b10;NOwChd#} z5P=lDjYoR2qhIBII9eito@Y_Tt@_ZQ-ome=_DPvn$WcArY|C5?Lm?3?onG}$TcAgG z;nrK-C?Fz-!i>op?>VnUkR|tPQ`okhBsc=4%nBPX2?z2u(GAEu&5#C+QFFFG~-Luvh!L zi-Mr!O9K3==cjv*^tdS*hlu$jyqbiyzAKvcO!1W;ChcMuH~u3SmUY z!g3ORp!EjV0W({#ekE!$QvW3erOx+@8l5|VR=STDA$!}#7xP7i+iOlow@aQ1 zoOUmcn`+nZDBqM6gEGls12VCNx@VWMZ4ZQKeswk^*3&rFIyN}lC3XM&73P#P*Dl!6 zD~u%usii_x4dm+3F+t(p}t znAd*&RBej`nwI`-{srOHIS(ZiFL;!RY3_*sBPWe+8MboPABgXCRJ}w-KF{fa9(8o* z6tG|QxD>R4{s?~Rr(+g#+VO}3t+C@Kdg}-3O2r{Z(E8zTVbI%98%hQ=&s>Ntp8>x$ z&>tj#MdwDsWHL(UqXgHavEY!P)zfkW*{u-Yv7e5@Xu^edV|)9{Z8D3jX%V)YP4IBj zlIH!c^0c|scUf4D+ju3Q6xQ2C|4JW~S@jFf=kwP8{7l(Ve+=DS)Unj9Q-`H>$9-tV zuV>Jd6h*u}nyry7+Cut9&BDn-+Cm7&;FpC?#v=A%t-C|zj($aS#zycwVtyK?a>Nk! zrM_>3f|OQKtQi;f-zeoMN322L1~i*|>wFZt&8mx`vH1X}fXMfX{9!0Q6nA^S>pAsZ`3^J^#LJVvChW_sw>@I2ympQw}x z+fZsG1SnHFJi3Ke#*af$c&?(Dxsy=KB0Fp=tqe$jDFPr88xEhh{`ejaEt5%d7Y?EW ze_q{LUySPJD8Rj}np{Zd)->7t(JZwwP3aO7dG7cmM`aTqfRP@Pl8OZj-heM&v-0Q> zBxCZ>PHb2{?#8#RzcRoKN4cZy9N;+-Z;eQX_&q7xh8g;pN{*m8OStgx=e#`my?4M4 z;ZZ*o1yMHCVRveR4VnvQ{LOA&psE=CnbfFiG5P-P$=TWo@?oEsKobrB4?O5#M$BD2 zaJxmL65zyU?b+i%a4LKUIg-WQwD#_DPjvA7R;ZM=zgT%DbV#+s1iS&f&bt2e5k0nm|* zR{Vx8MeVty0jl5Ea{)DkJi%OvUe!+16YAu-#uAfYiD=1kWL=|VU3C*yO8hLXAM@+nuw?hB|#|U(V{UFqJQjx_la?fSLICLX9+5hY;FYqe>&jqAY%DMiRt# z%9C%@%O!4-Jo+s+6z9k`zj&A$@8Zt4N3y*{(SwF)D~C2xCYgF)ZNHBEf|8MGb^RMLTm7 zc^OKjMVBI(h13Vt`cfiq4wQpiqErX?6Q9C+J`{RiM7dFPY#a%<#q$x)v)L6~!YQmn z)>2nUN!YUw*^ftUY?yy)r+s-T1Ke3xL4(!h zL?c=NJyR8%XymN3Uv+t6rE)3Kn~B zxe-Qe*Wpye`={3SK25+*g`1p8YjuRi32;<>T{`os@yK z^G4SYhHk?|B-xZ+yKcb>y{D?$^hd{3PmeJ>3enh%yl0jR*&m#jye~&@_XEaJBu+77 zc8`A00rf*MSnpbdZYrOXI0ttbD|5yG(xT^vN4AbiV6S>@j;8Gq-b9gZRjPxg#f=+D za*KRS#oE~AnbR6TtEZ*ef2T7HD{kA>mL&GB2vAz6L@E(<@h81?qqGWF6>>@VQK4y_ zyo#`su-3!#v7gx~HMd_0-A8pt?VdMyrpROWQu=GUOv8No!fw6%J^lmfob^3A5KD1eO)Uvt*NcD1VwgtBM^|`$f z=7s`~7_t|}SA|6^ciuQz*zrXSHrc@aiwU%zD_NT}EHQ^yem1Aw2*o1n1m~sqx`L%R zq4>V!-l1%wvBw8p+oz6v8v;>y+Gs634EYn3ZQGg%QQ7sXXgfSSuiY1}(cMuz*_q;H z>OR;54CH%rA%ACDyuLw6#XA)E#NgyRZUA=!_uY%@yVkpE0+wP(3VDigN=g{i@AFly z76y$UD8H#H{OWD7nt0MeJuaPoe4Q@QL$4q z<)DEwuJxxIN!H;-`f!?Uim~R}-^^e%n0~uw7{_l*8MkY94kSf67Zo(I)=39kWHs?| zb$&8Xs%OUjGSQzxh=F7`rDA`AZwZ!xC@rikv>-e$J~uc$#TE(qMFQxv+^_Pwh$%F% z%cjVP;kIepd!pZNClenSvjWpIb5zSl)j4M=B|0WV~oWTk9h919V$!AVv{A*glx)f+T=CugyC?hFjDGLpU z-Mm*|QWli7wv(!H*Ak9Z*zBj`D1LhS?h~3MXKwQ=cfMpnJ~%f8Y0Zg&6o_x9H&{z$ zz@Meu$z!3JX|kJ8%24HD`?A_HMPK3O&H7upV@+|lVRHWEBbR-P@A?B|Pn~^!5H*38 z`{vBiwi!puc{Mvx!YJ*d70EIKAiyG6%odQEV%B=ka-4QvysAT0>1>R}Pn|4_BL z*aPjQB~-bK64F{UHg~RitXzz>6E^bK_ABAL5PXD7YEs%#O6kgYyp{$URL|TWBRfj1 z=$yJo^H~;qAqahO#|X{mkDpaJJ%+!rHu~|h4P+F-=a}uQ4b-JDSUl!4PtB@^$J;ex zOEV%)CaL^`*EB^0&N4$&0_2!Vb(Zs*K&+^%?bV2N_Pr2NHyYYT5ZXH7b%`4lheBiL zb*8N|SuB5VK(A_Uxp(IybO=IlWNpc;o5J1|$UiwgxzwB7Ud$M-tVieTBDWdhTFWNR ziv9Jq>OoBpPC>3iPNM)rAt&c4^jWL%%q+ zkz>+ux~dULlaziK&-+ItpG?AQ-KHT2 z^?dj%J$@p6QW%Ma-?9%wo=X+aL|rOfohVtk-7hAK3JJYfX$dgxxX4r%338Z4(psTI z;P?bevGcr8Zm{!p=hWgt`7W}F^f@0#sgrZIj97GyGAzyDzB9h5m*`r{+eJ_ViF`0u zphvw=9$ev12(2GUr;c%zVUsqeRWx-YX?uZtuFHs`{o9JARw4K?9|}_CD-n_EJlG;&=Z?$gM)h z2ojw})j7H~oR%G>aBvont6_#SRkE{ye8Pg$Z9R{XeP*+WuY@&L?Yu+T`S;wJVtD;$ zP+X&Ui9aryr-a}BoXD(MaOraygr_vGDVd%GsW1`Mk9g_i{yD0I32G;{_EZW@2T>h} zus=+22?nDJ-7v1Q4GrlU-K4T6nk_Nrn%{DlG_@=3j6(n^;&TxjgFG<+F#k2XY9eyvBY(PdAi+VA^RxiI)Y zlI~||7b<1~0j~fPKr$=!bTcaoYhv0$Z9#onsOah9U!B681T(`2C=m6y78R9eGC6NP zw*JTKdacbN1_zk+z8L+;+pLopEE#IK&GNp~*q8V!wVP#+UmOw7YfUzXTX>ub28VuW zD81kRVy@WiJX5>zrldWHanh+uJA7=%a&^n@AcYCH?JH z+Ho_YsF)9tlWzgq(-z6;=D&b*%c$qj4~Z4-66AcGIXC6E`-^FULJUGw42i##3e!TK z-)AK!)^0-HnmuJcRPM{)+E%R>%*h5h5{(i?^9MskD<8@Y(_f_- zZ+^aY!I)?M0BXGfQxeKVk?-6Rc=_z8(o8Z5N(j72e(H8$R7v=6UIjIc4X4!Qs1~dP zDoVO6Fu51E(GP|(T&((O!o)LEMd3=uPrf^WC|1oRqXuY5$M1PYaeXb(qf<2Od8+eZ zTu~ZEus=}e-K%W6p$t(A{5X+SOW3w zzg0FMY;J<54Hx%3aDi&XIJ{*Xzz(^^I7+pi(5RE8kF4C#TIjQ0QzQWwX)Etb1 z=L_V;ZYZcd{9_^-7lg+M#)$Ym&=NhKcH5pqr|yZ**T}A!ZCfn^lGe|Yb}~eQP@Uq8 zk73JSb@sKq)dav7ch=dI)OL7zQr30CfBM;0W-_>v{|zehxE)+{NNHq$Y%6B_56^{U z%pP@bcn9IB!qK=)DUBXNy4PL7D;k^MzDJ5!diFB&&Clbg>dz}5ytLZu@Z2sgPwlM* zc9#^It|oiMOQ)%GZY~`W{BWz2r_If4$l;#zHC3-q?p*@R&Ik5xItF)^y~dOZ>zf2m zt&jdY%}0zPh7A>#h86YhZYHeDSlJiZXNq^@#77Pu9q>rwMNqGpZ}>N!6C&=a;(Z$1^Qn9I&M?klZnm=Fp=vR6i)2~)Hw%lDcwlRm|&z|ReC?S1)5ES*4`EF#3BiC&uDtjbewD=!yq zVhd9?ehuaN`LT;r=24F>Fr^m8RbLF`ZcZ+yHw?rq%!#UNu{D~YM%1lS`9hAX-ReW+ z#@?vKCDte3D8EMILf*Z-{N3)C)=~Vfr8D(q~xhB>LQ?S(=od!3|3j2Ihz7gQ+Fh$Ia)H>Kd1!%dY$D{!|l$i_Z7I zxoge@q}EzaQ^Wn$Lt8KJbj@IETb{cUy6rE6r_=WrVI^){S0aAmN9WTqq;5r?E$v!t zQ;Lt}1xl5%#!?pardiJWK%#rT#LFuuRPFt?GE=-Rfz_imHDJ&dwa9+6jl~S~J>Hbq zM-0ps73Vppw!LJgQU}YplfAY;dW(>bWv8SLgOB;^I}m#1!2YMBG>S$!2C~K;R6;gt zEp2fB#18!ot#M6T5ReiLN}IWmE+_$Am%4aiNjl(}s*>BnaPvDtuAg+`Si(@*h6ncr zLHq7r(sqnXMORbLegE(S0e`9lK}x8GPV;G+#YNWo?Nfh?jrInU_ayMKQgzeyGHPiH z(winwC@%Mws73yYJKa4Z$WIZu*{IwKp zLiv);m?6*9lR;~RgO~W&jJmxm(G(R}*&kttTn%--Mph}Bf)OD*Gd-OgmKv%WhAZdp6SxG3 zIKLRsgW2v55M8C;>Fwjx7Mq>#jc9k$iSv>NcV<~0jgzhgb?^!sTvrEF#?7U3Ez*&EUer*^6SS!y zBWi`eR1BLe>ylO$h)I-`>?ZV#B`b~#($RJq<* z!avL4FMiEZa<>t zWVSJj2l}yqquMk@nRS3Zrg?C=-u$s`;Umae>3OpyUb!zY-0s2~A>7JY69+LzA$SY2 zjOI)Gvo2yeu)R+15Vc1+%~GZXF7bDp0@1cav@R*_>NvbzTk|Uii$CW9k~P1JKlg}w z%i%Ek-cUWl-RWSjPOAU-A>jVI$uW!V3EXhUdNt_r#QuQ=GNo*SvM*7pk#_pE3yso&}Gr8IX)q~d`2A&9*<9eh} z#ITcx*-Bj9Kyy@Ci%gv4TkGn;4ILcJcGzJ7x=om_m>^6(&)yI(8ram8ImG;{2tj70 zC-Lo%-Vs3uqCH>EkZ;eiIWQDNn#`~?pe^$kQf#f?rn<9iCKq_WE0zk@pVe@mJ1WOt zqz7a6(V`J?Us4MKdP^!h)O917@RgL6>x|3@6ysM=*(TPWH}7LyMjQLsRCm7KUM*)= zTev#dmzKZcUxyI9JQm|DS}!^Rz=(2t?>cDaOuAu_bbTt|XP22(5-)Fu4>qro)vuI190r=#;_*=E>&C$BqhtUDnBO|9<@w!>aJ5e?Tp9Iq$riLwVWmi$&FTIb|qd+_C zKUIQ`Tp-zpWNm98)!~AMQSSW|cXqd}Y!M|@ZDDJ^uuIia;t!fVuC%`f@V3FfTba3) zuTVoR9)h`}qh$Z?oWucT=RhcVURog3l;lrX+xbsz!H%+%5 zQV8O*JH^9_jT8kWMuvu~JN%H3qdxl)vvKiPTgD`F*dvEiyOO;i6zTw# z<jbzLvbZ;kqlHD}m1(NrO1&~dG&kJ*w_$az^oTvs|z5F>8%NPLKVyrOcnY*w#Bq|a&6rOmm2LPj=^ zn{R&goIuZA)&8sIvc@-lx07$hCxzj)waM0dTyFEF3hQszBk_9g47PUYmB2mnkoCXZ+`>rsJl;0R9)% zX>Y6;dSAp8Lrzwolg2oZZ(+lq8fzl=Ux47V8EAR@yqZTZgh~`^VbE z6F$yXiI#=VP-VULq3gFxq{rpx*xlJF%WCe;1b#pV#ar(O&9kDse1%GUr>70t^7qa; zs=0fQ;yL<5^h=|e6H*y@Z}<*!#1`Tv&(AcxHJTgMt4e0g9t>`@3flTD%EYkA8s%mt zQhyRo^t!6L3w?$(Iw(%1xu}8)6e#V3)l#t#6Q`~3RU#isd>#jR124g?_(Fb1ul1G< zo5$-D=tsIUQ&&B;A|AU`Q@=xeH2%;PtrdY6GV4x*L3Z$Q@KV-`t%D0_vGGy{hS%df zm!J~uJ}rhp`nR#1+wK zPD+dKeuh?Su+Io}EZv1!NcntwM%&p4$rxCc5UXwCP&-#YU8hJBd_ZJWs+`>?8&|SV8i@v;?1p17lskwco1Q zvgTE667!UBez|w3J>}+2@v(S&`C#`j!#(IDx& zWy?HGdD0PMRy4?r5;c1Ar~wA__?=zDO{8M(fO4^ zbhNO$q#Y%z+&|Q;5}9#kiGYy@b1k=$8R-5vw~&bW%)dM^pf?;|;D`m1O?$8V^~In! zX5w!!?>ATwMEvm9b;Jj^*=zDxIBkqcmoB~{U-|K~#b)R?0htZ%eb_`foPD4W$v_v^ltR8;^k z;`>l)pZBG?ml|c@c<5A?4Ct^$P#hAC(z+zBl(`W7Diin z7M=1As-%*zHpdk|M^2(=F+*4BaEY!6xfOJE1Fi5belzd)$YdsP?e5kahzR&SqHD_( zzXtGW+z}AX;E>{Tz;r00ug?ZovK)O#j89mn>c~ky6ji4m0q&zd0i2ECpC8^RHXS~W zZ!dVg2tU1h!Yqbm#<)w+&!H;ScKR2>=N2T7!fw}aE74#w+Ld*b6g6oQRZ6U{T z3Qt%>!pnP+k0{GZ08VOIrIjZoo<)t$D6JSbrKTIxdd@C~W-mS&9fg*bDl<#PQ%49h zj%gdJh=uPkvwUsfd9K4ohA)yFUT*>c>Kb5J2!2rP&BPVR5sVhc8KVsjl8_jy2^Jp6 zNG$w5ipLq7FLExvCEI8PF^-!-s*O&My7UnUbX`U}ry^gOgK>0RKlj6f>Ive!K>7Ov zTq^au3}&F_i!txn=WN)Y`iOGc4w^sIp=tTS<^xVXh)}`0RnLW3G1nHtkLksNHh8s_ z@g7VPdZTy3sbxbHet(U{8zP;%Vk$@36@yX__yM_P%qWSNreQvh^)FEMp z69g#$FcSb)oms`ND&Q{`LD-_$t`pbl3+>sJ(p^<<#u)~1b$M(rs_Tq#8_vyl^NIC^ff3V+M0(QJJikII1HBR2B|R1 zc=h;f6QvB{+lO=SlT)Z zmiH>z#Aq!aKi7riTPLZ?_5MrjNHyA!f67E9T6XP-9ekPr;cG(0_5}kRxYx7v0R&nL zy`QnoG1U7$^ZN=UYb)0Gx0PDJ+3RFExe3-BMqFPkU~u)J5^zZ+qHz8OcwbtAV1#vnI#m zF6dnsCn`}S-QV3{oV1^o-Q!Fw{rNJ*}s1Es)8l{40=6Wyta?Y2-VKc_8Td(+n1BW%OyNRK>eSUYd0mg7x^`IME zyKK6h=iK+5K%H|S7~UtbE!ldONiR7eG`}Tzu|a5>wN!`p;L=p!iHec>dC$sd}#-RrPw0X;)Iu z7ktCst;gOSZj;iJ)&2sG8Ux~SI|^EHZViD^+v%7wcXN_((6e9h@rbE4_IC)2C%y-g za{MqVd+PMpC{1!5&@rD05D1u$>81RhP3?{yZ4DUyi-s6T_IDVwK?(S+O5xj_1LkOc zMKBB*lj+vCX~D1m7=8B7r{JW9xBiuhd>3udp@26U`>S|qgAUWHi;`ycbE3kL4;b2W zXl}~#&0x4jr5r>TI~o<+&w&6(==+5kWfqPE%!_ia%JPv!H39gK(rN$3Voa2htb4)B zKy_IpP*qIHyCl2dSOrF?{1}!<`R4fF4Up=(38z^3A%J|w-c{q}cd$M|{^OK5OE~#} zB%--S^xVxe{!gGE&^BY!kw%$@c++)s6oAJCfvzGhT$_-{; zCKCs;A5A6;#^TsFnJ7Clm~5_#PRWYk^L$cAE)!c`=84I_`xngbuy5dtJOp;Z8Jc9diBHl!@(lBdmBRjXpulGpzbtx;(q&UOX zt*X1%W$~BGcAQ;c&J#@DV~sq^WxDiLoF`n)!JX#dn~&tj4Md?z|W%+R2>g``RYC0yBxt%qG|H1rpor$VJoaP^_&a2Rw}%E8BZ4e>sh(pqCSi;?Gl;vinN4})n;&)K%0d_@ZhU|>hK3Km!DG`+ z%f!)$<`ahoMK^bDG?F(rnVFdZtd9yEvZ+4y(Y>xj36`yN!}lwt3DWt6_i@HujsgCZ zd@%ymPUj)`smfd3rZOTS{vw{YXRfqBYB>~F7ABz>Tf=!Z64SvW5LcD z7%`MQ&O>bAddW(^H&HFxhhsBdL$`qDgs9b@^yvV70z;+|#JeRZpZv3GzlrlhP0v8L zPJUKdIa8wg$(anDqk6sht42qu^3;&U!|6oPj6?vjt9kZ|RGPj^p44}3+pIM95|x?p z@ImO$4mimG$~CvoL&Fz@GWX)C1$F{ za0eGZ8vM{a)I5yXwlPoY?0nC*D+iLa$^5V$2U@gYjsO=*RKdcy9!G9(2NUZ`wPNzXluBxx^0MH91r?$KBOb9F&cm7~3(go?9}t(0t7=r5PPZ z(XSBz#VKmnpU*V78JWv𝔪hJX)CF1}+vmD6X+dL2~2fzNx;(|76w!%VSP8)%T6d zn*Di^>d1?xh^;nw2SvZT?RNz~1D3PM9P6HGHr>vUh94d@WQqZF62y^42nx|Nl`BLD zP6;cb(%0~!OpK(!Y7DNZay}}kT!aWV*j5_SFcoWPZ22uyuhBt_EkRZqX|7g@wsE#g z)xZs27ZakWNj2ajbL*D`A}0uc7j=xPq&-(y1!FQV+29ehy0W#^4N){OyLv>ZWTCdQ z(qVaB2Y%a@>{O-1jZv=3pjl&Tv_$Om5w{S5Ua1DGTcckcxDY@TNt#olMP)*SR5D3H zwciC9`}L!sS17=sf}rdzLAJx-M9n^>F;WgbXa=6*CFQ|S##fN310pljpH1lg>Zjyf z$pc%-Y0iosrV_=@?KQ7I7`HUHGDi+%9m}mT zTdY`A;b>|CmY>(J`CAFJaiomEIRV(BqodjS>-}R$cc0mEOJNnaFzW zzh38)_Y3m8Ad(OCvV?=?e!}k7V%GgH?hzB=|C2Js!pi<{B8qn0hIIfT>d>VZ41H6C z=+80>eW@}l6o!Y`7GB>V0~6-hoQ|t=NZd7Y5`mjrpC5-2&1DD;7beI)%kpfD@l{Ks zy#Sp`F#F(N)g*&qC;RORPouf4@@E<8MrprGoY+O(WW?W&=gk^n-B!!5;ilx@tkITr zbcg3$7C5Y{2H$$WxswB^Ve)bu*gF~zD)D^#d#n^`iXU(5?D!BOst3Mp`(YWAh=IqO zku9s}5{{N?f{(~rqam5z%bI-On9O`LvtpFZ;4Jje0OJDRQ6?0X=O#4rL-qp-Nyh#V z_bA9OCgq0MfnzNoHFea4K*szXfdnST1uiK+CP$+vwjSct@7z1r*h{=*UZ-glqm!@= zqw!mrAqDTCj}cQh9<{0Woe1-z-M`RVcO#?}FtpJ(qJm+NwKp`fH?uY& zr2MaHUk4SPY;7%#to}iO7{4AL7zR~B&Ht2@{^=tNBNII{8z-waAq(3VQuGBFu`qt| zK3_1?7Z}9B#P%Q3)IaPLfRl}hkd2**o`r*rU7L`V=^u^i3!wT!h5l9iKY^+LQ}fGr z;`l1V!c5Q31mGZK%P(M!o|BpBA4`aplkw{~Ga)M*>({ma)yjXFSO8{rdS<{^F99r!^j}gD z6Cn#H8$IA3F6*CK|JX)f4X}R|XZz~)KW%*t_n(jdW-|RNbpL}P``=Bbe+~UB%>RT> z=C{??)9^pT{6BCe77qF^|A^zi<4pe{Mg5n}#Kin{Tfc6`f5({^|Nm4bCPqejRt{Fq zFXoAjlm4qF%NIVz%t_C|!OTI($-zhuVB_HU?<&*3`u(3{{4bS>iH(u@|DrN+vU2?U zE9l(AOif8_@ru*sGU$K}01!_IMi&r~M-(ssqG8Kv*?~mS5fUTHBMMKam^Sw4Iis+S zVb+^RN@y5}j!Qk1*xjV1NFhr{t{KkIcB+#{FwMSamUaGRPJ8s~Jbb<7ow~bs%+z!$ zJDFc8SN{2}rW*$F_fIb&qxWKmLF@0KFF4;WwB`7^ZiYHpRKCHj|Ip**>uN95;VB2s zsliRyFDxo2thuZ_eDC-CLAasD>mxrlkMYnUw7c?+yrim%7Qttu0O?#7jQ7}SaxME9 zuK519t@3$XOZi~*#J3|I_`jGeO`YZ+1_(ULu!x-mKR;UauZdNirh2MDIaFuUSf~$@ zmuQZ&=Rs4h;H+7eDrwEH2@STyO!Y3jx9=`*(x*@ACnd2^#-E@JYa&>LazfJIJVhQ#wlD?A|W$hV95A`72Cr9%J}2@ zQ$SghH)rOA=^^EtI6(U({&h<;K(_IvE=1THasE@tt`~I&E|uV+;KAK9S0hSa_=v7X z0v7~Xh+Fe(OVNk!{`32fmXuX^M)_6Ib98h5d?tzp-D%7j@h-m#(dpJp#0=7Abn;oZ zB`C;%V)b7J#yF6z*qpkErU<`lC;U)N-@9OTB!-~^#}pdA z@9OjTWNNgDf>G^wFr1`)$a!t)N0>b2Iu2&6DX$9z|^83D~1Inh_PRV2n9dt#X{CtyX8tEWi*1W=v&&N4MYIT?B= z_veKVPBMzH@`kcrsT<5|zu8950taQ|2^@F14Bi-m(XCA!>92`6q-y>+HfXe96(Nt4 zV)pD!8jcwq?wfRGzf!HxHDt%~-|K($Tc%Cc)LF$1_^lHR z>u|5pb2^Q{+J~vu8La<&g11NRXYLqf2VbIc>T%MyF@HriL>z~zLJ9Y8sr!K(!^slm zDr6zQu1V4bhZ00|g&4tF2Z0^KItSI%t5tOxCj5&@Y!yt^{zJyMU_7vPwrzeBYzaol zf2oG=n3eL6V~XtCCKYNA=qyBdKt;?zzb28o1|BwsZb6c1!98RATqTJobiZVa9y-Fz zvN@l3z!On}N6yc65eN34<-{`{C`Xdm>jitgu3U4-i&Mud9QP6kJ=gKro7n~HU*fh zY|dv>=!ji6q@Cq)J>}I8o+Cal0nn{`3$(%QO^WDr}fj{g`N?f56CReXDaR z_Hk}VcwKJU3mL{YWeDeIQ1JALS*4r5$bH~7m;J-fUf-NARmkHOduR1oBi=A(Z4Ifx_;o6)~b=TMG>m!*!QTcprw zHHisx<>h5E)o$!m9c_|qCjL3PNS&wQEOg24EPa0|hR00FVxn}gReAW6tNNqO{0VAm zSe~u#lga)uAFLD zO|ISlN8DXD#TB;gf=)t$OK^903$DSP;O^46ySqCfI0SchcZbGl+}#>)oSmok%+!A8 z-7_`c<_~nOuC?x}Yjq#zbt=v89{!<}t z^j6>U-pXh?yY%1g3xDr?-$9D!|DTKT9p_atb2o1$8>XLH?7aP65d2qWyKjej-Y<$$ z4Lt9gKwBQppi2S!i?^4xk0x-oJERBZ<8gZ+9+}|vFwJyjBv{xNl%?RftVk+VyVk}Y<@($O47Ai)`%Fjy1N>`_296#6{`7P2cGDSL` zLLmDNQs|!C{i^?{|E32Etrd!lZvkfSw_AN7x$tGb@C|=V3rUjCpqNU!Jgv(giEnf- zoH%j1?wZ#VZGZ4P-+Jg?U0zf09&x)62!GhLIY-)>=rLizc|{~S4)_`nK={{0a-=z~ zO74wNmRFlMgAdDn-}hk@kUoM~!0k3s$1&p1=QeRo9Ov&}dC*F{?XkxCa_u*wH{wmu zb$I-~cUmVZxkT`^GfEdgjt_@LsO3?3C$*aRId}8pWFyxHauTE{yUcq-zy7H0@hz9= zOB@PSP1i9wKOH}<%}2|oJT}B7b{#!PHm35s-Qg&f2clh=j>GrLz~n3HRw(tp+#OmE z0-!Lyl>>TZ7WcX>@lrCrpS>3Qp|^nrpKhBe4a#1Wvh$mN zxcg3zA#o>^L7(m;%y~Fuo2N8rYuoM-Rxki153f7Wen-6(+RBhWHHFQ8YW$9^-3UM;dTq70!D<4I47J4RAkF`-Ev2y5C^+65%laC4tQ+ zfvf)uOV`fwnMT=&to@rji;<#SzkV7++BXC;_$wZtK-y*e7ej+Sx)3-{Qr($v9p6dC zjOiW*-qWbKLug2~9pG6-7+x`9wuR+J5FgPLU^li&2*O3LXbHkhu2>0%cDTO?aHB~Y zA=~up-Wd}entYBMX4Q-^aU_Tvf#XJ{HbPYxrsc*g=^N0Dq2wk6Y%g-7`*I`5)*((f za;k3=R1UUjhA!4(SzqC0!Jrs30!O}ShTpe-$FD`jHbQ3_Hn-`ctc@+d-V`BzL^u!N z3Pr6Y)DO^eAn!!3nju9Dc-iUMLGt*r8LqLz@uLsyfYqDKQ{*st9Q4X>ZwD+*%t0Nf z!E-y9Azjf5W}knC$jYCjEX-%)5bxEV?HFM-NULCKp?wQct>LSgJOHa5rh2s@z(3cy zyYsmI&IKtQx+`gHhW_x=z@>K9lR6ET5?*tl;q*#p zFrhZ!5vEpMRaVoyu~a>9Uf;dMbIEhSF8gZCBX5P?WoujI=WXC`?_S*lnVU8_9!+^y zb*~0(f#jnxv0QBf{CK3@J68RmjW>Ny*ATqNV~HrGKX7rz_uB~avR#d51w8KAo$VX7jFozlZGq_T(4{n-M4#IFut=p(m&O>6h z>#Blv9j5`kBx%6ME6Jk-WiQV)nZwEZk<_6n(wD3=x=7h8@8aag;tpB55_ZNmSAVcc zqu!UT#OeG7J+TexNrh@E%W5=Ig)CsBIAJyavr{~t1Qc&sbPro>9Nr;a-mfKTi4GhL z#12D3FVtiw_~nGr;zNuu|5=aa&5jftq)cZ8JI?obb@4-K&sk^kSCZ8nJpo{K+9q?VlTEO zs(ad)+FkqxwFmevM@2)HCdQIsNcL0yviA`!$T-OI>mD4{_Zad~@+-8En!y8Bj}ov^ z?%$DZrQQR1cTR!zJKcGDc`zrTt+F0cYv*U#=wAk4ErXu->U;d9o_2l5DxzAiA*)%H zcz3<{dr(!Y-GJ-+S)3j9?)BxZAA{!KyGsOKfjYJ`VYE)TE);GFG z9C12!T7II79kR222fBlfqx$zB*xqVSQ zWHFl|8a?9@uh(FyK2>R%kuRO`H&A`K8af*OZFR6hwwY{`b%R(5HdIC8;_KixV+*8f zAAsS;d^aM`bX{{I*fo=@0PC`ahg1-g`gx-SA`Y)qQ0rhkrXKt3Y#z31(dGBJRLNi; zOONq7UVLOn`00MSWZ+XoReYI!=XL|lf^MhxUVe9_TalOECS2WDWZKwE)$ja2uNZ?r zsny(kE-wn7xrnsWx)CZpEtI~|G3?gudiH9^C=)s597DlP%Pj*88B$sc4;Ii)@Qph%|E$ z!#YxJrjm6D!#Y|GO)fvk9QQ_*RWaX(ZgIlO+u_4aPXiu~r)|UPVD$Y0OBVKD7DX;( zk^YG-sFLxw{I?(ySvoUgtpHSViDmMz!tA*xOXOVRG6f+_*8TEEoNV3;(Xg4T?_2%K zx%S%u%EW)3(Zsx1pb}SAZoeG11r~_>3}&e|q3wXWM+R8lSU4Jqe2wf#2w_IKk>d1MN`RjI>;lWDLnZ^ruC@ki8R;Mls&V z>~^rC6_D9SVyGe&V6@knB!3$+mF*UnDqidPRK92_-)ydRTQ#&qXWgLIz>8_$O?WX& ztIklTs-w+7&v?dAKcp?^{$_47)YOXVzD>j*Ks_^gLtGge>r#@+5%D>&O7hf2({|S) z+_G_M|GTss1?j(Tl=zT6S;z{zkKlsPO*Yx`#cdrh5b78~#`lN_jpXsy#eo7wJ^?E3 ziIxIo-#0fxw7DYuBg~%Wy|Z`&j~2_Zfk5B8e>1`IStUPM*wJu{YwK%w7v!zvO8+ho z+rY4Aw)%5Ib+OFgS9Jd1&{qZ|wW%^nYc1`%rOGBN{3e0b&le%)v!dk+F1Gg({?5ny1u&#H_Yzow>^}L& zJtrW+{J>$(hG6|LT|nJv6(bq7>9TZ~*QCMt)})Be0{ZjMme2#&iDD!g&3P-EM58`%i`j4@@ofK8{a{Pzu$*O7tHkQYrGo!=CHeOpr|{H z+qcU>Z-;S|n^K6EB0RzK!CCKE-7)W+l8_tSJ1+HnL*x@n@}12SQmrt?f`?NVUDOlr zp=3TNL?nc(O?JB;E3B=28zU){fz)7`oZ!3a_k8h{Q&{#DiOea~Ad^Oh5dQe}=tx2G zjU1!tZTMrHhd?mKExR2@;WzJT#8iwTYKo|b%d;E-^$lk1S{7zr)f%E z%>qQN9WCEM*P-inPgl90% z)}A3dX;ZGI<2x|rkp-rjCG6d44w*JYCl;U{EYDK?bx!xFG(-(_%$glyYr)+ri(l3X z`<;Y`R;$6rnxl}lMi>=k;h^A!3W52f>nXoIVsXwPdi)`bZYR)D_7`MgP3+qozb#s{ zN9qaiFx-tLE^=Ftf%(EQKB;3>l1Tb1g|SQ0ty-j+T3Ro5&T7)^rV33vG`1i4=jXn0 zPZBEgMC^rIJ-;+Y}z z>eE-OICbFtYT>ju_fOXhAXxKsD(Ln{H#_-H@cQ^v#(@O|3|`mGhpR};CoSU$2|Ec7YpCZ1G_Vpe{gkg z;yeTim~r2~+N)Rntqv+OpS`sH5M{kJTl0wEKC^6QlD`OU=xV zvn2zR!Ij~jmIBlz`6*a*TqtF~uPVb15g#WjVkRA0*~K6ki!uJgv&Ha9oPQGj06%&@ z$&9wSvgmhyjD1s0lYJw&7F$MvLIm(V+|r;-2UDK&C_wHU2t5S*Tl!l5a@e%~!L($(PfcSW%tM z^HJA6AGpVjG4Mg>cLN$@wngOUa@Md@loLKT{P@bS4@F54FSUC_t>|4*uf9?m@vDMC zC8TVY1}lV=ie4)!p@c*ZNs`on?$6i#)uja-4)G~|7veNzrA zAN8-i6q+=vlrz<&Ifl5E2}GI-hDf>7%D)Q#&Jy2IP8>tCi-^}PUTB$a`SpFP-a-yZ*rJG8jeDfwnS8wxaMR_prmOzz`7L^yAd9)2yCAPQwbd~EQHB+V|sqVlTT{u#QLW%)!a$#|-GF7QmK z%--DK=R(*BqCGBC`#n9`?`zt!m9lSTjAhDJHcOBD^tA`@oyOc0u_*Z5-IQO*t7TB@ zL(zT~zLs~u6uCGLkBA}B$wW@aF-I^**2JamMZXy4j%hoxiQJghSLHY6i7Q!hM)Q=j zhq;_DjU@;kU)9Z-x`!yuhW3xbl(^lxv}fKRgOPEu8>&Vtf2~vPId!nXAOZ6&M1V(= zPUW*o$ZU=KrfhuhWWid0Q`aLC*@v-ZuF`Z%l>hASBfD7l>H`>?phAIDYh#r+?Yne}o*#p?5L z^EFgRa*5{As;1E_nUkE0bGQb;2|b2-ie;Yp@(oy)StU2e<$9ihns|h-<6K|?vUQ!= zq*(>tmS2vtf=>;3sW{gaWcJ;=!%01-Ox{Wx)c*M-o z0XFfL4A)S~qIk``sb+mC?Hz9uFR2W0`U(c*@Ce!#4cXWV)?2&8>Rqi4J2o!~? zV}0s%ZP!1e>&uXa@lHm`b1WE1?FWZq(~=KZ(vWu)^PKTQ(l6eQEpbbL9*P~dCmgCH{ zfK!pP(x5|1e^x9B3vH#KlCE&ep>;7t&NjmCP>$d>9)7NYDq+1?Tf_4b@}eqv$f0Jb zswQDUH&7h8lrj&)RpAFbOA{meUnKT6|NK7J!=3ERH^Lsa$QxueQGc*DXu659R&F!Z ztiFc4MwB^LoHk6mx2mS9R+3R^{x44ecdwzQHsBfXl?6duU0!KQo!sYQZ3N*nvGdXf zM#q>5cf%xqz9v7K-(ujw$BVO6Zz-nmOET&xe-@>kc(xxQAOZ>rztDy(eKNsV93fPA-Y<$b!?Uk&z>Y;x&Y0>R$(9M7~OiM!zK@7E5XB6YfrzAi1C;@HxDURLr zauM^_IlBIw72uK|u$w^c;Ri#5b50gDVyA6<+!upGYmU?)L~1SF@7*OEa^+C8@2?~p zNi`{+636G=>58D-CYQIByfVBo4c5qFM@EN?>Z$F*$rdaMsmRGV3=)1hbB=0bx*#&v zelof|8DE?pNidas$!uWQ$-wbNEqE#rifkhtH(l$MbG%+w9V7ntG=q*CwD&hb+B8-Z zpKEgb<_Kqq9~=7>q${vHwg{@{{JLMlE(HUft?)+_ket7(Now%Qk$0%j`H#k=t>QP1 zer1Y4t$GW1l({3x!Z;%e6`tf%y437yeugzOC3@sBA4ea<#q?|ya3oJK`?Kd2r;+5y zq`OQQ{D|Myv>OV7Uxn%XrTXK7bCZmO1edm2J_4H}`bsm_n#>y2+RruVHOoxg@)Rs_ zL!oHcQn>_1^h!{$Pgh}izA2{(dW1Y01DEu~$()guQ__x=8pWkRMq&CBv%s*TV9sYm zbZD`WdVBA$FU?`Jp|lLFGp?q2=K2w*B9YZTVx_cB$BlWJ92Fc-1}+KkSUIP!Z{Y%> z#`DVpDP8U%$GihlWr}X(?X09|>?~}eUgxL1)PbVxP)(&o!e-E6xvxW{>+^L1QKk4H z2ZSIvu0&JQDBECZHlwRbfX$zPQAm?1g1HVn10@zJ zgOPHgMZwi*yh0I~Gb%5;*1FyM0Mf_mh~Ec%o1#XU6hMu_ok~q)tkOZ@`?$)N9)WN; zbSNEKss}O5A;w!fAs_b@J4dk*%>JWSFuT*Bpx(!f9k z`NH2o*@SYxK1cdJl{4$@!Az{(so{4>gu2Z(+TCpRZYyw!_1LvJ?kDlse%XfMHicqU z)E@G1{6j<_x{x;gNI~gV;0vN|(0kZ=KF^yz6yR>!k^j%EwGa;XtaTpyFzZ{!E>hN= zbrR2J_O;1z9pB!_LgwCeR(F3d?wds%#Ppe?36J{(^doogjr)xq8hEV1uFIzgT=jHX z0`3{`#rvOm-Ywl8R~?OQS&a&nRQpMb%B$UzX~>A%gL+9tp*w~bRevmSEKTX{kC$1O z)>zXRsH(L5?Xfnst%_FRY{#R+r^mjh`Sz;fm+9^m0X*gfl~rpgU=;u zoImF1ssrEV=Q|{Zm6WsGhA1>9Lww%8F z4?J!6F-p(Y1^5Ds#s?J21!>k;=Xs0sUf>$>+{qIp-A2Es+S8vRRyLb11avZ{@_P_$h^iCA^npdl8-6!=VX7zKC=8%f80p+%$yO3gX6eFFjyi9A_||9)j~DEDXPo7%a&#$WraH1@hDVdrVF~BofsQ2%c@gZiVYhSp*F<051A^` z!t4yU^z6g$@eAsk~>p-Nq%c*|#gQTC?W}cfI+BqjA<9&l9m6*6j>z6cA`pzu0x1 z@|e0CrdY0=mP-p02=wJebSn~7|1tuFSDRWo+g2-&PWp~Er{5y7f!=e9ZmPusLm_2Ki-hl7(P2-afSc1qN%W>mX ziSsS$jvd7>FQtd9d_qNG&pwryd0V|FO)^_3pYLPPzc(F&87Z~;x6?7%Z${6;ib4^~ zQQ}J81Ais0{Wp{>%zE@~VeNx>i)ASR>GSwP-aisr3^XOty&s>)XIMu5Hz4sprtSZr zToRTxE@n>uNTQ9AiAJoEqbT_o(R|CMh2)1y`)EZ;-Hw-{~f`V8;l7eTPm z7eUx>dCe%>RC}DiOd?hgwz?gMH$o2_+fOP!Oj!S_(0}3a(5Bfk_uU_Usf-#Z z_jHA?FgIN2@j^?!;2#BV=J6xka4YP26{(U$hLnc8nnmVc7*KcDj-kjnVa zn*Lwtk%jC3vv=g?`bQl(esGYma&dBz@UZ{*FShZ2IS~J+ZRBGA=Su(Gj{leQ!N$tT z$^Bn^^?4VJhuTnk>qnKAH`o+xT5>Kd#cU%*lLy(17XE~Z`BhcS<`@-I0PC0PdDL>; zzk;Tp@!$GP_@W5tH9w;wJTOtt1&92;_XaGjl8W7VJ2qWZEUs$}K zj)Z6kLsWpRSNjF}?`vwmg~A>~k^$LH=0{4@%xF(*47(L2xk4SIOp2{RH%F*o-nU81 zhGKAsb>`^0SmJrIc*{n{>y`VFBH#VvLU+okmp3}?za7F{{=~@~Tp&Nf#}$2%A4*3{ zC(j=bukA=qHrp!I(}=I1^>uV5&)uhhSJmB8N#*);bii3H*D->fcyBl!WXVGQZ#=t!`GUA4L-UOtzGQMz*eN z(7^lylqBC7^QwLNGEs;GnvX0wb~-oPl{|f6lSWYnip?K7tv)1Lwp@BcUsy;lOPG^8 z$1-r)P6v}*DC~7(!%=?+8rsakTHkcBPLTEFa=Pt!Ew497vX!>bvJooNu!Ei@6p~s@ zVQ-vLn5#PLa)X>DtE0XbzGJQNv4dy^{H(C7K*6M z6+0OqoXGlq*ZWlaYP+B3P4q^OOh5v7F(v+V7L1;)%CvEOn~u*=%sbh$4JMZX zyd2eqc8cZx?ARxRTIAY__pSJP-&DAGy%Vx_$1suRK$)(Y5BOSPOaxNacEinyVg`*& zn8zHYpgwzwGvyj2#bz4}=0$7un#?V!yCM8|A?K#$**$PWm!88pd+a&g&ZWBuKA~pPbA`@5hqd`s!Lpb$!*iZ< zqJGWeHern4kKD5nMaT49D&u)shgxt9VkYHmBO_?Gk#_;-N{rLsd@|zkY%=_9E#>5V zcFzPmk?1d22)|3Zt^@xiU3qI7c86{FKxXOPb@&fvsY>CDSB~plZJo292>@&SyGvW} zCe-T&#Z*i0@|3O{ISBYiIt+}{&r8I{Yq$ja6s%2+f83v&L4FzgL*G89i}rfyr*iPU z;V_uX3;X&xw3^Gt`CPCcm(C@DiL07go#+HeeSeV-4@U}qa-0pDh#l+q^c@l()K$MFQt>PGAx(xjrzC2}LR`M(Or zOFXFp@DkzO3(a}&ke56k^D3v=MKXiE|J1FGQj%r-W`%lMSGB5%3vi9=>;=4n&Zn4f z#c(jVDBeipsB8D;Pm2-$RyRdt2GBqaK>s`rG!GF5!jJyj_b$rVycNmZ6K992Gz7*uw)-bF+ z=3{-QSsT_Sv@cUKKAE~Qk^MG4y{c{wH1_t2QX5I5$`m}r)ANyC9U7h+$ZkPujD9!9 z&qxxQ%CLD(Gwd=QrltS0#bh{jAR>EMiDWVh_Xh``W{~~I{xs{Eal0m7MT}rddNZ5g z972ZH22BI|o6azSTPm!dFjKc)fFLo<-p-daq$91Ummf#61_MPzm+v`)UDTdCWvj&; z5$@0zXL2lqh&YW%mTz-eQo;L8hr&UwidcOclZkW|>S)%}I+f&V&1TCe#tz&eHr?J$NuG8-`q{TI0 z8YB-_G>jnSiP=VUSt6aiS~~uRI73A4_|}v6j(S^XCVK!${tu3(_fA%wXHQGq!*$2% z75n)`gi3Jkq!-FcRVx_a>=#kWQ5b)9u8-ARDGY7Nmp`vsj@VB<+M7$m-s~vAnZ`fZ zk>(?Qe22Zf&XCNU^71;o6hDkrQiru^NIZ7f9d41zbPK-}#{||ohiQ%%tiH8T?{2HU zAa!;0Z|5B-YtS)~OjwmDr?#*z#yAs9f0bmBh*v@eu$-Cs{0Y4S`bQp zV4i4C{$2Ond(f5Sx$Fq&$lka{L_b*12XC#DaAVH9{~$ooOdxpkHP*UC3Hc>>h1{(V z>tO|9oG%f7zPvUtJMBJi%!5io}S+a z`CKga{Jt`U&3toBO-~M|b>}#sX(24u*TbG$5_~6Htq-f)oNo8X@~W0#5@W7TNC4ye z49Z3biDuu8<^Fu7s(zY@t)N^J#NF_YtDCu1$hFznLc8f;>w|P+EH_RI&S%{=nA|Ud zD(-LK#{S4ZQ{%aod?`t8=Mz{9(kmbOv-lcdkTRK8SPqNpt{e%Ysa$kiEC ze~YLoGRU+9v(1Ii0lmwP#4jP}AMok&YnS9(Og`sIjXlIC1_?QG9boazfc5B2KjCNJ zW8biS9D_5hJ?o+dNJ)mX6gJ2{UtW+e<@xn@OU6yk!Ltx;Ph9xtd$0g$=u_>ttGW} zuh-&o%^xH)U%g7Cg4|W`JA&C+)88WoDp1Gm6Mp0+{+me^4xtr;7Q2M~edlCJ)ktPj zy7@V>V^f?DGk)%m!FIrYS6CN;LbI+G(N=Ihaw2m3lh z{47}IaLo@2yPIi7BN=o4LMDIYw9DVuO{Rp%+%j)nYl_($_F{x+bsVbH!l_J zFeBt{8tX`;CWJ03XJ%RjhHa5UN_3TrRh+QZ->8Df?SamJtQ5xqr zv|DuH?3RCy8~pthxvU6VlKiLu(v~3JiII+!9>OPr(*N|#yW=-XAQh@KBPZ%*e=OYk zDanN=VP}lx$9;c$7TIX9F-@ag!qbdGcY*y&QAd-KT~TGhvPM!7yumFM529yGHjM^} zeM@e#EVsr~9XK)iqT@jMK3jM2-67FSE3VG&xAUlf9LJnv;-7Z^`F-3AcLcXH_@2q+gXXq9(2*&*ZaLX`S%s-S?r&AeL}+of znN#I`#%+_@34`iD_EY)j)A!7+N$tz?MtvjxE4r#_?(qEhB}pJ&$^}D%8y|-ZPlfb^ ziZ87-mLC}5amS!mA+Fc`anuv&6?!|}QF+#^3`ie$7i|2XlYVHs zf`&96d8L$z=8A{79cc;WX)s=`>x{C`slWHdFpAiTxgApc4&=vD`08U`R)6W-J@T;m zwZT?BidjUfipSbI&WUq%&xqrWvb$IOC^FCQQ2X=Aw5JcWL89B|#_&En-RvVra=F`^ z)0>=v7*?vw!GC2e4hHD^=}6;?hz@n^oo=y^o2=c%pKW(0d8eWJvu>&F_?nrOY9(A* zrL(@AkSyigBSs_;=W_<(I`u}l=D+dK*yQIX@L!b2HR?V^RlB0>mh?G*vet^$gc8tq z%Uv*s)r3}@&URY+Z!vQNjMrk{DXqyOszPY?u__Bhh<;)vFN23qje3>qTi26%yUE_# zcP16j%6s<0uG5S2UPajho4A^t^XARrY3+SgZF}yHMea*RS2|n#tqXlXl`H;rLPP5N zWu#M9=us112Ees@T8#=PM-u-)}>q;)`G~c3aCQtyn=nDelOtV1-aY z6B74vKA_4C+qL^-+lk6BSsUX_j^m!Ms>E3Ff5 z_ngLKdE4@yw^lHqp4z&{FAPJO%dJn#PLE3T3T7y0YMtQ@=d`~a(|2mHY4U3&d$Z(n zP3W82GT>{El4eEzDmneoFxU4(@hdp7$kgt>E^OjCM)Likle}Gwa{rdR2^EA+>k8c~ zll8j@S{-g{xr7c{m$KpXX&m`mpi`>H5it2LjPF@FtF7Qh*+4Fai|BHX_)etb!}rD6 z6?@+Sj;;d<$bEAsr0x@98km#G&k2{G;BoPP>GH)~NwO_L(Aj=U ztb9{e5E}O-Z|&g#2~l0RUcR-&-1z=%+bbmS7J?^kSbq=k;f$fQNAnaqT8&@-V6-W! zrnvOQB)IcI0iCu|%MLqUGK^po6>L%&SdZV3ZWYD2R5O!(z_6i7X!)lkE>-PM4cQ@~ zhKe&cU;rkgTis9r4PS5vv~l*VVM<0l45DI6;!RAG{%tKN?pfPsS^R95vU3d4d@(xh zK+zBKpV*n4)jYg%0BVP;Y*j3|STgfzd8a%=?_CL*8_EJlJ~9Uc-N-Fp(kj5xJ*Xdf z%DlU>TzlHpJkei{Y7j@MtCsiM63@6~qH2%?Mwj1o}*QQ zhYoys&!NLg&+az;!rSB}c(kk!DD}4^oV7Ee}YFf`(J9-PWA# zx*;DcYtI;K8E_#ydk}TJOm%ewbowa5_j`!NL7c>M*IKmNU z@E$3II(aj!wO9@sk7oYw1Zhu`D6}@Svbva;v8-1{eNVo9=(<0fXa8v5sLoGRXd5qY zpsXe#0L?i!G~2(A&p+Eqn71;ho%+6e)W< z!eIDrkFQkDZCiIvEAbQ7d8lh4fXsm90D}dlJ*zH@QnG@9=SzG~))Nf?On>yM5hFlr zULC#Y8DikRM`e2u*t(JOw(mM0Zw$m4TGx^DkGj^0npJ`L3B7nr?$)3Rg6IMC5X+|m z>LYAMNQ%7;!ZrP z73A=sZ942ub-@RP&lYgc{%_0eHD17mo%H6wC|^>$B-*8jwvMU$Tssa=5c?KruJ1CA z!Ull9GF~Wj=XL61H0{!TCC|dkm%#6p=PK}byogO0&cFJ+_c{PU@Oy zWFa{GI%ii2PC%uMpD5!?A@;C>%YJuUV;A{B(^tYvyy*Gj=E(hGz|M5)O$W625R+zsJH2)wvvulaXfk!>~TJGBiOws$j6 zV9_`SKT2P85X=m7s>0+5*W`8o(GTHW)b_-eLGRfQS2l4 zZuZo=&F|zz-lYN92IMdc{Gl{!!WqbqMQi!0KM~sfZz>c@jKc<#11Oe93%lN!Mj)N~ z7A~WnY2}YYerH-vKhCGEL9l^5PqNg@Zx8=Rxi1fUgYHHx^Jxu3bA#P!UClDuD_ROw zZ)~?us84Z)=gehb@$s&Zh8^C10|)5Lp(feUdHJA+Mcod#%qNr*83CG;hla=b_+*U_ z2Kn%wgjyMQx@!aW<#~qJMv8~+;w4Ij>^DI(Oqn>|93QUlOZ*7%d}u?d?V(O_WPVKd z8!`g%hEN9No1ygzBPh-kHq=fR&XL>Ll&e(~^TTdzOp!7lotFaRGt-=3%S;$A&ibF2 z5?`1_+)6sxS2`Zr0hG7vOH4pRZwg@_oUcDWswv?7Hzb!ikI%g2-UG6aWafCL7Hn^U zq3rwAxTYHs-EBe@bVDPjy`4=O&k(MLIxU0RL&1Q8I(hOLNXoH1{@uUnZOyqGuXl4p zo01hGn{dFkER*b(mFPvkCce}$xI3faXNuGrVn39|LS%CYiVR4>%o1aHde>Q`lq2V6l^qh=cs+P#jR8#t0$IASY?B{M1c56<^h#%OG+GpCw*7&1mil2Q} zEHyqC;LK=vazt9qdT`>EeBVV!{KR^D)-+8&QEd;(rt(+GHt41T=dB$z*BNia}TINTl93nObKw zE(%RFnoD&zsX9$gbI=AiuTt3$@Xuc@cJf=L`cd_38Rb;axIe6r-o|`2Ijl?3~C1GSA zV$)u4r$I%Y)7?0@-oQ&C>09qifkV&rG?(0omoC7~B)ibLULC+@{b({{L8wPV;??}2c+ntEI5a(SODF=GFK+O6yk{PTakxp= zUFZY-UTja_a-NBTQ}3qPwV?b1Hqw0zdQ(|adGFb={R0Gk`xImwT-XAca=+_C<@PlJ z&AoGOMQzs4a#?i`tJU8`4!!v=62g4DLp15mbA z9-+J02Xj0>xcnTNSN4Yc5YwLV1*8Tb^Mkw3DR03*;xDz&@`+w2T?3@?t(H`;SXmU|uO(Z=EayDG!e~p1V(Pn- zXrJGZk9Wp$mRoWTJ^8TTm?Vj3dbVtflmME7u3@^OxJM`M$jEvX5c{4o`ab_ciqV>7 zZy@vX4AsrImnVd*P&-jS3ax$F+0xr((aWTXdvCJhs*fbLWAl6SDSj#Kw#F36tjMLJ z{I)M$L{>0JsrUvr@_^`jDqnB;9*w>g5|!b#Bb-d0$yLi|`xCa5ZJ13;(ifbEeDrIaJtdBK7wgI%3jfZ46;Hr@4`Wo%B}Up@KrGm2^VZ{4eek6(rAU@*!2D;Uy(r^Lxo%fDpd)*bc5 z_PmDOwFst=kwc{T$lW_`63%}MJ}8tlE4?Fk8_f|*Se4D2%q>Sqm_0JxpT1lgE-=PS zF(#c_Q;XxhIPwf^Z%7IlaN6MSFrV&`<2QQllg{Dg$rPqjXAT$E;M!UX`+A4Z<>c9A z%5Tmd9+3hx*Jg%h51*#knucEaxeP~>s`1V|PnKD1E=HE_6KrC=)_mP7b9mUqOngOp z-0ni!Tjr%imb1R8DTnuObzeZ+ZI(K4t%)@bQO94WV+nl zv%cKThIoI!%zGF2=#u1GQdb-|uIcSY`EjwLvsInLT4|sqMBN~9INA(-BEa9y`S*zQ z_)9P%(o5A-#ZCXyH_1-^`YYArub-)x}fH3g+^D$7B;z?9Dx7b+Ib&{Db%y zM6-GS-Q_9v?y#RllFH)*oj>kI>I40bFCy}HiRZ+SHaI^ngtZo4|8^JMCF3?P@+xe`;+`9^jp?E)L7TczpsssINiW=ob!r{$yME>Y9))q&<1&* zq!u?2M=Ph&^U7HLs8^ydsU%aApvfe@;ils&X69l{*XM#1!WZm{q9vTB->)q=I?K78=)yvh*+}x&d0ww!WUn=3HZqBmA%_WHQJMJ(!$i3{ zA{)6Q$gVT*$YhV0zh(ciIs>d9?MN=|bpoauzDNK^O8A6P9*0M5Z#WA*l;@eKTytOH z=A=y&jV6odP~O-V$ap4lBsHBmPCz8swHn6&lX9GnZ%&0>s$Q$vG5%|DoRS|H*vesj z+R)MbxCqR=4sU;bK0?o8Ewes8$d=0Vi4|O*Bce`Bm z-7Y>#U6pLlSO0^#w~C7E`SyLI5P}5=?(Q`1lHg8ohu{$0rIFylgG+FCcj-?J^QKFs9M!MX3eU(x@*?@eAjBkY_L{U2T=rUwSpkNy}zxceCOw-rv!QPmnADwfy~=@S=lT6T zNga3Jxgi8DkF)L!V?8%_1E1HL*alcLMLurEwer68<~xC21|Z?s+>>9Vr_T4VNv<*F zNW?cGURbKMu09ym%Vxb$&r1U}eu~`y2`?aZovYh)KwfE=_siKz!KlndFSCa65=!7*aNJMK(q@0-H7f@--z5}y#I=Q;3+{mRg&Sk(?Q&5RJ@_B??kq#1Y=8yZax}t zZBw)|pv!eEsWVRH$f=sGJ2{pR;9A+eP!UjN?O)^mZ6qOP{FasPg_V}jv_7ATgu-A; z+~+3;s#B8*m$qNzevV_hEul-0sw&Y;DRMd^B`@-a_Z`uzKgzX!q6cpfp2cx5D7gC> z^o&OwdFm(O-Jh}`kz zj^7~2f&)3~@7ul6T6$h5-BZ{bE5Hm0jyP6(qXia9Iu0)CCV7_kKsMQ(FSKyxT)vl; zgqGctPRH&o6h%x3?=|QN(cowF-E6E|ygwe5m~}2L-L|Vk7Sm}+Z%kfDyiWMM7K?sS z)h)AP4(r?AI5zg0PlA360b_Ly@M)Osni7BKnVI}Dz@t?5rfu!7k8y%95|MH;yW4)Y z#=Z58@t208#LwXQ#{?LUhwIRfEx-Z7qRM=e>!Yq~QJG+`AO5ue@(Ex1(Kj2Du1|#F z)Q*&Ewj$Y__dV(@V384AY!~IjO?q zbsPI)qptPB_%ptn+R!qHz?8dCHYLJmhv*WruZMAn+WIJZ5mkm`1O5!}DFo233l27V z{PI?A5Zi|PTwN8h03&wv#YIX6u`7NG?@rzDfz6=)^xsCs@dhZ?Lk~1wS*80W6hnbg zEiM?qAQ;mQ)!>a3jjrMLW9*+LraYa@E1hPh_z%3b>Fpox>USozzz~C>cSoMx(5uTP zxMn%m26r-rzyM^F>tTA=|CV%L5P5DXXQ-rJ9b^9_QAed3Hu2(guEm?>v-kOD$KA8 zqAyx8W}-HGI74{y&@$5=z|C4x`1?y5App?rSl`o z*NO-9-9k-dUXu_;VMIrm)R?WnIgx5ySaUccIN&dkXL*9|xn0`;cw|R|S%uyQ`>@f5 zhuM{q9KuG*&{y z`GBJP1H(@|M|h;;Z3e`89xyO&;Vbei0^LSP8daC#az70Fgn2->jexj}V-{Dt1UNEc zD)N8#H~w=@ZLiYn??=;PRto-!M;&}|v1ZHeD{Cxj^Wo{WZ^^_HLG!JVWRT6(fR95_ z%&Y-8s5V^eSKY+~L=^_H`z_+)mD?2^y=klj;yYOX)?L$ackG?nF1*k$y_Je-5!>(N zPq6LC)N?$zzb5)2mj!cOrg8VhuBg~9sgxH3nyEvq#29VYOSnj+Pr}Nr7&l01Cob^p zF1Qh)PPq0n_>$YQgM=*jYM3U*zh>ul0`iN1DPr34s?#spIPJ>Vnz`NQgC|cUU$^$( zG_O)-;`6FohhfX=rmJpnFajScD2-wU<-gUAUlZ@!wESZHkt}b7Ag2Rz zOyGaLwz}x`Vi{AL{QI@b_iwN;@AHZAcfY)SQ#%EHdE9++iK&kdrI)h{vp6i{@u&wI zp4bW=iK7v#w)5&L(PH{*6;D^}TGtPU_aykn7~9(~Uidl45&_(P;o;SUz1{(CMxybOYlRLe;7dJys=d^|O=AZ=EC1y2Z z++_N!7YgJ&xE2W5fUCRU-tncs9U^IbFl&7-rn)vCo(|K+4*?F|wXhzOK8B zA%k9>?_F7df>$uRK+2zeItQLd{>YHq)G)Cj&kt~f@Mti4_f!9G`m%5iA~RTmsKa&l zQ&g=s`QInWbP(Z4R)0W8r(U16nD1bA?rk<{BMzc&o_j;#NHE)$48A%-u!m4OM=vp4 zjbH7$6~JUC@o`ojA7Sz#LpA&WOX*_*P3aYCD#w$GB9ueTiM;p z0_hL;1&FIY+E~)^1AyykdKEclb-et*Dl=M^djZ?*y^}h%OvB4$b$cb}qe?ZalIzm# zQtHBqqkZ2zFp=FV8e+A^RmV#1OTeF=44Su>DGkHEiKjSQGOA;y=CSIw=~j(LH|le2 z%rZ$D{J3+BW4yLM?nGmB;8_>yUBWT~?)B4Un&g7NX6&qx&C0+$2RiG5%T>45QMX2* zYxK%VND9A@X2w-i({g78cp09+8zALf;}R!9e3Z$$m*tB%YxbuX8!VfE>{fiV3itK| zE$N!AttxN9EsuadYacB?Gp+2LXt#p_6UYeVGhPInz^zT&@S<%YS;uK(8K1YKD^pD8 z%e|j9*xnfK4-|}<7a64-3Hj|4~Ou=Qk z2j<3P7M{>Xeax@OnfsMeu@Z~Y~7Kk>B4bN2uPWCou42KG~)%?S>7^>ww+zyq_XP3IQHzG6jIG1 z`uVn}b)WethI7A0sH1TP95A&m_a*#~l8iSvTXkPJ`Bx_I# znOR#gxP=|*j2fi*boQMSYG9k~RCGs(V7cN#QROPD=(;z2$+IvWZnp=aox1Lzq*aa~ zr8F4yU4Mx?rrZJWlSJmXiu4oQlh#Tk03BXS7*Pj83;eG|B-DUc6+r6xtbljlu7ULS2oZx;WxUqH@olt)qnhKmc*ZBj|H#{Wx{PDJ_v^z*1``y% z6mqc$nZ5gXd|;3(cwJsip8a(+6Me?)Xz4UI5RUBQM`AzWHw}*K?9(S-z(cU26mStC zSSG2@Uq6sm=Z(I$F31M%eepX<3HQUj$L%I&xTWJ?oyOnmUpmw%i?cbluf;oanOh)_ ze;q%PDR>J(w?lPeAh&lnPZa38CO8RxyX_O9}ey)<9toiuI^ z=s&^bs&#}TA}minkD51mV>EUdQTsfx3s$>V|JAJlJ5HVx6NSV5`y4@8dG5!^9P6X( z>*Lm^{$>(Nu65U*f{pfY=FPG%^4`E}y%5u(QVFB}Yuk9G(ebU`xnqCsyY@Ek6U{H^ zaq(fJ&{c&;aMYU#tY;k^stb360FqiE-kWL9IGr1ukR0mHmC+d+< z8cKJ3ZujS7$qgX?PR6H-WfO$34+77G+@0lvmqYVMX4aQE*Y}UUfu@t!f2<&p90`4` z3V;xQMF$~p@6YBPL3c&N=UZh)*(3-AbIdI6)0^5j{2l|Ir0aL^4xQ`uzE{WWz8n+l zgKEs?pywQV{+~X~+Hkp8O}f>*kwnAJbIGR9Q=|9pgll}goaV)kAnlW7)4%=wplge! zf)U9Q`2gE!C^7W5Z-lR3Pmm`}#v&Gu>RJjGL#k^&c8s^tX>cdHGUQYWfag!HGb|(F zpx7Fv=)#y>XWw*x1sUSq`|F+%TdsMI9Dyy6RyvHg{pRgr+f!gobPgbA%iB-;Jat|Q zH3>2;T3PWbB8)>)Xlmw*AsSFuvgP!AOhjJ|PFjejuPyyFqytc$mC!5C zb9#RPQY#L)wR5LhArzFAJ(XTFxon=9pM6_da1jE;ip^aQps&oaCl!BS0<23_=KNVm zK&DYCa7YxsHz=j;d{vgy@ThU=o9ra!-S^vNDu5C}>Xq(U*dYuN-~~i=Cd6zLZ$y zH8EcGP_KE?ow5#x*yI{EgTh^sAJ@)o9E=|nR-5nLmjiZv0=D4#)mQP2vrpJ2moWcN zyspcI%{~5K4SYY>qO5;TZpGsbJEeh-3-b2 zg^L|O0B-vVhn|lrZ&%ka&T&i-&r&ZtU065R^H(rlMXIiW0`!%muqx1bYya_C7f0DK z`}}=}%^VVcU+9%YcGHJdp*|(P({yo|hvynARHZA(;H@C!?r)CF5wG!1Q;ff!!dLWp z`7Tezg94s~;7fknR>`(~Tl}k0+P%qdXML<d+ckDOS;>y}&1@I!mSu2O6M~bIZm#2lca_#{$8*^z|`L@9x;xQ?=M~*TuoR z+ekhHi6!mZY%uXXMDDzZo>y9{>#KTVKkjw2>BmBzNrYv_okPaW?l4V${Qe@UF^@4W z>*pK=yD=uP1j_yxHsn5$0}6gb=hrer1oPsluNAlyYEmqc4YHH8f7`+J?d@Oa=#N*q zDv+$qYP6xgae95@NM!N%fJKC2n-3cAKk>Hxb3ojdm>hE47};sAq}f^>69r~2Gwn$f zZJq59oYlM^m_%>lhGwTGht^gis$%vy4c>F2Jh->|^tWK4a5PC{V*N?+_V;n};Do{o zq$X?!hcO8kJ88TZyT9yr3hoe-6|NP?6uoL-;=pnFR}&-# zA*yLI6xgmqYOrIq^Ks~fn~(&Zd{7Cd>*oNl8>hc-*zDQ^X%N~?0opOv!H@A7)iCkq z_v-s-zl(vcKHrfsrDG+Zh+WZd-MKAzi&Ye-nm-;h*p#qYRmTZ_C)#%gHEtVB#JXAu zJsA$D7m-@MOst+$_&{8^Zhsu&Uz~pKc4efdAhhAPt``uM8wQms6tlQII#vZ3Yjjzx zSxt>w;g(;oT`Qdd&zNrbxNg!Q7=^%f@`0PRTV;-Qat5vdPs$jswQt>U$Hv0NV8wK$ zu6wBG)-P8TkbBWRUs0#$!*$06T!ih)??pAA=5*Gr3AsCl)^WW0r~divoDVIrA-H>1 z*7q9e4dZ9WcItdRK2Y=e!Fic1`D~HB)mG8JD#bZw*5mIGkAgO4ZI0mGN0D+%I7E!{ z`W_OhRqR?^^yDF3>_#PQ{8Yu*UDXM}Pjlqa0cKjc=N!lgO(5n1xGG|7?oC&b_hlB$ z4I^qg7Q6|0iZQ zaZ1nE;#eD(^(+Fv^~`JBB@rd4xW!>#k*0vUH(;-b*ve9 z)i4|Fq(62GJa@gmYZOUdzvh^)<)0&XfbyP=f2g|s+43s~d}dOEdn|`&P~AhQ z`KoLhT)+3h`wMxH$G+e}at1l%_W{LZbY-(+P4BP?=WxR#1tx01>saE%&8k^25pSaGZ&vr65N8NR$;U3?>6fv-SAO)$A|g(kg13_bRBJA zdi`s?*6^o+#^T}40XCReK_%OQFsacfp|%cAMogwkZ_;A)l^fGM5;Za!%Z0Ar{6j&&V_G z?X`5@U^(0HSa+(mKrix3^v`%d*D`!re;hj@yW@F0xgJ*JC_IaHJKl6C&!L~((PJ5- zFJEx!nF)2&j<=*}g$Ql#?`-TdRHv4j_uanUJ3{hZOrDDJO;4JPzIR~+e;2Se-MgT% zsa7tjDw$3NPSgP>YNqpqd?~6TQuXoR&2z&A0ghDx+8&s`pK?@W#v40jV(APl*JL_m z`mn|ttm>te`DhZWm$lw!F*1{4m@hARW3f%-tRW zE2|H7_1wlx?dkoR1^K$5Vt3_P3q2ZOv~LCA@4sPBBz+m7ZtzPLWi_kE`to%PjG-ipZ7hcu;Sk zW$aHitDES2d+>8~gp+i>GurY|SIPc1hOVT+vF*ZYCWs4kZ8pe{_4MEBTYbvO>P)-8U$PsQey3 zip21!=S_#)#WwK@h<$MQCUSb>z%DjTJUR|Vx|-hpaOIlMC&tzJtZ;jM7xEvQE z!YB50zE{%4Cr15K&-0#rOs}N@V+JM2hex;_~l+nTY8Q* zElT@u1{DvR4-ic4ZYYwC$v_~;NKY~QrOapKnh_R2WsUKH;qb)}(XO*|u+KF@#pi;n z(+AGi>X`H_@_@cXaD8y>>^{;`c1v8heWX34XLoZie9HAL=gWOepB0Vqh?T~?oTFci zpTO|SEr*4%)mjWdV_Y8OmvA%L`s5&2yUe-f)1_>&n*3pJi>cOf9TqGH;P>WwM zF^_ZJ+pIpOvD9oB^rf#Uk_gYZb6Uc1)gvpbD8?4%(`t8+DcG?Z`yv(M`K|ITEarZ6+%jJ) za$!!(cR&edn>==Z@-(>jpdSu#4r=IKf}GTm<*^Q8ck{_l1$e$T#MFlfVsbW*Qpkbt z9V{l}Cifs8?`DPj6rV`{(px)@%x50kX^ZibV?U-hTUm>#-^aro*Gt2*sswZV@-Dan z)aC`m4s7Yh-7akduDGVg3c<%0!Au827EAq{#bOE7!=zEhzjlQSpq^Orwaf|^j9@dz z`3rUg@ejut-1Yjg#oVDR&4mx(Ytrq+Ug6Ar-=K#-@P);)?K#1v9H~d<+R$|YJ`Q7) z=VbDN{lfF1!TZ6&b-EivjR=Avhu;}3f{#R|L1&MKN1ZyO_Xg0&Mylj~L-KP~u&++;-DH7asPj!|2 zVO#zRNUq>M2@a>N1iOPq?^li4p8uPdJght@w+}pElqPkKcC91db*?v6&+SgpBUm3W zN5%ys&z~~N@>bOMdL(O456Hj)ach3z7eP z=I1gT*U9PE-NHZei|TY;(1!XuS6^icre5d-VcsFZuVb(H9yYm~mb?Rjr0$K` zeEUYp!8KFpPojZW!}tpqitwr@JT>w;1%lOsy_N86{~^&YmVOZ5ig8?+#$tKP6`gDR z-}DOU-THpo#7DxLueq+It{+s1b!`pLx!?nzigL6^-R35%K_?37(K1$MGAwT0&z=eH zbk)tc?ZQR~WeEe6hDoj&9{USi&_}`=Hr@b)36;b01no4D;fgi^ve_u4z7LQ0=s>a@VYIq`xE5%!Ee>T=~rB@8(ycrseaG$UKrO4P)li|JAn}vT=ob0 zwdHHgM(u{lpUVx{hVviYKg>aVZ}5@G(b-T%NguS&V>a~v@cdB;GDZSpo?$NLePLjo zVVdEXp`8(#VV>c87Ukj8AQKekrgbwo=qzMTs8-NRa|2Ep%6ZbSrb53=naPdQA(#)F z3R@D^RNypr;fXyqn_8Pjn|Yh@o0sPtn>L%eo6OzM*d)w#5)iAeCv5Hf^=5caTzG1@ z@p~|d5w1UdC*^~)ml^ChX^k;)dQGuHu|mEg_6=1JH8u5ad;v8MwK}!7RCUA(fi96Q zo;Pnj=cWA4=s?Cmf4n^*BcTnU64!gqFe{Q@9Q0-?4QGp1mg8fCql_bt8RVm4oB=M6 zZ|_jw6%zou=7Kd|F z8N7kKe|Ycgks8rkPA5FX7cc9zhntfGPJgiy))o2=7Si?u1S|&I6!uu!3YWA=OWDfL zj8PkT&KF5()APrVc8YOpW-16O{V4 z25s$QCl8DOerez&<>Y>D2;Gg^nE#U_t~Ed|k8TmN6e1Yj9_o`1KuA!FKlhS57#@^M zr8i7(q&Q3|6q-y0CXy-oaNF2SUCA*gYSDHDgGy_~GO=SrmP77C!^36r0Q4a$7{;+6 z16$~>GIaOxTw+tETP5r%0@N7eXw(%`g$5c7e9=U+{=*sB)=y<|u&ND}?BoR@dR_xBl9R+_af4RrWJM}97 z)6bxL)FSqZ+EOR;O)nMWSU{#W%wV&;I}R1hvZC4AA{%LmJIbb~;}$U4ZpuieAza?t z2^-i7?H!IZo`Fvp0DMx8QKF=VI0z~@HQYs9!6tph#dla= z;SAtiG{c0RM3>quZQPd!z)>hkW@9omf~(tD5MMy0mZXr;;KQV@b)A9a$Elbz?FfYGsx4(RZ9>cI4-0j zG@NBIraq=-^>cdo@MuSCXS${ssML~iGdo%lRe^dSq3rJAuH`;`BZ|LU>x*3i zEHrmPLtV+q$v`Gg&9llC)wZh3)S<|ss$_j8KJ71^261RF6!s-|@{T z_)UNutoJDj5e4rp*U@p7=+(r%VrHvJ({`5YU5OOH#n(2_aQ@tD%sf_;X6WqDyNKJs zG*-@Ew)HpC23JNKT4k+0Sx%)l*_oe9RYYYFuYpx5yVCA{tfK#%Bsra{X$Cc47gdcr z_^%W*CfR&^y`>+ z-eC>g_U)?X*R9=J*In0W&C|osICFsi!~U_q&wh!f80+1|7Ww#t^Guvb07JGePlLI@Stv_8`POf3-#cg5e&n7TlGqt{HMh$;3}k?;b!H3nxWX;KiCuy~ z&6{6~j^^NOg#67=rS>X<=@IYQv=0 zNE|^~KZKChBJ*>+O)z|@if?0kZd@Ydj=gqp7v#>i(v#zO5I=JXf2 zZ#DoE+n9@hZiE0?-SFY*Q*j53T*C6j_E7-I;%gr)%ab5K69N*eE}cRvF!9<3O~#-s!l?0LW9pzrL$mermbU@=9VaShD(*8{xl3qY@Mfg_*BbV!sXJ9k(MEV9jTVVD8U_+rB$_nH0 zRcXb%q|Y@B45Xs$V>R!%!p~Qt^gVy!_nyNtkNf$SzdE6MZLY-P^x~u_IdO$&A4<-C z8_<{$iu1irr79DW%zD{JF*8s5TPI{jmn$U&(*W#7iLT)~$AsohTV#W?Nm z9Tz6bfey?v_01d*#+w@AA=}43(n{MXvv!>Doh?MyaTEDn0gg!@;DO!5jq2fD?HKk% z)OliUX6+fW>j(b`c0nxvMcS8!@4&C3&AJJ8gxFAIsODd7r?S7it5bO8LHEFDM3hI*oH(Fd2d;N!B{__yUwTDP=b~b3Z9-+Yk?07uN1h$y#89;3-k6 z5GS0TBn*JGiNmeSATp(g$Yw zV^N~gwU`kH9wCp?1TB(%g59lB%~*ZKk_)Ju6JW_cO6pNsEaLdj`&=2=mMokTCDE_C zGaEP;hB*nsDYg^-5SUNIQMAVgR$GivM%xd5&E#4z4yxR=+;I$Jdt;-|yPfD)YzwZR z>_1toRvI$Nbe$O8(4x8CsfC2p#-vFOmY`F*iRSR`G}MH)ro^4?72s3e6#MpZ2h`hg zT@r3TpuQuIGY5>$H}9BE2E12!kB4JvfNHJ((lsl@K|SfCnm_iof$UxAU9}-Y;b`c| zTe$-_Ns*MaKFDD8qtNA#SZ`ipg z&6VoF0zH>Q=mBsuJfto;w4G&-lF*RpuRp#6edY1W+i0sxw;iKWYTx=}t&es6blWN) zIkKg!a4fSva#Tw}Mba-L4_vX1`Ml{9o>QV?gGs6(q90G4?0@=8mZl*n6@ih9>rr)F zCO7M*ORAOgw60ns1Ro3LO^%o>PonB>O^%q0AGV6BSsA2NqH;XvT{%1&(nLchd$$}7 zx=-SI*V{r`TH{!xnv~4f3My-?v2Ag?qVt1Qlc+-dWv;#>*h59QBeNn+anvWf817}0 z+T^78^0y7u+QfXK6_^l_Wcvw`DjzH!3KqIF+(d?aWpo|O4!od9sWXm6*N(C`ZP>o)8qkuEJ0=r|NySV9(ap~8aFXqC#yGNy$)a*< zF<3!&xCkm{SYpDbuqZZ7SfDoN#K=oeN^C4|tdQQ~)>r&dNKa1!_7;Jnk*jpr*rGH+ zOPW)p1X2(@E**W53T=_n>7Gitlk1;ru@GJIpio?2o+@r$xOa1Fif&^rICHa8CEQKv zid+@f9wxZrgfeuCSRBX@-Y^D=YqU-`p5Yk~3 zb?OSSo~JPEhf7*mAy%IWN;_!Bp!;AtL4XY_>lPc|etLA-o7>a}GN9 z27~$qz_DeA`L)3LD6|r^w!Rcv>3_D~KVyD<3#!7<9kC#F)hHPzIq;K^osS}Q`esNM}kE$z6pbL2(HJn;&}WTo{GS#=lLp8Z#zPc+svDbDKH!_ z^V&m@?|Ucy`MS$+JSG8+C&{JB*PId3xCi77%ssL|gZD7Uw)W^$v;-j4rNvjjU1b>% zZG z_v6F_h?}H~^;JGG-fah*Wi`Y80$$4flthu2P2ledqm@#lMGwY_N3)?)qE*kx8(XxS zZ%gRkGx&|9_jhm7Ab;xBwiZ>AVc&w}#whb)GL6>f=VQxZP%fJ0Wl z!WwAeBj+ciAqqkk3G-64#7$0d=^+zNdFf#S#?(4a<-Xxfmk<(0oE-)Mz1a8&PPy3R zK~AaSA!nD)Oo|1jHu>rl+udgpA>LruWv(y<<{-AW4q1G_!1iUDFr~_ndc8)t&EIfO&`3TUU#2oo0VPj#ENe0i?_J6Fkh4PDn<-KYYkouuIg&Ov|ue9IZeI zC>b#B0f*Rj-=8+9aqgm}o2ewp>`JvM77rV%JS4I%=5*>|I(`sG}@e+ND>SttvWZOx;@y3eVJU4{m2uEOu$j%M?+I=>2N4;@%UY}2!~RK872!JNlsQz*b!iltz8R1;N7<5W$#iwt9J zSjI5KV<5U|tZDdWOXW`$$OK5pt%<&(ije9tN(xaRz1aH9M(1%(MM?Ds8;~GLGqVy! zgCjTON~B?@v)6mF37>mKOJVV|uZy4+GVQ_l-r<9fkNMsUC81c=~^oDzqy)zNxTj;V^2+ zSMr!f+&g|Bx#~`NV-bu$7eL?Djk-KFMYIv>hh5Kh)9pXlq~){u{Y{m<5x=S(tr5T6 zQaPg$KTFT4A|R8B^+cxADJYL3ayJZZ_{f*I3eQMn; zQ^+(%Y!4UjF%5s;5FM8I_q5g1#jMrE!$DyWdvdB)Ns`tJ;tw)O5@)g`oA^%OQBz5< zuwO;IHlA%HC!oQ|opcK@R5wJ-&d+uoJB5m7KZ&19HCM!LSg~*C)tVaQ)54cbU=x@a zsfT!ot=jxy+U;`^L&oP5rr9a->q5TqdkSEI2a;8AE{Rd$IGgX$Ic>Y@N4e{Z2@|5UT(NyKw_KPrfZvWz6*FLVH%PEb=Qx38~%@!Opw|@BIh(ff(vhWd5 z_bc@qexrt9zRw2abY7l{*e}+?nlNpZanKaS2Iy%tsD5sU<^@jJ1}Tyv_S(2(gYk^# ztdohn;?#?qW9plgkfix4+`A5cTrLq*R`F^*XVT0h1+Qt_0edrt`8Mfizd3th3w^_U zpXx(N7d~Ld*wD#Q1{;CQieDBet(G>=@j>e!`7AuF*os z&2Den;!;UX)E>R5_`~;-jM4>@OI^peB?I!eiqbBn_t3H`#YY_w*=gebVOR5qW)zMP zXBSJ=-Nv>=H_^JL1Qvq~_gU4C>_0<92Z^mqF~~aOIH-`?FxHYY6zXi7NjzN_#|>L^ zKTX4Oqlb*(NR3`f@5sC9yQ`bwFu=s!`>CS{IV(o@~643?p9)e4T# zZI*>$e#qt`dhKaZVyUKCx%m*0^9ZS2U4f6+UtB&W+oxDR&F%^%J?P?u^a@ z?{nO1ks*5L^3xfbiGhO&A;B3xq3^eemx-5ryZyU0LvhZNXL8v%SlLchc&O63_`k7- zcnR}*hNy{ZNE!cy($u|>bd}@5iTa!`T!EV8@?LF7J%Z#< zl(l}EjwAsJxk?D5JUiYPnPr1J(=Ja+6C76lN3COvk@F6-=;mow_OO7fMN^xfRN#?} zvmrqWja-z>Op;tMtdggZh1KPzDE!BXoL09AYrK|Fd9}lqAF7v1(?cM=}eI_1@y6HH@Ds9s0evf;S<~Lq7uf^;+GSU{OSl#enUfo3*|#?kAZAUem{?3 zXb;{tADb)+(`@sGQ^NCxKPc_(XZ({Dp-I1`@$Lr!(VHJ%QXemW^&y4Za^o#dM3RD$O>V@t+9wPH7-wfBbcHCFA9+&$toNbG^-UYOL@i0pfwm_kkjomq`1cl$e~VuKPdN9#_;}*rVHe=}FAu2XuHzxLwpcn?fU(rtrvO~hWp*?T z&)MqrX}bed>vkd}pWF%vxpdsgkJ_T2AVoZysAM1eyASmkafRpoQUQS zvE$RdBP^AO>ZvE-Ar6da5qlJS*j8ay?0CG)!SjZ_4tP8UBRHNe$)BzqpP-KO&*jqu z!zR;p9NEzWY77)2%>7tYBya=PH>XS*c;Y|0euQ>QUIDl#10MU=9}gjtdXKPv@Du9_ zDk1lm($lzrhZ-=#cpmra3DGb1xPJQ-R`F<{O(ry&{6?wOUp&gYWKeUeF^9?KkPe3b>^DlqZKlGq88nPyd2w58B5^qyM}J(}-YB7qb^Ud! zWoc=f#EZ?0b>5jGw$IB|eQ?XD2=gD&d^6hWzm*)^5nAw+WYEB;-MOWLYU6br8Q9!5 za}bL&q!L}`Gfyv_p)fVjhCQk|?9?yU$R>A{^H!-^Z2M?(IY;NmzDt(pHYCCFVdsir zX37@M*k$gi%lZ(#kRha8P~(;qyvtDSdZHPV_*XL~SSQxm3!Bm~_u@`2aE=xhub*~c zOJ0TxVOEN${zkm-{%9pP9q{dAPAbd|+wU~^tU*XSudVNnkvSW6^sVX%39$U4#JH*b zUj2||=?*1@!CLQ5Zu8+y6f8*{{DZEQis||x@z;IwRHVNT+Te1wZ)nb!aljY(Qs2p& zcSM8A7~J?5C#)jVv+r259=PQsmL+g}j9ruuOQy`%;`c}nDH+aq_g8-GxwHu2Z?$=< zz|2>&6D6xUrxXtbwb?|~1V?hH1X5O@(vaVG@#SsO2mvu-@4ek2u_DP^E;j!0_|@?= zGB7#z+NzL>-lPiXVZKUI&l~lT-)*mul@?<3G1FU2jQCFJBQ|n+HmYZ}_&rz9I2Wl8 z^VUr0JLCVy1bn$_(!>Ee_1Tjvv%1i+0O@~mkdPqc?2>;wMzvV3Nh0* zry}jI$}#kSBcuAj9^8sieJ$!Mnj&?0<*$pHCw8>#n>(Y79XoP&+p0qLqQ5e*pA72T zL5Dpu3suM1vMOgZ4zbRKBG_LeW6Vex7Z-#xnx$JnA9>@7NlmU2kL+dKj+sIfT~Ubc z%6@9Xr;Dpht`u)3dF;`DSta}|yY(2MJtb-fm6e>uPMgTgY-G=WC!K5n;srwkrjS-e zLOA`3t`v*9V_eBs4S~^xX?qoI+Y{-$f8vCr-X*qNZ=6!vPY;0!Ar?g_$^MorpkwpdJ?Dw!0Q9VaK`&!#f-IgTM1v6Sl3det@PtMDLGz zM(sjhCVb~@QF+>*ht$%lTyM|yg>Al2opgJ!=z|b;7AQpa5tAB-8DLVccnfpio;-bX z{gG_81|xJWni(y%xcfb>nzDlnE$ZiqQr1G^(miZEg+p@@c_NFdNdtk7b|RL#ikt+Q zvZ>no{E3*tC1ox62ks&Byb5P@O3gxnSIi>rw~@2&P#b*aOQvSu(MRysTNnEdWdUB7 zqf>eoas;W9gIG#&J<2PDXQgk3+=pJ>z0pWJXlj$ETvjP-C0C}6v0O|o)}ecwZP#?^ z>}l7r4_CW0>9_G{1A|n2cC{P1iVP z@z<8rw!ajrqt^2J=6-aEvB$pBr@k>@q{y8%sQZlcR|UNUrW!_Y$WVY3z$WO4jk=R)vEH0xyhAQxS`0{!#I#FCM^+EuzJveal;d zkuqL3^pm?$C`dZy_1m;<0bVi3lL65xMLorY?SU6VI70P~m$syIDagvtbJ~zNAlYbA zc8uH8;TG-etKOuz`SL#UW>gP=_RoJ zv)p;smB5B<*Y<5EFOf~G`J0CF`<*1TL|T~s57apqD4R){n?aCtx*L03{fAWYK&DuL z0LwIb#}dC|xq)D`nKv70lZFDY!1@I@PTvMRaldk;Ga-z@uab9Sre9b-a%k zXpL!n7^}55Rf_=5i0K>#Q4cxscDu@1V;@jYAja@62J%u%o0h3ns~SF?2JzYSlb#H5 zKhrBI5`~PtWlncrMiA%_hjo?306`yz?LmZV^77}kaS;YZii7`n{W?F z*|spMP5md&tm3$~YA=z!cmeHBva5m2sI}Z2JPEgcO7tR?20A@s&o}3cZObEHnr-G5 zhJCsP05OedW^FwR7{j#F`m=G)`R`E!b6lbeDtR`t7i3#Xf;fhM@>``YqkneoJLG#a z`pC5P!Vbd$3;1hVNR`y(hw8^nqAYQw>l1VvrKKsLHyCLk6yOPJZ8YK|<< zPs<1RN-W5#?5Rgk9#k0YK2t1Zb@keArV(xQO8%2Zr1Oq<{G)NBB4mlOp=eEUrsT>PCsC&**k`xP%kfvuu5xGW@Ay%%BSdI^ z6=8g)uCsebP`vle)~>;zSEOp_!ux2(9Ut8iB*Et=pT-u8ZZS*!< z;Qk#F)ccv;EE^Lljg?}}oQeylYnBG{ZGvPr^_wq5=<|`CS|q#d0RuySyCCcQitJ$0 z=TQ5>5}X6%CZ6%_C3BV45EnPgnZ47n$Ek!d-A^`AR37%pXsfH$KFRzDahfCZf6U^h z%0IonlSqh{YIRtDOgGA*E~+qae96b<3E{Z1u1dZCN|8QFYSJIo%EwD7!3vFaV=3j} zYdR*xY&G!!yvb)=pM>7F(%IJq*1nJL=Hh7 z2-I>WgISbg_+!Z8#ovYPNFpbjN9-~gicmAH97lL*qQ4sscPj8FIVONd03wZv;Anc& z>p9DFEO|jmb{nqo>v(7vp}KP~;>**pMdb*tLFhwZSqqUyfn!x1Fbh5>&hV5Me4r&| zcqgItEWPRw714f+%Nuvz91!}9mC=4_tj^sedC2>jD&hEtEbSM7v^IJuVs(CrA#S=V zj=G>Cpmy^YZUr|el{2_$io-}o&MV+JN|DDI3kKe5N>}{@jvbCGv+!P1l0Q3sdD)<4 zRO;H$Wsz2Ti%_SWZjT8<0gjeHv<$_X@5b@kn(0oKtmyb^R%i>mNwaSA23^4~mWZsV zl|}X{bjG4LVpO_LOOix69QE+%HkaU0d#yNjb%=6FLv4(yR$Gq&IuuXGiuL=Ado7)W zOW140*q`yJxc&4go@F&T8K~PL4491Kj6)~?E#<4x3K&L@MVdTYD;NqAW2tMmW_IgA zcA^!z{viC*E8|7Q2mC`o%5Z3K7RBQ%y*A7za+#%`9%y2ljrV!_;{Mw z!;{z#37A&=yg#cN(c&;mg}Ggv=t}PChvV`<=z5rJO@iCrp@vB4;qw*an6l|O+|DNH zZ2fO)jaw8YT#+(b1VlxP#+cHHBgE{ZMbT>Yqf|`xKKHUfqMFruJ2B>F^M)QOUe5{o za-kBn_0yFD&vxffjH9dOrzddZMopv!5_D881nVU!@}hp<-zWtyN z@>vfl0+752%LYJyi|xB@XOrucO#P7{z9s0!M-p#-!;2?U2M=J$%#U4?y}~#kk|H)) zT;IdfDI)aD={QHs6XirgA-TfxE;(>|V>ga0vTP4I9>&l3;t%ANhwXN*cJoM~T*d7( zA}roXRoKQn^L?LhVyvAg_2!b+oa)bKZ%Urz@uSx3`;(%~*b%7< z#>P}dFi-PuGCPe7^mr|N${%G6xr+?Fs@f;cxZIeAKi6+Xth^=KhTUcjmIltw zCWIl0%BRYyzW9FxEu2(H3seHbOJX`{ZR0{3L+ns)7hfP71Vj+`-&HVSjt_3&SjVZx zQ9;`0rj6@diTw=e#~=miK2=i&n_vQ|X!( zQ(}o_L_~!RM`n}ry}zp*XcO>v{h`c4)GV%J{Syc$O~`3gSZSBMi;A=d-UH{PNdu%E zt{UDtyem4xt4+GRcX;h{`VIC85c?LVlT$a{|E|kwFeiY-+`N`Y3+lKY;c;vMY$s6~ z5AY#TZ2`8Zgk~f=QsMTqrHiby6VWn$Q6FfU#zz=WelzFLnX3p)pykwcFwSc2)Qwrw zCrT5lFYonVg-l^du3WWGeH`R1GG2bMtB?z(=H3n@PWmc$~wVv|t7ylT8VM_?y zaRa00Z)e^Y{(9cive;oGl6C;S%OH;mL$>N^`R^ z=u)}zy8qZyhMcDA%$h=?w}l0R4Mj_evOOcx2p=#E3*-GaLqE9&Cd<6hE5CSd$l0E833zw!o>Y z1s>SqOf|@a{W9xr?bmE|zGCRsUp6kTGzyFGEE$mKd?|^<70O7;horl=??)Pwv=KW} zMRh_mT$$5^2+k5$bXv2TFwTrwE{Jyhx0?V-GY-ZdM!A?Cj@ZOc>A}Z(SNro!QUM1mqqwW)#^#^R4Vi~(#;Ec2 zj_6merfFri=`d?SV1Hh>U20yBP+!!2sQC=A|Jj#`toMkclSI~e$r)Mt2HQiEIZ$A@ zv=ENGqCI$+6bJl-nA74wD3@*q#BMw$kSH?#(I|g5w~_LPZYdMFM0tkY`g~%Iw^Ohp zRHunB!p$PzdDdp9#%r4=*&*%o9p2r5*xyIO=&)bc>}MoG>DeDcJTShBmQD{A>>RNW zgOKWSf{g`jT5t4sbRdK^^Po56ZEFIY zD!!jGokTy`BC$H5rBD-}vczmN`tPxif8A3OKb_+JReHMjwquvyS5QrcI2FM#y)-WHi`CoRmH|xe8j`1xx!cN%hx_^<+vPA{nV~tT{^&{E^Cq;^JsI z1EQ73-bt;U$4ejia?}n#R&@Zx#Sgcbibs7p+vgp>A~I1i)H!5bafbTq&XBLleowD9 za56t2LGrTD)43FS)fU$8e}EQUjm|Wc2(}sGuaHd+A76be-e2ClZ=G9xs#u4g?#cf4 z$)%a23~Uw?80gGpHoPhXDOF|jc3<2j;&cR6PJELkkGmrACJqsk!`wb7EjJ2s%7jSv zwG*&dXH(EuehWUd-KGtF)KKs?%a&S7TPFxH(^=O8^PzJkraFZh=4!D0Pq*y8qVdVrN#<=_~w_4 zQrVlG=9iAH8b7yUAvP@WPHjSN<^+2;LuOOmA?hfJ+GdYm&C*vib|W&Zzx^ygBBC>k z)#1~pR33$0CK!-hmm+U;NkXo%rT-dc=%~`k6sgKQxK65ZMv)0CqS>9_?Z6`H;fNtY z(h(Em^-V$%v%7}(>#{!TmBEZHz-h;Ur2ZmFom%htQ=ZF|6Y3HdhQNE_@4?gvy*{n7 z#(~{tZ)UXbD?K?*isl6#Es=5Ms?)_vr2zGVF4B<5rmq(H$-a#Dgc8uu={W)@JuZbDyLZ!)(|408Rs+U`+*97zpCAj z{b4P;Qs;Y6{NzbSW#^qss0sq_`EzFXcSXz9dDdAFvjBtdr}apC*T8f_fYQHvWppdP ztnQPh6|>2}G`-LCOd$~g5b*U@FLTI#oZ3}#tq@1~4oiqxGw}$-2zGI@$(qO2%G-|X zBWU{U&}P)Q3cNVyY)Q=k#tWf6D#-Y_K?E=E6P#Z^5;i<*1`+t10%IMc$l z@{@b}#`D?7D$AqeRp?GJW-(^loFk>-`@qTNXlV_JgbC8Fs>*D;Ayu0?HQDp^kH-|v zp@B^VUQ7hx)7R8t2k!pI`6PJI>_{$i$O#&pwPwuVTGv>Lbfp(oRgIPCSq>&O1;996 zfVNvFe|jlXQH^A76QNNe61=TrQBSWP9rQS!e0*&5(zoO)LG_kjdye=kA6@c|!^>hMO|vChJHV$;K7L zIB42xb3L>42bV3~bO_b#zvYQO!6xss+Hx?$Sb~Ab_*-PdFA>fO-y?Yq1-) zm^Nl}vRXX|JKBrWH6nCoweal%w$(&pbq zK>Ib!fNW867@Y8m#!!?}sU64DBV8F~L0K2|BT1A!d*RGTu(r;G=$l*tZ#R%5suIn+ zBEddAn`A(muu&d%w<}qGB#WUZBw=-Ccvq`r{mw$f;$asnkN;2c7NzQ7vSf6CM~ClB zl_9}tP>Y(0klF4>1O62nFJPnb9+mRt>J)r^5`TOyR_wB$FkpD}umUXwjEi!1?` zP>o58hTMxP#`z^Zp^dkO9$&b|AL680~JH8hJyGk4ATG7h=8yXHV;YTyF zkkSZHREb@usw@4&6Sr-r0pvE7>+e|ps-KA|8Q&=R`j}?V-FM$j94X4`#mV}mL=zwG zsxQ`6NPV8B`?J<0yL~OU2OuD9g|gmZ``xYq%a@j%jC#s-CW}CfpH$?#+#NiWN+7b8 z4L-_YRFx;k@z9Wrz)BaO?9P-7xL_5;zih5j-Qpr6<3Zwnnp_t-Pc1f96EPF2N5s^% zWhrAi5tL?9%d{kOt==V3+jDdM+<_v9)$tBP22w@YEKuA;Y-%Cof>hjsg+*ckZ16%R z98pP8byXQ$byJdb$m*Q?AFTn?e?xYR)H}TNVLqqj5bUF@psFmH<1FrfDzyTTMFk_O z*zf&F+0IdDux?WO4da3Hk*0gDrg5ZLDCwGeU#UrO|qDwWy(nQ|<*d|hsw8|LtDiw9Vdj@$ayrDnoE0DE<1sNIlI(M^tN z`A)**c|TFh_?*HrbLqnIX(v+~5J5I3Vo9s?wG%5TCQ-`)TA!=O)$hsi& z9f!(g>TqQCs#*;+vI$xbXa1N8I2-ZL<)*41RGbz2i%WQ6ID(shh{HU1PNK$AmR{X6 zE;;hziuSwCr?G8|1{wy{MDJtbsM=!hC$^YYh{5U00k!Sg86x8t`C|kE|io0k53!mE;AhthF zNh*Q@cG#~{we3jx#la@125=&lAVjm6qg{N64U-M9KfiImYFn%7Bi#Fh=r6` zQ4O5=Ya3(0u`@B%u}2>;yB(P`Ibf)Sgg@Rja}&{-1N4x%X_^!>ohg2khiqlSo1xnM zNa=BOi^TIFVZ$<|2uKEQc4EL+b?M@rd`1Jd1BS=*j@b?5i1|vzG<6?;t<(sn+3Usu zOPf%VkEXU$HfqLjM{0K0QOa z@<;?ah>t!R(=y=IuP%O@*2uW4Btv!6|27_2a0T~kT>LNt4t`ROADcc0GL6KErMZoA zDf>=9K$EC+*?!Gc)@k}MM*sJ|sWaxZ&x>mEq^Z|s^p?Sf!K+QbqemPn1Y!*`sviknz_$yKZh>KCN8rlQXQ)u|m=9`WfzP|& zivOoSm7-glTS3Cad^Mw9Aa&(!+67}@k5ut`&9}H}U4P)d#k^{kD>!ypX1I+190(r9 zEr6bHOXT)fX>ZqHZYuN>zVBYZnP2Sc@g*>5Atk#NeAN@{7w6vISf1N(K>2l{(sVNO zgTO(Hi4#aXRahd+vv^%Y_n6t7iOLXC*xQzQ z62VHb-7)3F5tUXZ_<@huML*j&lP*Td1J;F8D3le62t*jd(%#gFVY8&EF|wD+x~+rU zmG#@O-{^90_~Iy=uUKK%NOirQ1-(pU7l$z0QUPa6suHQOC~G2SJ^-eNxR394RPOZ} zOFFTP{2oB#N&tkPKhs4?07@@>&Ke}2J1a`EW0bK-NAZ`%PsZ(D1 zGfIjEZk6wiCM81rl>NBMNtMvj)Li^Vl~9N9X%gz{%iI!49DnU&nOCZ!zLcjI*<7M@ z*qK?+V=`s-3eB1?m2%}#`=M=kVH?NAGIMl&>b>Gp<~#AvneAd`{OJ9a)`EKeEZg+p z${-R6xZYuUJX11SaEIKSnw!$*Bw3LlxUhSSZcP=;kny%E*6$5>G*}iEx%^{*^0{xB z<%=dZeAt`gF1Z4=@mVCMSqy%@LANAVOmIBwl`r5V5>*3XM#9ppNEjhQPzf72tWbAI z$*~FY&12$Dg1BfP{G67I5tFqf$#h*-6VUKDJ^7v483jciX9^xQL0p2%ehQUll3qP& zoxYX~_7`{UKHk3by1V(aOCD9N_3UBb!x2{U|&IKBQ?968#FKt zS$t+^7ka>Ss1a2t$p3gwoRLa4))s;dnqTz@Jz{Xkm^fm<1{zkCty<=!zQL>s_3E_# z4h=72cegb&+L)!O=@~aO7-P)PZ;{5g=7OGicVhF$lFIE3G{usBvm=6+ugzCH%rn_# zT3qQ#CChmK<->EpsNOx-iY;yi(Tj6U<+(JXw{;2s_~Qusg8@xQXNP8Q2-rb{#jptC zkA^m0u-RteCQZR&Nm3C5b|1uDvkrFY_kuaK*RZfXyAw8xUPYKpGC;}40Avdzo#7rt z1}NhLI-0q!7y9mWGj2aczwzMji+YPM(IJbYG$Gj4S8u{sHZhz4^yg@$uG3+ z_vU(o<+;rH|EM9!=-VGW`8B<=FO|7QK0UtbkLsy*ykGdwXK$VIc}>^hzqN}F5w3bH zt-GpNW#7J+qFzV23jW}rm@!T(-@CobGn6hPPGiSU$Z#YmNd92MBAKrhtZ*rncdd3E z9;c4IJ+?gJyp98}8)Q0y%bPKFI0@6^rhz1fWQ~2k2m}M-TeZe*0apTIry|TImJ>l* zCo`%|6*3TN+p#f1gpfY1)V7J*y%pyR@bym%-SWhxS6 zE$(Nt!g5c{IA}S(_UdB(9QqA%mCCr6JuG7M+&U?seg*areMR&x-IK^vf8LP5wGM$@ z9kFq`0z>BtO^x~ll_4chizav9|aZk)hfw#GRxk(Coa<(l_x^h&akoA&jX zSmTR8qOJC9Nj#7JW13uA>k^EW4dCTTyrY@B=6gMO1rU4l@N+F^dk)jO?f-6hEf^P^ z@xUCR!}hLFU+BpC`OiY*=PJt+)!2XaENK`I6>mPeV-XG&?7F8sqSCNTAnPNQIUoxA zZNUDb{3QiW@g*KukvZDei}_?}7#U0TZGXDGuljG|-LmGX&4A(oidxgh03lq&mBX{z z=m>L;Ac1JH~p13iFWmhk&{jsGjWnih}(K-eg9~ZTbQ{n zJ8PHSodFDa?l-0IZ1^+RLSP>3hN9Tqr~YV77w^rgR_G+IkRV%GlYKO#)Ez`I83A)S zqgGl$Q4uq0934!(z~-DO&BD!wh*QX9dK!b-Qc+)936IBoYT1oy>$IRE&P)a?*|Ro6 zo<+ZBhQNt6^k~q(_6Zs#6Xv@TTRlt4gfd{ywO_2dG((7BT@wVQ+A}^;4{VWs{6RVL z$0jMQGcg;&LFVBSK%VC_Xr& zfhZn=7pyf_wsVHA0cqBw*v-%Nn}U*wEzg-qAP4%Lmcu%)wkFIvD5}amxBmWQ2=)wD z*WUx40nLd@ou$+qW>CD^s(REbhZ=N~D2x{hQR)93jU2jT#hwZHE0rQA-eEc>_va8&EU((M7tSb@-aWg4T3J~U+6B^ynq<|hE7`@K*(NOt zRcscRjm3}hJTf6+9Ak;Dhgko;gBkWsn1Ns6q1I|>F0um*` zNBiP)Z`n0Ry(=_+{Og)DZv}`otQ@@G92H5-NNzOuw|I%iBtB$R_Dl`u(@*-B6D#o~A`8@~%@gNE$u+bb96OJ+T<-Xj!%Q__ zX>7T7g`4d*f7SXMsW-W2A?fctv8ieDz-wfcV9KX*$Dprf%r~0Rlr6_UvYqA7*34BH z{C|H7ai{W{TW-_@KDgM4(wULat~|cjd3*Q|?K~~Q-y_=ilt`I+L<|ir?9nqMT}&5C z;OH^kWpK{$kQ!=K2b%KEik6qPY|DT>uJ&-@%w1k6QS-t!HzWkHd6;FL*fZ}h!sTS^ z*ztFMMV_g^p9q`Tx|o9t!$RzXMdq-1Mmq)(_{vJ00N=3hFr@$JX=dqA61 z&Vq<~VwE@#LUlki)_0RxA}Nx7i{D^Rm5*BK`{gvZ!fN-Wd$>wC`G1kRzz; z!)Fs#=I#q}K>Q(M^e4Iv>IZ4Fvmtp>_PA%Rd;ScWYtbvB&x**RQP*?Ej@`dba+ z?Zq6%J4WFJnJVd_cr{iPIKXQw0n)SBGo?8d9&*s*N|Ris#M8uqm}q#>uUPLUNyoXk zR!VV$H%+&3L1di0?8JRNqST+ktE12h>pjxTW|BpANedXviqlGEBaRl9B)Uw|^UjuX zNRMJgb8D=F&2p?+&om1^VcXSm6=lfIBx!%_f5C10Ac96&Xla~~Ak#|Sp*q8t9Vujj z+tz1CRSAPiNzn9f)L-<(+gM_aGs?-G0{BjhYIx%hJiK|->U;$O=bJ?$vwota?_L$Z za`O59`J84OD#5*|SSSK_vZa%FlhR$(1H)ies%M&rW9A6dvPd2OZucw;>Z-&=zz8MS zN7mvzwzdUKqrzYG1HO#L|Izhd;YQ+0!doBFQ>QZM6@YCpP;m@wcK^t&+=YdAVjjJ5 zdg#=E(4PA;qGKmpdq0(07PRZN+W$!FiQUIt>Teir$$Y9=K2m1+Pi2w6<7~6o65W|l zK%oF;f6yVg$ ztJTqd7l3$mvJS;WiKcc#6a{F;)Tal^qs{+ijTUDKwnj6smlrh`B_3-?^^d>RHN-_` z+PgzPuW;A6Pm&+UVTYckryY)^qRyVw(uUr%r}So(b;dCb@m2cMc5_=KKz;;UktVkU zefKa`s(Mw!VKX*Us$$2ljNXvgZP{r>f?rk#Z#>yW1+n8Fc8tK(?|0)8k9jdD%zq2B z-_cgoQgJL1_^1(xopyRW`YZO|-CyABtf#5QN;aCPbmjL|=MPHV)aTZ$!7p)N&Ffx2K1dfcPjp5kD8C%! z0T%pcxcOaqXV!H|OS>E7Hjm#Hd1$v~bT2?;#3^#Y&!bv4--@Y)(X)Wi^m++}%7bKs zz0kl2t}BoZ*pxx(gWjV(POZ}g|(?0l`yv&uTg=CAg9u74F&N48`R zDmvsU?$zY;9cS+}%*MyjkOIfxKB9FmcK*iVepOG)?Vr*-w@wVJJGTsM@7cHo0f|1K zcdo6QC7eoa)hf{OzH$!8dJ-9J6rI_~tCFp)>A8D3xn2)Dwn|J(djAq&|JvPNiVf#O zeO{YF@SxG=`~j}Rkd+gIgxmD2-c-LDXW25&_#D0Go1tTov>ONFx_QE5z#p-Lb-b@S zJqK3LJrwXZY9s5C`L~YX75%t>9;04s8v2FW7_cPad1p86a+at_#Urym7PAbPIUn}_ z&d^TC_AVgK84*iJSM|c=Ix;721;)9JWiz1HSYiY~v0ma04erKPj$ZlPvGj}4_6bg8 z+!u}{yYED04+*t*a!MD?QINWQ`krs&e4tA4AJY*Gn>PFm=XeT8eY-qyC9GNkZb|D* zyj9%p@7yfc4Y42JH1a$W`PRxigmnYl5^X(4WQ?}8If=0UT}>ZEwl^O=@@(hVxM#Hj zh^*bWwHxebokOl-;z!UE2*(qhAG&_`R_cV1-MsCFFRbVs)ly_Zwr+ux0<*8v+k+Fq zxQPln-Wy4f;F0n8o5rbP6+;+nco1x1?=DwEBZgbIj;;qWu-+r^45L-NEiJN*2gX;f z@t_}+0RQL`!RshMuIPa<#p0@#+rXm1*oeQG^TBKDUhabpC0~8TU+)~ zC~ROS;lS$`uCxxBwALrUvROFR?ly0>HpC@?Z!~UCpZcfr_JLHBpH-IA74bkHSs%b%YG%F#}NL~8>#eSa4R-V8OY zob!Qc(v!sy$>@Ux=L<>>z5XJVONC*GWNawd_?#%tPgKVC%`Xz!%^ z;6fCb+u*GM;Ck(uqAI+7jnf!!bY z=zji#nHZFTj3yu_VQ0Q^yz3MQMfEaPxf!1uw5VA)oB6<;?SEw&hjyz6`<8m55`yeU zZ&jm8B#9D1>T^2!yD2-(brVfhahEI2D!h8S9C0VF27D1Rbt}(3!AUufb#+aX`~BAv zxU-rEGwY%*pOJ|W?E*!c99b2fD;*SZm_Hqzuw$st5V4;V;LDZl>kw(cA>4AT*80wH zXFHZG%Qp$NJv_hUFE2f}YAWt=>u{TB8SQEQoza-~jCOO2-e&Njg7EQD!dsJ|L%Eu)IAd^e>Rw zXV==lYFbw23v7K&POR&YITzSz7S@2IU$+5x?_9YiJuI9?&{Tfo0hAi@;44#P@H@j9 z`kSYUo7U^;MLPicdZuy)E0PHRuc#+f7DrK>?pX0ZfZxn zQs8#nGd;X_*lE< zw6|QnYv+?(cXx%|jFpC^h3`=&)?p0+a5oGfJav5G84QFZ7v5B zwXg$F3>UM2!!4$V4`PpMK!KRUJ6;}`~-bG;db4cWX{UDoHnpW{x@>QX?fW{ z`1{P-1J^FEko_rJgNNzj&!U>V6=UcyxV0F00{D$0Td-Jnf{w&gQD3Y8`qnX2pwdrG zPiRXkqpzR6Dd>JiK(Grakm5u|ji8g%VN%Xf}Xhm}4<_qOa)*0-1tAnAbTHTykBDE>Jp>scRb@G&2 z&|jZgF?*o-r%23p*$(D3-tv+lHSfBPTF!mBC*?3gUg5Lsp;&jFjbiIKFD4c7{-?;^ z@V5<&_HDyW;`HG0If}Q~cb}BQ0QW+4$li$}yoLGUd*+3$?u9k;ZB*Weah@n|iGVbj z3gqw;&xKRSvXRqLXy_Y(?w!syYx7C1ulV2Ru*FoZj6GtxTKA5^)W=o9!f?LEB>=)+d(=S z?OJ8^%yykVrb|rfJ-r6Es;S=Y;k(j^@!_4^+=(Z=shN|Pf1#!09td70-qY5vUI?mG z*u@Ekj$mz;bn!tSK$u|rjE>}_u;8h>czEbfD5oMDedTG8j{#{{2Cp- z-;45Om4)q?jp=3;2W12{Z)rkepTWAzW+M9>l+i2A31yFGywE?~`tMSd+0KBFy>S4s z5Lq9OS;CB<@*i>21V`fi9tH=fL$N$V{?`nm)U7curR$%1yi|B>ZYj^xRHKm9=&0sM zHu+f0{k;VQeqpfg(#`~vLuYPjXHt?{ILob?i=AAacADFwms!q?*yxGQ&q=!oxgHpd zjM$u^$6cE3i#^YXBnRpMFDnULW7>ODhwYna9)_SWkSYd)e|SO%zTYn!h(XiU{oSX$ zzt6**1~#(At1Oud&ZA^1bd%9~1m9~aS)uM|A*9-1i?$Jv0^QfHZ z#Caep#5YFZ`**+5RUxtHlhT?3^(XhK^1_7$OYJ1|f*wk`m*#^7p^r}$poe3h_h+*A zbM%+|;6b5>==bY^_orK*%bx!s&t9PC-k>~PAa}g&cf6$eJVg7vZj-(5KHXNmOZD^$ zgKrdkg344-Ze70+3t3p8y}3p(_Lp6wOSG-~xEtg>BBUL#_&h`JaDPSGUHJF*Cefsh z_4UnJw&U&Y+~*~kA@W7v=k@g8+rxf`<_?eTj`|C7e20(jRZ)?KOIil}&!5?o4*x8M zF#xAeDkAak>G?z1$Cif#G@sMH2+>8mM_$YOn@;LbMrQ6w#|6);_E+Cw$*27(kZ=9L zw@2^)El_^wFr^G2r7Xx4crx@IsO2Fw7B3%p8t)Fu@-)nY7z*8&A*abY)*r7T4>^Wq zlPFyn6Ue+Kc&oaeCNf7Z-gQXET|)X3b&|B7cW}x^{$$;U zp>J%z!(e?VxiyEOt$4_e(ef@{D39nR7dPPWbAd+eKB#K(#vb>)b3sfA@7zyErSc}X z>7~SuxzwkTz72um(X}kjJyrX;a5_cH+WHdtw-9aDl9n~C0YqjJ_W=4~@(gcCaLxEVbUAnlDj69LE%Lq8n zWe`Tu_;&XjG>YHTL1a{p8BVc+ZdqNlYG~AtN7Nola(}A}+YD>t2hYOn|EMEtIh5_D ze;&jCwrk5!M@GfDb)3z&Y#pz%7RP54ZHCS{BsAaVaBReAe6Fd>lz0 zu;@J_?Z(;Mr0p`H>FS|4tFCj#ZiV0aG7|E=r-wDzRRJ4Ko^L-6x+yj(=ISBaI2SXk zq(AfL2hOrl*eV4%f|P?DWDZXWIa+Mx{W)j`Frz;aKM9Qm*`}w+RFD?gWt?ZYeEOwJ z>3%hIfpr_x=9mJv%C^=vnBVQKRJ_DT%;S=W*tbjgCE&2dZQyZ_H3kqz#!Xn{Gr&KQ z`XxKd-DOvlm;JD1pC{oecc5q#y=9EOO)xoU>I+=jZ=E+ilsHF=9&L4spr$FlqcM0} z;i?KZM_hNGN|n6(Hyouj<=;yyGe5K(k-?Cc7_TJ{zKPNsIb$u4=%JR8{7fIM(K89? zUeQ-wCFMa|{jlKXn&9fc4pMu$*7O|wN{K@ z!SHn77A%?ljWwz+gHm&YuF0$%Sl-rRYx~pKZ+}GB<0lA;OBMkA5bP>ral5s^8AXc2D#hk7cF)Tlu-u!ta(6k~hVU?iN!H?L0ypE|xGeAJ11#hSc&DV!bj)qGYbb&vjbM@d?%^Qm{*VO>6dHnmv_7%uoq;xLMoUhQgdFsw+5p;7J zPb5X$^bwR6QRg>m;PadhRQeW@Gi9qxj=Dzz6#RK}8?Ss=6oy7XiE>gvQ$qm@wQJHh z=CUixdo;1_ntj|#fyP{K{P>6I4$K^Aa?$y*?HIg7Gr&d`G*1G^?#49q7r3-uzl%ch zgZ6X6q6EiO51X~7$BJ@`AYHhGXk})?5VLumkAJ0{5E<%jHaRgshG;iC6j~w8o;;p@ z3v0DIMRlz8OUk3eCn*wXdo-`zB7twxxZu%;adwUX*IJf19sFhk1 zDv|MN3?8F`U9O-?JpnNN@~x-5hOcvdM!H3Ck@Qy;@&=+3Xu=D>2;(43=LP>mR*c2b z3X9AbKpksep)ylt9V+jk+02thMDub_rt|4`1VYi&=gL$&A?T3%${YT6yJ=bRVQ;yTaTrDmBsH4n_8RxXggPDdh#9LaG8Vi zoS@J*X50A-YRQ)wgj#!o$<&CbeiWMJ-|`n~w+Gj<=w{Ek+y^Xu9d zQJ%2AfT)wG06Rlu1R?s5%b3iL(Vyy0h86ulq$Tyb!lK#RwEPTtA~Tu4B#I8qBboQ& zv|O5|mY~6PjYZ|+T3I~y_pSxVDLs-T6d8`hrE_2}SA2?i#!rxIW4HS6rk6iX z#C_^i>#FVkWYi&Db*A^-@|M@6*k1-_s_>+OG;86TMSeJDO!&Y*BKE~Z7`JgccaJn* zH`9k>&Zr#(5pgJ7+VoXT@t=MEaC``n1h{=m?1XoOKFtI^1AOn$JFk;;3`XtPmPBAK zLcdZUSw|8^U$KjJBywH;2c#TzG4?I7&nNm3ti2(N`b^MWk|yI)@4B+ zv%H(5Sen!U|B*nkTpmSr>Zjw{7;8U+tow33KW^SsIthkIuTh@ze?E|L%@U3`8`?M^*<@qzXBv$kN>m(7ce%h#%b<;2R%159GjpA5obX7dD; zOA!Ot2_U62o~Od%TJ4l1;oWmf+a;y0Ou(Ig$9q)9rFoz-|CP41?I$KCH|=nkvyag$ z$`qoe?&=LDbR4jHY4N}^Oqk$-9cz5pCXq?nb7e?}-`$)6sb3oq3K+qXXBSvsN9<3A z2FD0;?OO7egB1t&W){Q}eYzH!4kVAMVhtA@tMN#-^1rHL+i)|a#wQxt)myXAL1vGL zp8P+?buPyj8h3m2D4|AwJ>%Td%Y(%MqGmO}86tZXVbqEy1iyIcbj2vNGAL>L$}dFE zDZ@Wd*7OXYtN=Q@{jth{NW`%~e*I2=_FGcF66!IRsXyv8Js{Eh$SXL~-AlHr%{RBX zsBsY78=T{}IC2{{gTN}F-{P{?gh~IYzSJ}Snp7rWp15_1UYs~ek2@pOkd;Yvtd?BLGGJ1sNHcJO(n{rdQ2l6LUx(w)GyOB;YqgCO?7CkhOk z@vzb0enDvq|0{{|O5&m*mN3zYL5pdwAQwMAYCr_+yxtlqXQqo?oZ#N7}%;E2>4>u-nTl843Y-V`?RlgQWGgP4_a=8L$4MS++Oh2jQ-{EvrzsBqzv z{h(IhP3QQiwghi}>uprd6=xK$jqqC*fR{COZ@8^xIu}35#_Hlv6pU`ou`M|EgR|1Q zX@B_`y8k`cElb#P!D(1CO+Mdgx6H{_1~Y{$4w&{e-^pI9qQ_x~p<>>k2>1lU zp*~vu0qBuw_@*?RgG2<)DfUXY97;O*_jp;7yE#!tcW20TH9+7}|5(q(@Ww`*_0?4N zmML27AmL6`mM>5sJYz-`{Gk~&5a@Q52!5~gV-sLAu} z5To|$!A02mx6=ar1H-~~O(SyFa`$bk6)QMP-Q>Dc4`0-{PtP-GFxaX6-!uT$w>tE= z0QUM&S;Dc3j=wzv@Ts*n^D4FwGZTdPp*6f#vbbw-cPF^(;_mM5{@>rdxw^W#s(Uj%Gqu${Jv+P8-Ou;)eNxoP54Ntg zSET5j_et4WS=?s$C*Z~gW7g`z{dH%?76)Rp?i`?52ymhr6_sspNRRpbQdhK@-(~dt zr%PYhOdhSk=XHJ<>nUQ_yOFti1tHsON*Hgg-uXBXG35S=1hwp1JRpaIgxPXxD>Y#_ zRrB~#cmfEa)x2|CG##?-VJi`EVL01AQSe>-YXl9oZf*nay~O9LaMR16x&|!sGIpr! zX?=4m+z(&Ql?SYkvk46y)I=@ysJ2~7;&iv-x80?C$t{j4gZ8CEoP=9T&d*Bo z@$TK$joH+sMbOuxE&7j2mfjHN9lStR9RNcjazh6e){j1NlxEzCD&6P0Wh(dUZxL~dj(%FXR^zY5r!Dd`c4O;Bcmq| zEGe$IUnjRR5<}Q=G4W_4nsXwA}HMz-(tSu-eRGJL1zRR zlK#Beijv+niPG`jFhK=jSQpFdJd8+?3_T|{niH8i-mEWWnl(SpS%Y)I9#@fFwiHp= zAX{Zq9y?k;f@^w(9r4LpspOlIM*oOz+9kP~rc41MO#zITn?*hV3MP?rbo#p^YMFg) zoqc||!C8OfR={1GR$}rrwZN@UtxxMiY4G6_d)}cb;MT$#S&o0eems%@Mum6V&!IN zs&0S=;*hRt$8;%-F(YkA^)*tHL;m)k@F;xo!Q}Id0Z7=-gt}}XmcA#W8(_Dt5@LY< z6U1z7UTI;x%)7&deYEO=sYkj)iTNN1C4A6YUsxI7ahBh$dy*JQ|Yk_vA5(GL0)oJ{w=OE9arV+L7W zj}ljE*hFk00iCzyOL6j5NPx%)mnU&bYBh1oP;V21l3| zbjijs!M){=IUF5#ifb*0AR>-{prpQXRvQyTB_!1D z!TRs3evIT_<~wbl$dBtkX3xGVgZRmzmW1FXo9oDQr@+;UVPB zcKvv`(!YEdL+JHS{cEr=o>Ehjdtnu1jr}@ub6|GmtOemTR&&icHWsf6Qh|dxoYbr^ z{{}SiYeVgmU+YkiH$D#OCsGS#jrivs&aRvi2jx$J7`6;@mQ{!*YGzYA5IclV8sy5l zahJm1vs0$AB%fRh-Wb@(Fh>cDUitF3h0pVcf8aFomQSTpK%Vdsonk)l@l$mc!R&cE zc2F}YZ(1uhc|_0J6Mq`-q|Jm8Br>S;)Ird@{`M#tDYW{ifb1qO&*8c4KBLuSudhQ6H2LtLtB?Fk7hBG{M5SQ zpA7xU&brDX2e4RP_<1e^yW<}M@8%XTX8R6H`bi&vQJE{vYOlb9RVVs}+`%xZdTqX- zT!i(=Vtx9%1!5#&eR9_IXA*8&Mvn6}oWfO3;EQ6u#{mo{t@zZe({*+tI8NYEtixf!M4I0S(hEFICYoD8o;y6}%uxiX+_7>%H#~Qqa=lT^IwfMl3k2 z4hSFD4jWi^QXIpQn!@?2zhn`+v<-D!5n5K+<9yWk&Bl8&wd5a>->SURz0`Sd?q=m* z1*NT1rcYBH{TUxL=1$!@#uN@Y%;7V}S zMq}~C5N4^&F{6syJ>Ao8g!os{x%cyl=;$^qUt@|DRozPzp9->(^F|F<<*)H`o_mAJ z;yJqDsdnG%38giVcZMh*$orJH0=kwa)F*yKLUJLBR!Q+_%ewZO0d>!5Gzt%_!Nf_pc2DPN{Gr`n)d(uF6LtO?DAS%$;mnw|Bf-A?gwYPD#oXHPTN_jIu+;ADnG6q&{0Iarqr z4^*o*!2C#nGyl)9hUL#cJes8@QkPp?>NOsIZL1K(Ty94(ObUk3)~h^Xub)Jcd1&>) zY0}9PZB}qgR^lv^(;k=eU=aUKVtc~`q1ZTI*(y?ff>DOFscO-`vuXC^U7UinW}C2H zfY@0!8ZQpT&;4ItW@1qjz$d^QR&RScw*uh+RtCX5Y3m*r>-Vf&+0YE>0*~$v>vOJM z|lS2{ZF+bK#U&9`nF(ZdrFCiDG^5->G0^q;&`zG$0CK zySn$bg2M$S#WH1;y)^g=l7=WC|30UnLE#v}GSiw*MQ9l0s)9g1FiYpmGWTHzyR?6b z>-1C@8X;gHRJ{o&k5c#ICilU8tyC*d+cIgOU|g*ml1Wu;$^nEwfdz!n%T;^CLrAG`N1z2c_;cfk2OZx=+S? zl9*C3+cDIxmWHLOCN#8-mR0zIjVH6;5~lnIm>jXgYIgG5jynPq4i5Ql&uh1$<9k|YmdiwR@GRADjf89F7}#euKgXyJSJv(D#HWcxhzAw?3Tr-l zfVD3%KSTm3Z2iTsI-+Tr%ttAEP{%f#EycoeHYW5TdbU|TG6c-P)9ZNoR%QYKwbyr$ zD9^?hboxFWE>e6HJP=a!>K=WQi6r`_%BT$!t`v!5YEXh1pTjdp*gD4u_gS?S=R9-He@C5nWZq&#f_jjY!f|EPck=)&61;N(zljf{&rXGWWI3TUprR?yGwJYr6Ag%_{(h zl6e4rKZ~Vrm_z+`lzozVGAkN_kjn6i^B1&ZP7CE)aM{YrY5l*fU@s`By7V{Pvk2{h zwW_IiWenz*+!)*>lD=|1@!_Xe0T`YhP6{XuW@wHVyOZsDacbKOxCM*&GeI) z(X8p8xd`67Cov{QGehRG@+ocO<`q!UwefBi`NxOlYptya*1*d6$lx@-fRPE+$ePpe1eNiYv0&Z$l{Jt0 zha`_uaIOTM$7x^a=P9V@R;i#Hx7*lJ3$})wt6EVi!F4adD@W)f6Y$=bA{X zY?@cQ(6;i4J;|7F`$}q8l=j3$8RvWf65*PldX7oD#v+CypK!rQi=2>Fg ziq|!$!C%c9cp@9Ux-L*>|j=yhfR*0fephlSzfhR`#Lz}=P2%ILP?%hbOk?Nrc>*QvPe z{4!RRA2adv$U}`Bs&H)OR;d`0=r+6V{j|Yvh=@*_jJ7|AT=i(~ZS!w!SNXNR9W~^h z>~*hChod0gwViSYe!^CTj0^vULRRG9Pvj{2u@s&T-#h}|d{EmzV)PWC{0I|UG|Xaj zUXO^MlB8;@xh|MSdZR@kZZGz~W&acy{C3Ui61$;?>vI3h2+E|ao}$A4c2ul1+LRon z{rP>l?NmaWUlG1AX`@_;TM1s?*}zD=v{n1|F@LJ%i#z!gWC0n0offpT9VXvhSZXy9 zv=2Y?>m%x>!o5dXA^q<9@5`B@!gz%n03QG8Z+KGor%?s0%JGLIkZW^_-xm0fuzwjf(1U8^OBhYN-r}%c7*Bmh3(%%O5)8? zCc7&^_(R_Vk5$MvpHBOJR4t^ahZOsKI%SY5zxD=%xmjQw@7v1m4Y>y~&WcP{Ofws?*d#1~!L+>EtXgB_eOHWjwdTC$=v zqB2gPaZp!^L*A2nPs1!aB0q7N~)|kB4i^tvXOscjCoAl zr<&30PE&p5*MKf6%^3{<$yu+dXl(9ODFByMt{4?3t#11Ra;sJs=6|rfJQfM}?x$Ke zrB!W$gW3+uS|%(D^1fd`7DjRMzuR$~eWmRlJ6N2CIc;0P+BKpWwMi!kd&4Xvvdv8HcBJGV4{$TjU$}qZ`gU<}y;@Jnc`m^IAzkYt62v@`d z%_cdpi`^dx9mI!qyD<9^wKch`w(v;tTm;oZfY)|Mr@d9eZ;hFD$YX8!3hEDk2c~e} zLGV6H3JC95F3Rx@HNb6`b@Rubn$l)Otz6gW9sG+Qohyh17_Wr`Ps>L1nT-Cc^tM4- z6Um}48yB<2FBsFcYS5T36bG=#o&Oj=zugr@je31&PTPjV)scOGGDu;ZJq37*UNm;f zTAz6_8bFnw0k?`mD%#P_J(<0>|IpYgi~((W=7HYX{7qoxvWbc}dZ->V!Nu`4<6D#_ z$C?YSIF6rPAk(~nwEzWhkO&G3vsY@XqK%;qB1|{oJ%(cOSJL59jf|_Svig~8HD*-sf zo(#qJk^9vgZxxqDc86M9gX^J9J<(oG%|KMEdpywRfAuO~=hx{6W*-0yL5hB6cL`B0 zEk!28i#*-7Pltqp_aiLLed~0a2Bu+ZW|W(W0uw>D1q!Mr^B^Of*Ufwh-nx5bq|~j6 z82U`$n)-zeLrFa}i&9w=>S6w`_DvL?t9$>IrpLRHm1z8158Tx?d4`Q%QT@z<-ZGj& zg-q7UISb=F1zq>Q%26g3MXx#?gB{+d3CXO?i=X--%o0ZJ#Lmh=7yZP-yYklPeHYYe zmhQNSXrS*iX($alI_CF76PiKd(qj`H%=>hBSGc3fo+Fur3z`f(5Ps2W;&VED=kLsJ zoT`lhg|J}R@dAedSIj?VXL56%W@m4e)k`udh-&Qz{&?W!;c~h-v11ftRj}DCLk`MV2hGbdOOE@;kJ{0bNhE56 zSggzo$z?loFg;?k11_|r7nGNXaLtXQUdT`^DHS89WD|2&qfbA90XSnZ?R72~O3AEx zj>u+1>sw#@XSrr=_4ass5C&IXP_XT}!@KCsU9GEDLJdWiX*66^+x+2%X16a3N!tvk8L$($jVU6JB=5B{Yc zQwPi?+jD8s1-OZI-xIGBq~3n=bH#xdsc?Sm#Yb^fSLXd*35U}019jdSNm%nCe zEN_sU(+SGh?OS^|M}`Rs*XWxeqUgWvTJ~({245)8MTl9^O9G|n(fu|<&2Ck2`uDkX z57V{zlD2w5+CqJ%IQ**`gnd6qkcu@Lb5r5S&T$7e{Xt^S5mWF&ljbq^Mzbk@xpxiU zIkC7$-dGoAIF{BKlmV%mmg2z`4EB(?_0y3BCeU?&#KxlW)p0&5%5kt;{_o+O<`|(7 zu0f5PC)g{oB{a*cB*3uXM zSWqX8kM&OR=LqMm1OgnPh^Pa$dDTUA6;{YF=85?rb@2cr=sQnxXI&@8Q>$tiuNL?c z+YC$ww7|F}#b5(-MBdkW$Z*-s$Mc~)N6Vh9Ksgp^7_51tG{2>MxYeB{ykbFrJ6l44 zOkOoV^3pgI4atEw$L`6#l@BCoCZ@G zvon2TsV5eO`*dY%6}e}8jBmD>Hl2Tb&BlHTc~-4cnBF6b`E+XXuAG$CAJp~Nd3}7E zNTJU<^~6@QYjl*LPp4OSCyR@x3{q9)@uLZQTxOxzg}ER5Y`VxAJ79p} z#**`=o&|AhO{Q5+!i%IW9o~H@SB#iBV2_3! zP&%wc^3*0@Ooylc?CA9auw>pbo1JP?VC~&Jxa3SHy^sW7c9YOvuu7B4BWJTy{LEtG zoK@TOIauL9`CEtyw%7p^?uroif=I@q(v%iUL|G?mZ4=H4Z$^rxp1^EU%a?Hudry9j zH@rr#Bo+sk&s!puuC{uLDZF*>tzKR7Ec}4zvScyt69txz$a&uH6!?S;GuP8YZqB(O zCQ15BkFhDY%`!~!M#=%%D(D2Z5Le3L>|oppEtr z&NDx?nk=h9M*Y2xrN!+>R5UHtlz1JHSuGh9RuNxC$zga(e;d8_MAaj2t1ngD(}Ry} zM^CF|cv^q+uuEm|e@3^l)l*W z6;kX(TCvU6Ui=W^a~65vqjHKKrT`HNkn;}2gMv`tdf;#q<-&<0oBe3i66WX9JW&*W z)do$qX+5(lH;SqcMS`X}htlZ{g%_;e6KHOsEoSPQH7&Ueg9*~Xv{dad)uOwct+>V- zL7Mo#$o}*)*?ZzE2W0GrgD$;$S@$D+=_X0cXO-g4YeD4zsnIx*8?-f9!eo9Cckpp)W)qh>%hCh8*&No9&zIQ&RjW@~`Rh z`^&Uz1GWMrDYm)%8}s9MAmy_iWZV-9{%hoJh>(oCJtyAQ@W)2dDBd_0ARoZyW`^IT zfGu-AR3v00H>iyOoc|l2&AJi3Io2G?@~5f{w~b8LJOQ8Qh{8KTivd74xc@b^o+ZiX z23jjm8W&T)MO75XlaUjyR}}kSn2L#~NYqUM;Xe7`)91zsH38#dYa*1MPja@SID#qP z|9mesRd=4#uW$12w#WBbHqZC^!h}DV&5jEXJ_#pS>y@g@yUrRU7^u~N1#uRSLe~892#tBFP!?K;FHCj*VoeH}A{1D9C@!{P(rI(#P&Y zyMxc;5~&f%3bu0VxN{AIe4#EfQ))27d8DJJUwlCxP1S9S)WUT2Bh zPq-Ot*x4k8R=>KYOJgSeB#54Kd7&bEa{eLP6wMr`=gC#1ND|F_61^gaj<7JsF9sgb z((;NtrS~BtjBlRObbWvBerCMC0SCh# zgf5*384M4J1$xaP-VVp;;Cw!Fc9_w5S+B;tLPsXW^%oodY8yl{i`0Vk8*P7+KS+Yp zWPIJgjF>5o4}PAug8A3|pRVAPe!?>z1!zOc?lt=Sd~kyF3%akjf-IayVw~Wih_-X0 z4T8xFH-}EJ{$Z)whmxYSrk6d0gQP94e%@R@Mk*Z-EpeK1Px;nAXQ*IJ5iYKl*L{r; z!8v{86(l;lNg(9q;+XOYn1lYlqLV$==HH3>qww4bx*cl zI*|xpbG-)#&o}$ofcEDVZ5n1F-UW}yJetM2)L+^z`X=@Q${_^5ojHdtssp?SXZ)-C z14zZ+GNjFJlT@N1w!4WZ?)4sZB%c*cASrVZJAw_(0wV-+q;~ zILT(aX?@*aHbNPFu)9zj`zahn!g)gDn$aDL3d}(=bA0rJ@3yB1$#(HPJ1ZMA75lz$ zD=nK1Z|1l!RNFf%y$041o;>v6TkvOrEbXdZlyh2S3pcWU0r`V!vUxLj%Ye9#T}n5* z@rz$|+i8aeB#^raU~v)hLhxOD`-l6z`q5r9GCa4eVVJP|#cW{YH8ZknriMcMo&R+~ zmZvRO86XIxMeXE*{~zG}ZoK4Ǒ=H=6uGM>Xvl&Q}RM7rRbY0n>~^JJJV%Y##*X zH9h;Zz9asn{I$Tma_ec^T=w<ZU;zhUh`^(0>;t1^A44F%|I=y2^~kJjCUkA@nrZJ50)B% zIDXYLFD~%oo*Eeu+i`SrXULAsK|DqFjLJOlRY2TT;bZq#f-56qm@DjACf`x|d|*J8 zVwrQC&xlKEKZ@&I#9-@>*91h~%zi+kQk}=4&!c5|rR!S}eA-!D#}mYRHux%dd6nlY zjTx~HxP}bt-d9+)^E*^bA(kDPw|~Z0&3Bh?GSe)wRg~+~OH!kN$B=^!6Q}SA zvdjaGg7C&f28#=7MN#3CsYDj=fRoMAPS!b{J@Rny}%Q!u6(uhjx;g`17h5MMlp)FYyG*_FXf{F{4$HDzKUclWv zsZHP7gxZj;G;}U2{xvz3&DhnCT=arTlm##?Umm;h8 zBZ1N<_R7)F+j|4zT-m}ngbN6R=P1{_SrjSHI5O;lg|LY4n2e(d#BLl3xV3zX88gKh z*VhdD?3^b=QCw^D+fj=d`A#Nn=tfi6rE zmx0qP0+d+*4uxKECwHH>dqZ0m-AtqQ!{h-g=i3*Yvrr1+vB=lQYA?+0%@ zE5<^A^sHrXjx5W`3N`oL#W2HRHf|xQh?VsxtMWnjkQ1+x zXNOoB6*x3dYfRa{bk(ncNm{XbWs>5Hg!0QCcY__A{H<4J2ajUeUl<7|aUdkD@h{~r zI`k@QloRgj`(4c6sGbMRs||DI8GJFaf4d@DvB)RvMAhF6G6FW)(_ z!Hli9!#Y-p0$|H3s{QTJb@zW877PXLOOIxFgM_&S`n_g`m+jgYR1;EkL;jX%wcHW! z86VfxzDVcxbX;kM8R3C|Y105cBHTNq{eNDe!b+1jWrTEA@=6+dS0FL|HF~&t<=*z$?I;#+* zgm|MAs<&TM*92QHHLm`*Dqa632vndHSsA#+6e>Y@)HR@M`Yo=^>lwBzTl`!0D34@) zaDBW{;@GMxO(Y*yn!7_)sSWm1TaL90+F10*++O2Nx>D7suVwj9o=#AUMMS69>K5+Cdj<@Kn*X1sY?`eL>u*ORs z;`t6x(72Z2aOhwP#uc}KQOM_PZpGC_2NCxU0Ya#@Ij=R)WUxdt9Dm0I6X1Y_F!&(R z#iZY!7(-zM7&AK{EKlw2$Om0;Gmu94nFMqTVQsC*$H zU%_qRfntd{1e+1#aQ4gNb#guX^wxN)%NX4FDQailMCi2IjuG_-r z3hYtm4t|7J$zufG)u@yf5hC7Il+oy-RU@v7TSmLeJ?aNl80|}{bLtG|*ECP&X4uyJ zScpJ{A%`Mskhz-v>9zT!lEJ3&)9rq)IN9(_`Rb-c;|!n=X%(Sq=EgiG`HOfJnOUVA zig*v@Ql6h102Zwy(z$XD}58k{$_s|36$Z0<01R0yPog!F_ z)PJihQO`KWKeGolJodK~u~=KZVIi^=H5q3kar#a)Iq!~HE>x%NcB3H}R(XVJE`bKR z+h~qII)_9DlEpT2m=6O-_4oLzwCWDtn*8(c@e#0mzV$<1!Zdte#b{<1j1qWRM?&6k z6^FSqw9D*p(TA7&zQ%XXNaXFFU)E2V%e*^YUA#(=QCE2!@qk7&wfeas-#I=sjw1xp z$N6pGqT^`cyK>Qil+%@)H~ojKUDDM7NseD$e~)uSseY4>XM(1o;+=D02F8*|9CLey z*9r(pI<$2vDpU4%kUvUFG6EfXcPb%K?BEn8;JSA>s(vu0LY?uUSd5reUbJT^d{OmB zCIBc8Y$Fv0r%0c2$i_K+ZkeRP?+cpz?!Xp*uoc2-QEW3B@_mitJ^B=_Hz=7m$y|IG zLU{bbi2z;i2$x|7gZb7h5eVp!pQ@6?RjFVE&+q?BFXi|x=)oYOvGUjkD zXW%ej4P-`J9~q?izGY2idFPPp2ViPe-C(sob?8ZfOY*`G3k+6?I;{a zTQx&rM}$_#ziPV=<(spLUc_gy-0<4Pt|tPg7i{rGa~TpU1)odcwm4P}y6D39Gfxt$ zXiwQK5(ws%S(kgvgS%4-2e7kJtVH~~+^IX*yeaS2y}Bu?1;moG+vFLdOar;`AUfzv zpN~*1;RmlGX!h4U1<8QF7wMf&iVRk$=YwC^SSwTh{Q5``w)gJv8J(sor10WP^Qtla zn4Mf4j5YXd9!~{ongYV?%T&B&p5QTyPaf*lMw@<21B3RfWmU@oA-w)B4xxRzSVHVv zPbdDP1nr9t&XS`c5#|}J+@hStEy}gsI^c;F#zu7y2a`~L>)#Ka%wM8*q=vPy%3~oD z2{yD_Z5lGDG%tj6?L&7m^9~a<`?Y^w_=%+Wvz*J*p9%|&<@mp=?3rQr=V+vUA)Ii= zJh|Uh8YGCT-n$95K5gMl{7Yb_TZG$tOI-w1rh$^lPU6D>4>aVtzuHgaPoiY0h`XrQ zNkA>StE%U`r%qu5Xie=_6o6@|pEt)7R1tguJ%so%fE9yh`%aAm3b`3d>~yjJ;S-_-)7mJO0HE%CUCyC3|OSGQ%vk>IyXMF7)RNsm~^(6OyVA(scC2`r)=k8FgG&sySUj!&&hV{D*xXk72V|w@cnnt=8Z*^2 zuyDRKSNtQQU=Uu9C^OCtBo=P%_@$ zpi`;np8udV?{JQgJbAZZ2G{p>0N7x zZGo4YS&lDOj&No?sOpbN7rMU4GGeZKpE@;4=Q_`f0E|FS**|3JJf zY;6B$65I-}lTwraAl^5OyU%#R|NxMuX?Q2*iyJe zo(>ao1h8p7Iezc25JLEo<6-nb&*vjq;I&_p@pfm~=k0Cd<53P=@S6SKZkEp-gO?$~ zm9GWTK!}{hQC~c^%q0<))P{lM9@Eo~C-=ZeP`3zfME7H-W6HHfe-pugC~W+1xo8SL z_}c_%x##%q-N)e?Pk?v$NBw@Dq#QNT^VTobcS=~Lh_3ke9{seJFGSJqe=@4(`MHwy zuh)Bvg4OBP1M;}&TC{*L$pRz;PLY_{kwskPErdqSC`tls{rXX&%=7KEKlDf@c^CF& zI}D%mH{%)#Nv2*Ch%_$(Hm;aW#SQOgyd{S;6JY9XbCe?!Ha_DNoPh z-!glP-_DdW{H7X^IrL%fu9hNH!pD!1o)a zlaTu22F)6-Ul?lRHFH%VqzyeGhIJYE$cfw3g+7khxZ%CkIcQ(<_N>Pp5su=8ZGLFw zJ00d17aHe_u&NEH?J=Dxc`=c*nP^rlk1H`l3T0VP8Zpip?^3^d!3?$hxQSd0VK5~R z>vMA0zUkL3MX3vVH~A9|_tYl%*k)I8L+MQkQaz;;Kt?DO!Fc1ecAe2IPP z)Vz=TZn?edqs_=mUi5P z0{qN~{^O2*r?-}xa^(A{XC^m84EdglH5G|Mtwk;Rn(9Gc$`@udw?V=Dr;tf;>Q)3VkOlfNgxE&b9 zimqcvZnpQF&>nU5ws)w0Arxd8rf?VN){OXFn<_o_W4yaJkH;VoZ)@0|Qea@4_KTa- z)n9Qpy6@|n;Q2II&6$`o>JXA{tRB^2S{rX*P$M3{9=K9j6}@^d}Yrq|KNB1Tw%#pzr-O= z3~b+^li?3p*1}9}S_beqe*+y|9q;~T4ZqmG?b>xY!gvB3gzg@faj6!V4BBM(d*oHj zw8;Vj^v0AiSk}l=SNDuFQ4{fN)}`uQ^hqnWiZ046*LUqcH0KPdSKk{v+zKSJRLuv$ zTSaQ&tg6mM1IBSuiq8)3v+~n;WF2W`NvuUOM!&EHSu}Wn<*$0WZNFg2+GMV0hI9B_ zN*)@iW0vtE3ip1eKN8>`BnE;l1iyL{#~;r6?*aAu{7xC6AN+>5k5gP!qrVb*gZ2%m z20f7a?Zl4gh9Pqrc)`@b#IWamNY{Syhkc&}JQcvK8ytsn0L0 z%Jv7s2>l|INZ>pvi5(SM*#ve8;|U&MWX8}?i@0>|lRF&j{VqBKEj~wYW+FkO{Zg_J z84{D-?+D%fBZA5?ja@eF(Xcq~K7i~^#Xe5Z9icw%9&BxoZJb=@tuLQKXpSso+p-UWyEKPsZSMtT$5q?k5iVhQb{42Kw@w8B1#L| z=V*PzT!qx-%|?6AoRF1j&?$r=3NDV)@{S7G2>w@{i^V;j3eMP+o9iJ%*hvylyP+E= z{2ukKyrGNU=Y%@lDAqhsT}CCD1aFRi1wiUy$=sv{W(^_B)t`j+$t91^I6gF{nfe(b zLI&frxkYG50h;vsg?jEFIDtYyi16Eb5SCi~7F}c&jz>fh*f?M37O1#Q&GR~+@;WA; zQqnyfcIBvRi#>4_Q!-`5DhrDHMs@X-@SJLEe=R#HxJ^042Y@T@Z>(Jkz~<+Z#~Ejf zHy#q@W}|5$mC8Fh0vCRD?A902|Lm9URyyjeP^=f+8E5w_KU~KCbS7-W^!nrFK8Fz7 z;DhX|WpflBp?7?QmO`-j;s=zG0TC}#&xF4ZMvgFY2k9$Yu=u0=&>6QylDncvwA>(m zIFOWkLX+V@y(dJ7K?Rj!Jm_`Mzqs75C@%G&Rr30b5J%eYmcnepad53|;zvK!Lm?7K zB2K6lGp^AI3UODv={xM2e)Ae(j6`C(B>0Tzs1BX9a(~TF{1I0oYKLjtGHU>rKf&+o zLm__f1Zw3h;Lp6`7fx^m;SJquhX=WqjoV^r^l72PxXlV)-(qppUhxRGgmer(LBld&1bjEzvh@zccVbH#YL6mOwQ5IOJyDc#+|z9zh$;-Ag zXO!C?D|>%7Wme}TJo;@=ZayrE;%%~YRBood$-=s-m!nj2Yn`RN>&>vBxe&;uQ(4ZS z*V}K%G>x2B6TD(wt=onlQK!rBu!4=4lMToN(80ad>o0%9`2&$Dgw;y?Y7JIjr1}UNH`R{dK5xf63F%q_fj<9TvP&@NC3$= z!Mj%$D8I9HIk&UHL{hf_K%o0psFz2PpV7m4BFPnKlQVv>i0hM=$5yBhw-416>h;xL z5ZV1vCTwu~{PW1S4_MBD-6snqUpYqn8oARDG`=AsJzY+20cuwJQ^D|kydB@(t2!2U zU>funl5|F3-G7a|5*yzH*#$wWaH3ALi-eA<_6ku2=Fos^?}CM8orsq{sifVVfG>_xkrpZN1MHq_kSIYS7!dN36|7A z)7~PisK=t8w;N9uG1-Ah?pslBEKCrtp ziYR_M!-G6g^daS=D;+1a|4sc4ZSc9wgBMKNJo zSWyfW02}-Ret-`9*J7zNrXDts%}3O3mJFVwmnHtshL%L<)w{N{#&2n(@uA2OPm19~ z{NMsY6HS+>!CxNCf9`Oo72PMkfxv&9y_E>+y2#EA^YS%T`zg|hqC$!&O*&~CW>^l+PRf#40q^q-)giIgGc)7iJ6ZuCk!4PohdKj8a>#CieVn-%*W)NN5umsG?65=ROgS_ue`*p6 zhU!8FFqNIF+Tg)WfZ&5foT$$6#C8|s|4mKHH={-l_4>&p^}j`u1JTM& z1s?Ya7&q-$VaWj2Ln+R}?ngIjTPqn%{`*o!iMBH`nv`zYV-~ioAv2C1`W@c{H=<3R zI_y|PFTZA{r^EKPv>2>s;;A^Y9|TwRV1dUk#>Vzm$4Z_IF7^#FDc@pT$=HeQNM-{T zeMtvl_wF0i)15ef*Kk;sk}Xwtz}Ug_dccV`#(rQxI0B5Qg92E;=sVTrkwG*R9O@tZ zN%lHJX$~ah06Imwju!y*b((UmTj2H99ZZXK0myU^>ZI_9 zl%d#RW2qXRRoS11dK{3%++2S08hF^0{w+N1Q?Fglso?hMZqaSfJRa@w>isq>U=g%? zm@xjJZE4GtJAKtU3)D9r?5(FEfFXPfj2x`q5FojcBaB9cdcJz>Y9owZsV^SPOW}TT zd#$i&i5h&JG20nv@?f_;smKK3g+g|ij+crjZUF4Zq|X^3)W`#`FQ38nPN6}(cwx@4 zWO%1rujLfOVf8DAD;6;_QH#9cfxOn@n%d&A9is|aSom59-sqC|PJ|qxuzbdH6Lmvf z!uR!~XDO!MIsU8D-@-YbdXTpz$TtqRW>y`*yQcJ5BR8J|z9(xDQp(=Ur^;C*dn4o;-#m>QGARepT%7>O?48a*+b_#3iY^aHDctFMH8rq)?QwxB$s z9mL2#VSMg{(kplSLshfU@1TQtRsS)Fe2Mwyvy7h7?D{w&lb5W5C&zi?E%n&00c2(Y z;mZ%qMLyH;S}({|>Q=s+?l*Qov*dc?qaN_?L`6<1oeEPPa0pI!OS^sj7%49y_;?W< z4NizPY~W2Aht3POqEDgKe(5UST1E!pRzgy^+MmJpRsn_rQvC=6oL2{35{_+t`Z((- zpk8;{MUG>>33{Fuc2^JDrnq{ZZfQ0RcJQ&x5WZxlww%^*u?bkm@y+yFRY^D`DST>{ zgtg;G7XRf9<)329d7s4O_sztVspre}6?6QfvKvC)n@ex++B2GO)BEt5^+2hPupCd?J})v`g^KB|uueG$oTwm2S4MX6)#6Z*TdBC$qSc*rR>mFZ<{ zxb4UGv8d0teIGc@ljE0VHb0{(~U%P zxw6@V51;W~5D4+w;dYns^8FNqo43A$;zvta zW*7LH>W`6-c{3v^QFq7T#X+}HDAv%|VqbT}=rmuj zD{z1U1|oH4eO6p!SQgIY?Ps}F6*q^|(ff*~4N&8z8D*I`>^s^v`F>V|I0>uzZurh3 z3om{L6$)DWZp|=6F9_PWx5O+)ebkLLZqmAcA>>g?K26>&6(C;>O22zWt+=z0tXzO!`b1r0T4m%lIV^lW;)IqRFBfz4y zA@Z?H>|;AH_!@iv_O-g#e&)M7I@IdKzQ-hujK*~7lPEL0Z=)u|aU6o0exo)ktbYkR zG~Ba#P8EA7+=(1qz zd~A0fb2gMS3M-x++n{d;_8~A9G_{Gi8(_MGO<$Ru?6e+Vb+SjUb5*$6r#?MR8DWw( zPjew>S=gaV9V>bO8L(%zm&lQ1Njy1HVOv>K@sBzomGkHlAX_(dxwM+2etKEqj=AC= zsZDbpFgTjf(!cxthp${&(P;d|YvY*hpL-T(Vle->tS?b6x?Sn_c>k9w5(6dtAEOpf0cY%1W5l`f<~E z*o(Oe3T
60T{U6Tlg?C=PMUPDdFTobF z;tSCfKc;(V5VmS6I8u@i=n8G#x(-R+ki}Rp5l&{;^+k5*oBo`EvZ_v{v29TwAI?U( z#s$pu`B@K}rclRDY;M_Ej&17IVP4pun@sm>=?`YD);K(u3SFzlN18>0euZY6A5$#tCaDZL$C8loX3`RSzCe|3xPtzoF+A zXt!TxW8{E{?h?oYqoA4MkQxljSI=V_S0*TTbwZtjEVet4m|9g>mQ5-C8|#%)TtDFZ z4+;$@B(Zgg-dwW+q(-Bx> zZz|nvhMYbcS(2lUWe4`=I!bCC%6PD;K35hsGL&c4dxJgZ@`$0+uD92SoJ6N+4!T!K zs+x8X8O)xv=wQINecKg^L=4FgWgjnyI?W=sXo)AC|4^?&WIjgX!XQj$`4J;ATL;n0 z)!wD8vE(ko4mVU-KOu|FzLTsv^$h92BQqt)D&DZ`G-Je#yuf{y4a#Q_b<~L@cOlRH zAI9D?Dvl?38%;ux-~?N2ad)=`5H#jN6D~TP9H*g=bAMN%5d`#;aiX zpPSCuIXA=9kGx!mD2kqL4-!6Oy<6OlEwW<)I#4Bw+MqOb^kxTHX_Ur(9(Tm_2pLEZ zgz-ic?QQlEWzD1S`HN9f=*hbLG8r%gKWt&11HfOW`;EYwNHtJj`>&}BF0;pW9Nk)N zN4JfOIIp!DOYxjDdf^t;m}2EkwsF(XMYQwfda4HuPWh(lgyCDVuGaQ*dE;_zxX~vm zk*Ks5Y#N^{34%8>EBmE>9?$_F)=7h8Dq_YY_OlxZ5D!5ObT(O&N3BoS3tRUczbT6y zw{Uj-iZjcLUMX6W<7au75szBi;(Vh?v9`Qcx7 zn+0xw_lI57UZ`DwEsnefyLNOuGAw&e!H8xC2WIpJXnO(q_)`mpw65bmKCzb}SnNLK zdg#O2o{T1ghR+ZZbyk8+a>u_eM_5_QEQVpe7HG}K4al)~aR$2R3VGRXyY51wzKV82 zJA=@Q!5j?1rmPA&<`B6n6Yt_FZtkzzJd^0d{?H3NycB|MP6|D*AS+HF!qjhz6arfE zxS{A&$-4p)t&u*>igICHh{rx~xqYjQQO*GO=fT$d7>B zFRs>E>5`dc`*bRqnJQ$$4jS-Cd&#^$^Aen6A*k3Q#JE(B_2Y@;y0;2*|rnb z>j|<_Y3IK*s^o4EE3$zSpJy!rYhHt=V?I|H+3Xb11T@eKg2QJk*J?AKIhM;Gbv6VqA8IY*# z%?+yrRLi!O%tR$^yEfL$Ko7+>YLY&C+Zs>4$|dE&A-92_g!{*gz!&*oKN<69=$W5t z%_Q}i!&kFROwbiHr=H2s^{vE$g?iUPcZPJQI#=`{&s9Sl;|R)8&*eb1XmEaBI&@$t zt5JV)3ctG=CIGar?!qRiQao&ro>0DrZjmX!(!d#8KWRk|?s<{@vN&@R`R}JaZW2Zp zYE^S^>xbLY0#3sT-biSOr?i4`z39w6g}wjYu)-6N1(T=AMbV)2r?yk!J9c2=%ItTs zx~HPFCyNyZwSCr3M;$zG0kf=onI1cp)73Os9QLyjxF_=CW#afNTekV{*IA_wO8aTq zrkRORSjU9lM?i#k8b0mOYzEMI-TGwGG@ONmA2`LkaFnQf<2;sCa^aomo)&HN=QO{3_vthJZsF%Do2jD55*seVvsSvA0(ZS> z2gPycPjfFUSTu9wLn1!1o4G}!$Sp?gvJhd@#QyhGS|oKs))LT<+R{r_1Dj^*e1Lj5 z@+mJY#U2N&U*$(6(rp(qy7Uo8861svg%R`v;jmSbUvudAqmKl&K9l0RNJpllL?DCv z!67)}<`bkMY2K`zli=(%oWGwqD_DMCGf%=;kCo`=kcjM~j8=-$^??9y*r!|8FyRd? zLh}gx%2p0NokH;K&3uE0IO0?B5s%^2EOmE|paB`7D@OE#^EO9b>EY3m*$Cyg`9A8& zzBDQ~){fF!Sa52zumf09WNNlSzDTgG%qCgsx>$zT$pryxYi2t8yA|+O_7;RxfF&V{ z0Hc1vo#|2h`HdXS>}X9?iOYKHss&j}rfEi6*c&NFdIk!?9H!2G$=l42`(I zNYvl195=9*^~sqlz)esueT@8GxneZJ{=IvqX?>0_knLlzqN`dKEP$|61)b9e7ox>>j0Ry)8)o8 z{MZs%yj{>aMe!%uflI^s%^9mN#g~i2 zxcbCp8e`kqH*wUooo1Qfd<}I_aAG>|99hIP{(|KO^phkT?p0weBjJbI>9G%Zhf4Om z&TEZHmKi4k#Id=7wngT0z0)1Al(gs5hDhVH@9U?+=zZsRWF#6?R;6%(nx@CBfgMMl zAaa_usl#dc%plMV{N6w|yNWuwE&nV{+9PE|I^;K+bMq1OMv=R;Ub?3mivlWw!%u<=csM+3E_Kmi>6BE8OfUYPWzz+sDZ|bES=A~oZ5sbdfAgg<=PDT7 z(dUi;(4-)bAP2Ll#amz%^f}YDa2kY)CL?c_xpz|a$)AnHl3oPijgZIzKZg#}v3m~# z3;Id>?%vA4U$NN~^o8*au-c9&)vKc`mUQGYkxr@^q09-N)W zi1cj?X_{Kyj!8a%`oxKeo)&;aB+sB)r$LVdy=iYl*%2_w-aN9z1~I+4$uQ=c}&H5VO=FxA8+;2j@Ul$N(J@G@kV(9R7w6j`UAhsY$188Chz z5qTN0(vb77M0d(6w~@~ThV=2cs`UyxQxO3-@1BnF+4U}?GGs-^sT^vDA93s-+-S;d ze@}k6$^bek6bhdeg(1C@;yx=<1ufM*z4-Tmd6)hbIzRrL1qxt#J=h~ez0EvhV|w}O z>Uj(FF>Ee7@zG`ECy9{VzL4)E2ROw9*H-RgfLz}ww|E;iQ)~?R4BZFG>_BE_yOwh3 zgAi zPNcxv+~<^UPJR%4T!a?z9G1z{(DX7PnM)-Iu-dZ0XDZ~0RMQ*FLb+M$nv+!rrYv|R7tkbS!-Q*0fI>QOc z;s(RosCORym+g4XKLrUMHN9E1NVh_(OxXz6c*j$%FjO{_Om4Hvce_n^>ld*Y6QnM9 z2q3YDXY~b~iza4Q?3rS;S^YMs6;`#pXhLLhI7ZP@QJSNZg)M=~Sbuw9br}Z%+h9qU zQ5^ZaeDz#j*T1My9_p_f`WYqQaQa4T9xe8ryn9zBg2#5+P)%X{a6ov1Ch`1M>!`o} za6`IRvMV%r#AjMk#?^Xa5?TTOJ90m~r94ygSnAAOB8unPIk}&nz01$4UvIBGv@TOEuwiXdwyO?(SY=^8 zwRTbIIZs+H;@j+5mL5KxKJ>E_8fYC`7@xii zINLvC{zUTR6{~ml+Opab8NOj+#PJU6uwcxfyV07;DvgQ_TTBkNL5zl^e?Ps`l zO0NQfo+hs)m5(%9ohCh;ZO1Zs(4pTsia;%%84wI@PdsjNJ$i4H8gG9YISG5^j8+2B zkLZ{uldw*@rQw#FAyJ=aT9uEv;{8!7y7u3WJrAGq+EtDI@-xD;ftQ4)!eay@&Y?-k z!RcSKTPXnv+cFqiJXw5(W5@L4wSb#7we@$7_n6d6oom2!R0N0eTf;0tOG5HOnY<ngaYl7>D z_=ek_Y2;1Xs1@76-^`N^wF$Q=VO1_vuRlA6S${9`23Gki8vKz5QR^yuA-wr_LRudU zq;u+0=m|kI`-_8YAR3-nLE5#SleP^v`~-IAMS2q;4yC!%p#u5|1}u_Dd6IfAn=tUmuB>R7)WboQUMt1qMWGP1Qm}x-@mpla86ge$9C|dBb23 z1r+s2XUWX_!-WhEV;0Fg^J=NY?wBR~m`P9as7T>?`GB%8fV_~HRU$QUyO>h=*pD^@ z#-p^Lq`CBMlF_Z#5aF1-sfatg>r5ga76Cryn7pgNy*_vrEk$f3dS6(D*c$$LXnSze z1Fu*q3{15|mqscdV7T9RPsW;?dKobcX9@i-9fw5;?i&XD?NV8o!}r>`)>*tZmMMnE(j)}Hc1j%P;#5JW7NMVstio^A*8P7)k*I};Q~yq0&Hl~ zq{mE5wQ%TIiueQT_6G-(mY6NEmtrz$n#tM2Xe|>c{bf;5++XR8&omH^2O1dp@?{7` zD51#*F;qXi_xUVqI5bZC_Q`H71sc#^v4u zCbjdrIcYEPQCL1;T0t6^sR!SNqJyXf0J&St0zjNlH;wEWm-{?Qydxm?rO}%PVO&-l z{@MS@_ffX*CFjBCK{iNT>atCkR)UW?NPesCH;=P(_WS$(uPK7NkJbU4BGlxp*}^iS zrQz)r>MXxwH^aC9snO~d-H8o&7qozg8~y<{kGr!xh_NmVOXot%1O@p?aV{=Z<1> zf4m%aZ@p&!kq3`Zr!)wbuW7cNBwSUA1!iptV4JD`WmNe&c-5`yTBHAU6DanmG z+nd5@0R-hqw>gWoiqUFuSjRv;$ACspG0Hl2iuwG$%(rP~daw%wt}o z32r<<8Mo!wqD@3kA;1JtSq$mvOHK5U?Q&I0+0cGvjVD5B!}^swULC@vSFbXBmLjHw zg6mjM=6wyK2PFo|JVcm%Qc{z_92&cquPVRn$$|=WS`yA@jb!3m5k704Am#`n1o{jbm9i8yiT!m)dNO0-(a8 zb{gGc4u)S`CR$B0s{&TEDE^HHxl|AnLzIUny$PXYq0#NS4Swa}#)%3LVJtPfuhF8h{Ns3^=VD zfvW?`Uj__KI}`rRXr_JP&cyYD4g{3QT6_zeQyQZoF=jE^ZBw=Y!vlVex`>RUKHeC& zw^OL`ZG2#1mrYayzw?7#kT@B<&uS_!1?CO@@OyD}IIH>?S_M!Plyvn6U+-y#g4ruH zkB}!nL|euskiQm+{Rtf;3@Xvj$htr|T`4Xj9y{ zrm--IvfMCkZzddJNtA9LG=&L^;bl@;B3(jTnM);SN?L-^zGdnX&D`#xCHqI>9=1!5J^)GoLUrHLDI7?N21Rx4opORgNB~B zEe>`-zJ2jp1EcQsD&=PYxs=m`1=3}|;N?@hzi!I>P32jT-$>ygb1UgOcZm#+B>Hl4D-LO8m=B#K`s5>%*2M-ea z`&li5Hrug?HnVUH68tAlR@jq=aMegMyH&_wysmV%u^n9)0QP)cl?e%5ZgqQMnQ9^06R&ga_?(?%|z?&d_UE&@pmXmoDgYT zh9YN=N%(dJ8^ptvfcV^B3p}7oriM;1TCJdr4Ha6dp>HsrNLX^LXDPAT4`;w`i+>Tpzdfz^P8z4BA2t`8uN(`(zOQWG$QcOrfco)PCKSb zXlM-McNi+>d7Twijz6`TsyAv&DKkgz8R~nwfmMdNxbE45UC_J0@L%39L~dY1fh<{? z@nVRQavmd#LQP08-fNL(A-IJ(>6zty&0cVM|M&3<7HU*L*)rVK?)MLCAJy<2Y48$G zz@1*2J8pzNM#{&#$d=V0?%bOvh;F3)BNbstTY^@WxGJkP{BtxER8=(a2%LXHc7IiS%ni}bj^amXLlFPH-`qCIw0@Z_PmTCg~Z;{*mb&FloJBi zhb?ECU?JME!-u>N-NNaXTt<_S{iv_gW+xu)o&3OPZdWuaJ~BnR@)~J|DtDJGoFWoW zpC`?OT|MS$x4dNjQRS1>IE)_Kcm32)b$y9d#nTb|Ptph8eP{epFA#*M%y7U1zU-Ef}vebcjDo_34E>_<|}^T+SkpquGR+00JXZxWUx>Lw+) zI&h|D7dz72L6yr&RFqJ4G(U$XNHubh{~@I6+eoe~7qRE{9D1I5kH(MWJaw99^Q<93 zSgHkwx7P-d3jk=%xN~c+QLYzO9d*8$z@fq-ZqjQj2MCiYMcdaLf%P$8<@0AzjxxL- z9PIu!aOf66td)jo(3!UXacJIxZVTx;aTmBAVCu&A_s^aQcAV4JO4g;~tT)Jo&%|b^ zmSs4OW$EDC$LiPp zpNswMe@`}lcmMo-Z4!H(eSNzBN5{h$a8vepHQc8A>*ySTzjH<)_MaYtJg35d{>rOr zu-~46WzPD)yXEY^j(@!W-ZuUDx3|uH`SwrL&867SXY#T88!>qFLowgy;dP#i1U^T? zqXwVT_l&G=C@rOG_iD%z;gt^8O zkoN!XAHa``t=^#u9n-&knCODVeY7&_h5$`8T)zZRJB`fNAgjm?{$yDL(xYz?Zv zJ-!$H|8OCLG(DB1Hb+#!l)yPtrH%>bUM$tB6!pWs85_ZucC5;mdPKz3sq)!(FnQXpGiBM$DJE9JchSblL4yx8e7l=aqj z7=#C~@k(5&UR zNdojG5Zc3X`H~9bnOZ%YNgM$UxY8s7l{#R2N@G0gBZX^&1*-4J==wFIpx?KYs|vRU z3pjMSYHmK!f8uMBbgDnNUGeoMAx!=dq%tC)391l+TxP*0tW#S%-Yd&^iI@hbI|n8} zLj#$iaRL?s=W=2| zEsxubMV;%|W?*l{2&{OmKC6OLJ1JgmN*!kl&MuWwcIHv9B=`|D3!M7G)nxyL2L!>n zqmj%{P~G{r!LcK81FW#3{YXy!8XCNu5lh;T)-G5j+4ogQExZfi(rQvSEaA`*omNvm z4Ry4m|LK>E$ly00Dg&_xX~QzM3wET&wGDN{GV9tp#KzJ_hsG}TSwJ&lqbtb6@Hl7^ zK}T(xNC*Bnyc4mJEv3s6xUXZ~-@Zd6+kF!3@|`!6^PP?=|2oH(w?qJS7SFZIulEfN zlmSdw5`Bwq-60!4!;84iJaywvXqPmD7yx(a=k_82A8S|2NLa347G9pn!6bm|);{tB3#f}&OD22l_EP#_|x%ja=!19gVFhUUdFQVKETgN( zeZy#}r{!n;a#}Jd4IBYORc-a;$rf58+kd}>l_rIU5=)irPkTlEjR(OH9K3&OKr6CESLn} z);pzKwiEEyH$Nv1Q|t1ux~iq19h#5Z;wW(u+`PwmODa^W=eU=a9Mokg%DBfZ6GPAK z>5=UZI`vZ_kC?9{PDsr-<`K6gM8DSrWqL+J2R?nJ?u^Qjn{VFRJW+<-1U*7vF-B=# zoc-3O3ir7S_JBs!gZ1^#_&zKEC^uk=cQHfyf-@lkKej@^gj;WU1q$mV6L*6-g5L6F z@s~hZ(TQS^iE)sqoc9p3hCVxs8Su84>~ZxVX#CT4wQoosrkH4vpts2Cn-IZkYCJaM zDCvf@>4~cE5eTBwAyI3hpn0*|b2XT{(_x;-fy8kulEo)e4hhT-LyM<*_)i zsko#Y6ng(Vr;sAn?$lRsp>HbP*gJEbtv8QKL#Uh3yd~*RTSCPzo5*{uWe zdQHIW%@J=!$r%3h5Py5O{_D))V4%V*PL#{}8WZ-4eU2s=*(~|TsJXBJmI12Xzyf9P zng&Q>cW!U$zp1JEZb@dsGNr&qYS~laY+dE;AA)9bWhZysKk|EhOXFyhF9Bt3y4;c9 z4qb{*7I>fA_jNShI+{LSYopua9+jHFmZPW%-+T58W@Y&gxXg&sUF;m=))VSLNqvTp zrDV;R*EoIG^V0%R$V~XcdnVgdRW9pbEE4E8BZbI;+`PAbJFuPN#zwT$f_Sf0IR5a7aOOXKo!<~N}1 zE?V@+HP2RT$MSy1HdBSYq|C9}z_U6|0xhO#-MM6DbM4Alh(&b~_9!*7vO%RRpREyi zhxLRT+!fw5cCp#93io1q#4d9RO*0qWxz4D6^(o}{+aE(W=#SAt`> zS}QB@nK}r*$MzEI@nKa>S#^Bl#j29Slyi8sVtM`8BNpr6 zCA(<>qSZPaqz=?(-Yk1TuCv8KEj``*#8IUW@Im_ zAnP5jrf~JZgPfy`#r?mKZZ29!+Yt*z7Q|=D|HA)*l4j?A882aT+wC9`UZCAJmbI$m zhy*0LIF(9+4bts7K!hUA_xAjTPd}OndF^>ogU&)LcrFdcv|bO-%IW_1FIgmm=lc6N_LkgjyAk-k z*apzWg#3Q%%O97)N7tpmoj=3G3#JZh8*=O~C>}?YfmE?LcRLId3+k_wDnF%BnhwJy zeXcU#R;Ajb@1ooD?3kyD<(Wp)+l`T69mg@|#VYFpu}XJIs=k_qtP?mrq6AWPN@pfO zjU0&Hi3uh*Xm*aJ;iDCKTWo9FCXPm#{ck8S^XDw-=l9dKUDQ&wp zH(rG5OC_RsQ#UdMJ=pTH}r~@V!TgKw?4o~*MrfTH{S^)?H%D1FgI;!6k*|qD6 zaNQx4UK5HvnpGK_Wsjy68f-g+Srk-9;&;%9Y~o=luU;2Gr!MOU5ZBP8I_X?a@Xb(h z$=t1+lW(9AYYJ$Z+f6Z~U2cD*BM^0kC}4KJ{XOxZu!{)VPFbTW=XEps7R4>8y!z3{ zLF!m1*maE8;2};;=tqmMV1!L{nud>cii)Sq3@lM%uf|WH4}T2BSbx6jJdYN_6Jc^{ z9L;Su3`qPbjK>e${F+mZ+*D680>iM~;lQE{;*vn;7a<>te&u*&yQtE+h#wWR?l$-i zpnE~Qw(VYTpHu{QVv;L7I9!^iUc?KE*}~ROJT0RofNP8eH`4qJG4}VK^iN6?{ml1*JuV#& z?LusE&P@kjLWraPijzbzH2iJ=yawLzDvDN#O2O5dD4i7!nLx0|>vz&x$+-$$gG{mJ z@{Guze4rKUF(cl;lk+06YqIFl5o(Uy(7R;WUUX*I=j6v3%kq&sihHIdGn`{k01+SS zSGC8ku=~qPL!pX?p*u`=YN`%SRn(i7d}BsufG7b=ajbjJH4&{mYrXL)RADW>sxsZB z$nC!OyO`;2%!j3{u|f>G3fs2`%8_Me(NLiF-H73Ppyv8+y-yM>Co8c<%nJLs{N<+z zx=eLuJF{p^mcOR6S9S)d^GH^c`2BSdPO@+apBWO&r1~wrBlqNl7VnuIKQ=HYN|r;} zH#nBjHL-`}iE^qh!+` zXD@LUqwc=eRB>I|F<{2WoM>W!x`U+O8Dp-}5Ve$uWKU6l#0~|1N6=_wo*!GL!Ue5fz8Qqj`wZ^(s5Z@@)dR1Z~tnQnJSPymgO&oRr17~EXYg~AYgqY@)1l9|DjyK zBWK^=Zu*NnSiNpOFAfkTDecn)6Sk!GFJvwj{-Uyfomo&^^$bt0={>L1{W}y@40hiw z7v^Pt);5nBBj;JPdYlo>J7C;Z+p_JVP=u@UYhreUhDzLK-wlfeXO!Ri@2<=7njs|p zH07whG}9Kr#_;me6v1}$s-+%xN?<-TG1EokbGLW7CbSAmEg=W8-#1%w@#1Abv>r`j zB!aY^inMf!ogb>ApAKon(LV zd9F5^rFTa`m4&~cqt~z`a>a?(#omLHy=Pj9FZ^#*B^nPGLag_>Es(*o_E3;G3qa^- zq^2mv|L%dBw6dy4_FMV>Bm2j{A6FKxUkL4}Wx3Lt>;kyhekzgp>Ez0Oi^xq!E#0^) z3Q+xg8UBHW%lrp5?(;x}DF0PjvJ?JSV4!w%AQ3?q+85Pt;9QoN1; z5blln;NayQRb$N;x>9ZIj|JX z+Bri!W$B5Hupp*!C5tc460nY>Sp9{xFG^^`Hj9O{VP!8j+7W8!KAx;XgO((m9RxnJ zll;bBtU?~|P_KjLjU(_^ZnXJ$R%$*!;v?Yv#fhBJtDo~05Q7m<5gH4K<%`=ua4Kkr ztOe{Hk?(=|;_~abZ|9DSCzdWkJP@|Rob17p4xzPMC4DRDIz(7-%gL~^N>PkHYd7}3 zkGmRx3h|Y(hPD7m={t$a??p5qq#5AZW+RjwyT-H1i=gb)W`opQ z{FZd^?eYksg2Aj|pJwqY$*ds~@+NTB#l>kH+sYxaM#l9Aj7(qP6k*H^&(i1DXL|l9 znP!{Ay)tjM2H$6>Fh3tfDh$(AnpRpFHcyDZ{;$hCw5j9uK zdei)jY|4dgjh8V$f*96J`r{5zf+ax?K!lb@wf*16@;5apQgTl<0ipzTvR^DHza^W; z#&yPqOW+VfjD+0qEICu6MJT`OP%IcOFRO@0*ZQAHFLtR6PRP7tewKDDZ{2+1VM4nI zP2_q*^gqaT3d-T5Byy(~ZniL`KX*QhUmOJ_}-ArXwDT zuHjHg3UYXj9#N6$i``l#SHg!umI3VKKG;7(C zA-oVt+E8u{`|mb~0b^M!^iM{DAmJ;$3zq}cO~wex9w)tvNp@O1++diN{`yCGs$gn7 zzgT))i<1nz0d+liQdsrh%hp=6d`a5szvYkg)PFnNm6gk_3wUTH!a1)}>>g3|;ZoN` zr(`uE4h^luO7%1N0HU?k&B}Y!)Xlnd&D7vdMQfS#?%^epL2Io<(HS*$+QtnccYv8% zE6ci>nhuZO%@cOlNxb&vOEm`f* zAKu#`+-SVCC$}CEKcg%Ee_!&Y&BA*MT!lAWyk@5Mlm<_bM&YCC)Q1nZQ~$q`F1%Eu z3ordo!dX#PK9sW$pVo0Ayjy=Q_^Wj7L;t%U8((&}`&!@+;RBovf)kKy&o@!+L+FCx|6PQ|^I9gKI{1nk*E#Gx zwv}7wYyDTvEm6(Rqy$a}5%?xTg720?8!!00kKrp#bOWc_Q~OyhQ>GaF;eX9e&A-Bz zq7P1*n+_LvHLjoVVSs1g6byzhYNa>4F4;JIJ*y7k2^_xhzv{Y*!n=I}whxy^wEVZ8 zznkH6Y4~r`fZ;0$5rMzzx6f3zbm9HG{7A=Qmuuem`Lj-iRD%`XS{&#N7%=;XdzZ1r#VqgPil69VUy=If0|+O#VEb=PS}Dsl@w+mRf8sp>-$m@ z{>KPH*;jCm9^swUepdQ}hY?!;?Zh)&y4Ru~L3_Pk3y@!EIj|VLKq0|M{kyFKXM=X; zpDjp$e@$l?=#0XGmuHHRHc%y86=bl^rWCwTQKhEhM;tiZI_jz?M}#xyLF-x*Qq)w{ zf@>f620mv_f-RsF$aXc?B^W(3!g;T~GsD!g_9^GB#f6hsO9|DI9|@GO7tuSaEIe;< zDI&RIait|Ua@nO_^$2GsIWo65`xqm3RZtksE5)YuI?E1{5Wn^P^PNCp)D91NM)HE- zlsuf19FRz#!>BI~kWE2BuTO8ak}~fg7CTabPSW}u;})iq+U&L#H2dqp+*dm7S{6H| zV_4+zLO;$-uChqPO`o_xwX%Gb-A{IBFDzt>UC<1II6pz+OG}hEzy2CmSO|$84FbJ9 zb&e$8h)Id%DpzCvanPpLamW@2fank!i>o*&D~kjiM>!;wEa=}=FHZ!-W;uD5*9lTG zyMmnlKrd>r4~;metlu4`%9My!HQT4${$3bk%iOu?H2T}>42-7sm491Nhc^D^-_GRk zq<8i*$!j!QqOIsD9#`L0hp#JlN-!eRK=7a3pehNv$GlbeSOJ>J5u(Eym3DpX%OF_= z@}q&6UBCa|1EU`}Mx>1%Itl$S_467h@p)qne5AU zaZu)U>FKv5?>m2?^{>Ur@MYqnOJ~qxKB_Uqf6Q2m(k31brXO53Tx(G!0XmUZtuXp% zPcNS=k}~CGzF9tMl_^!{#&U4M^RnmI>ggL?hxG1UiV{LRy|crkX`@c~!sQ}o#bc#K zo3dF>@~mz(749hhl#)J6^#H!k_f9&_V<@u14Tr1bycz8EH|3)OHu)OyzI-xC#^5@lUN}fb-S^gw_V+X>Yi|n-?p8%%|7Ng8idKPf zWn~g#L&ei^P_z|0FI=Ak$L0I)1=uhyo_Jt6G}z3|LEl(H(k4De|B50;-re|Nj`OE-7RbBlBV=>-C-PYo$J-@fG0 zfWz`>iS7Rij0Dr%$938aD{h0PonG=&WkulOV6I5Z-P#)7hVT@0a`D>(7D+g!zELt& zZ{m!;waUT4hxR&io3?$IG*(laizX2v3D9VJeit58fV|Z;LOy`xWw`NC zdpw(G%z^3M`1}W_-5-bHVU;f2>%;zs!(VaH95fFu+tAYpp0*~&(+>D4V>1;!l&1Ok zRoY_4GAS{RDcplNVA!JocFkjiRVOx+a^|t$@;7#Hr-|U|a*Q(R~?<|bv&_%iUr#T1L`3G}v zp81>Bfyrd!n*7`xKX z0KK*Twx)dc!pe--W*La47ryT`s{VxPLLBz_f+ z-Zl3SX&sU}IO^_z%FaD(XOn|6apKmDn%nS(>L2+4u>l0DH$m=z*n;^&{(6{W^oNR* z4UE$pP(!T=N1n2tblv%sqSwSQCU6=P_IvO3`8PN$aeD7?(BE3c* z+G1n{U0B6@o>2E(S{@>?v^}5A;~*^UxRQ4z`LWo{yaCvu zUiqOdpE*C?2Gy8H1WI4o_h!nYYBuUV#^}jL!Wjq1>Rku+4=!3ao;tKb!w7wMOIQ1Yh{fTtY!BF1;LNt0ZQP;QOG9)W(S9n7sEA25zT2_wa9c0x$j?*TzruAY~_BNYx zw2oW=cD;ZFqOm#Cs5;LK&buI`3^24Vp2TXe$sE~!%BO4=b89~c;}=KaQ@HvV?Kl>L zk>*cZ#=zu}ZMf0M4e&hovn6t{-rvkII`%wZ}Qn!TX0@)B{C=U z(Q0m>4QC}il}`S1ud3~$Hk<1++~3->19s$IS}1ZJ)CfH9sO!O*pE8cxq_^`Ub$ZCL z2_}f=vF^C6c3q+cr$?Gkn+rXf22N!A;4??jnx!^r5I?yjwIHTcqt^ zQiDjurf68>zUb>-(F=ChQ0nx%#S16CuoT#9N_712Imf%k*X!u?z#hw3lOF2D?0$d! zIyR|PeK!`%S#lYqG8Bw*mHzkF+UfMQiWK&vex~WHwi1`l7u;ZbR*SAU<+j(L&6}>W z&q)wof;n6*#6DxhDIDsV=5ak5=L+O77N9%7L?ppap_84B3On4)>{R5ww5Y8-tB%rI zkT*d9H3lUqJKm#n6fIi@o6S=Yj?~SgU2~at)WN!bwAp^uLL{+QkTjj*_GHci3!h8~ z(v%r1$PyCxq!%uN?|Gp>9x`&oMV&OK*=%~lsR@>jW=6U`4EeIJ^gM+VPwNUuFBTR+ z(jXad(hF}>h0D_a5*{{|jnX{h1(BoJzQPrH7^eI5cW_T@UnAIg`rC?jKJ|+J=hE%b zQ+!Kj@Bd-!t)t=!f<55`2<{TxAq2O3w7TgK$8eD?AOTNk5 z-S^&}-S3?Jr|(SNmg=s$-M8yj{aRmnxtQ;Z|BwFxbe1!EB?yItONakEZ;%1$o2oVl zpb_oCD6Z0HSN~d>Q5IHlA(y9m;GI12#RvvhZ z8#uZt;5{b4uk}>R`f+d>8pWk?cCC*U1Kpbg1-eMXXH{RB!l4t1kjk^&0TkN4^m3BT zaE0gBUiTWkair0!w7-<#619ubaFQ0#d6waWu?&;AdPLri31OLD{|!l+>t1JfcA!CT zvXEKq>pRz#+GdHX(H6>z+caDZ?fmBDkgs;!6269^C1ONFw?>ZpU#!~LcVeHksBc-a zOrxX(+qCGlb4e_(VkBK{I3EXy7^!^{Wx_mrrY1vi#R>aoEX97IUr#x5vZWRKUQAC} zMqrJsJ8Piny3+L^zNrz^364h9?uwJ2`guVnNE0W$ydW(2W${v~iXpq9_95${;bam{ zbIt^br>B-p&6MHvx9O+$)AIS#lj^DG6bsAm=Cs@jDt$@H0PHEAD52pD6N1)QM5LsV zOi8+#LSo?Lk9pIHvL=-pto770@H@-5_`Su)Vv>b)KLMO(sPDxe=2l50ZUA$xM^yaM zFdF+Ev1J$+KTMoEptzan#Ex`?KDgRD(6@v=v4w}WpUQ;@+B?~OkyT#+8UkRlWxV0> z?}Y-f^-5j7*+PbjUO->Amx>QZuj&#b*PmHXKp|MQRm~r~`%30-1Y(Eb5sw2`jau-& zukmkkdkZ%io(Mc>R@RK{lXLYsP)TG_q#HvNokFnOf|p4vupH~SYSqY(;pARE*@k~; z(h~nf;6U!*kGQHueOA(U(ry)rUiOR}vBeFx9i73;uU|Gc*KP}@&m;{$Gmu}CbTibw zo8|td4sg32v_9nt~)&{;mo@4PSQ(uYO;Dvu$TRw20BJYGGLa&EA4Sp{!(0piqQl zCx%A(Kx5XXvAHG?aGA(V4mbcNs^M-@W)r$4w4o&DI%jhGwrwk8yGyEj&0K~TLj0Z2paJdO;Pbp$%PbExgw@bKA zC*FyyiG>&e$jSLY@IAB>EMEm&`yue3`O3y=ij(@VT>yvHPQW92qxR3+F1bw|w@Pb# z!F||vcirM5-x;pR9{c?Gxs=T2D2_!abvM$~?`Vpcvjnm%OL9*p7hP#3`qYRR>zNtK z9G=VhHkh1;d}26d>?G5eirvk5>rl3Hw(uN!f(`vioq0jj9%| z~!Iu^uMR9 z($n`!2*fD*dKAsJ+3D44A;V}d8nh#C)@3ZAhxHvj!pmxy}*M)D>d5~ucp?lA5eWltnZuP`0H{QhkW{L@-`Mqql6_;EJWxW z3(1^qdvBNHIhY_nWY+)Y1G@y>taqIIoq%J@ExV#iQC2@Y>0pTL;9TNgJ~Ad@MMui*?X8 zF7P<&jrEXsDgV8Vp`&@{YzxIWyKY`|ogWyW0}mMW_e8}^F8!3@bP(2MyR zf|q~w`)#pF>BnYp3Y}yToIvziu2Ok^P|#c_u%n359w+kY!F!ivY6t4;=Xc7*abnL& zgk)s2-F5d`%vhs@ow@g)vV%RwOgNTRtNLVzRJTy}Rr`fIL|o1iuABb+g~`?MzWTqo z!gBsUy25gB0yvO`g(+M?F6PGe$euqc{Z7m5TjPHJ^^U%V=~&qy$Ar=6QGXT4qrkv) zAtvHaIJ=Kns1aIhD5S6KNFEPzRT?ihA8lg7(5~AJ;Cs5p$bKsmdU-O0I;{SEMKg@X z*y-!~d%p2@Q2jR~zB&Wp6nbhvzX8dgSJszD^1m;(Z#x@*_nHG&hY zQw`Vmf{WgSR|ZpwEO!DHhg~YneI(lmj4H(| za^U;Bg5uDimTrE6$`P!8mhL*Q8j()H*gr1?7fu0B=Qjt+VRP>W*#Dm0ehUb@`EXhX z8aWSmy^9ZMGZg;&bbQ|RC}-)FTJpRe@Otam^@@L>@>t#V_Oy8V_wMZ*Mz9E;{Us~?TGN_~0KD0%VpzB;$a564zw%=r=$d9CVr5|%@KVSeAo6`+AV%+-sQg(< zcbefAodjEdNW`!NbpaC%Q%zpP3DXEtYX|fc24lRrbR=4TSBv2C#j*Yeoa-5MbQqk< zDae{vfJ+90J6UGipEbr?or>Rt`j9H``h8ySqp}Z2}GdgFh`m`n_Eo*-AZxit_H~>i6kT#8)%Bbj?DT)i;&&c~TL;+=# zt~#fuTBv3XX&3#I;D&9xo#v(q=^>PevROaR;tUX z;Y4YHi$ZCWYMhcHvy*)GWTk%Qz2SsLOh+arUxeU!vuSpTKP4}wA7?1*|cdk4<*TkB+Bh+j?pE@NjHWF}3;fkk%e zlp#Mv21)#rFRR{@DDRxl8f1nH?#SNp7DE@ya-vZ^u9IwXcBX+oCi3c$d?)>`HsiYo zXsn<{P+tt#vgp&HuV2{L_QtajoXJAkn4Db!(TFuBe=CHpwwP%mpJ0-^de--B6cMll ze^#kP2Pa5B=b}oS&|nGS{gtK60kjh)Wlvc34^s*W0pwIvne$oSn-f)fmkyBdrHicJ zUTZxh&rYE(JQYxG0G6khkcvtDkcf0{H56#SJC9&_*L|mocwxw~3RERPOP!PFrByG& zVkoA}-BvIzLVG%j4QI73;v6YnsNDPY0dqW4mPzxTp_P;=V!}FdZxcc*Omo7TGdVsX zN$ru5J;5#Dh~P=rV8SfvyoEE5EZ+r=@%lF35}9Jlvy_gw)cy-kwXTOx4{lkE2$sb_krK~| zwn+2v(E5po=!?|GSS{UA%mUutBm1`Vey(oRLi3%CT`ezmkI+|V zcTKFwA`PhOCcCOQNk|`X1zo$o^>blf%rB!iJHdaCfyF;O-OR47vmd*}Gva7l@gZH?_qe*96$A?@|+xG0`*?E_#K!^gl~cw>uPg(Msr?3w?==< zOQODCTT;$u!oqTNSZ7nt3R#eH_<0$8GJBe6M;#9JdK7I3O(J)p6>`?5ct^`00Gz7Q zX<8^Chl1QGfn){C91yIU7=W&vc=0vi`X=`6USD#HJ>a%2_U9^A0z>sEub)=m>q7mq zI#qxA@|%V}NY`-f+lPBydW6w!Z05ekg%G~+1}`BytxV-Z^YPa~aQJN=I0GzYpfvQ5K+;4{Mt7Cr~>$t&b;!zxf$Tfk2A;8pv=!tl$3a&m2 z9dXQRsU><8^umS1j;FLawn|*H>Cl)>oWM92j9Ayry_xePeDE8jF&>4|WkZQb_(n{t z*eOFqcnxL$9Om0!O=*r`GbDtIIMgGMvQ%7^^?RZ56U9FKe~=t=)D&VS$7Q& z_{(8KpnbOpgcJtvy7QoDt2Y?4+iRn~k(%Vy?e{&w%hU+lbW{(9{TiW!>4oZ*7^<#7 z*6d9f0XGrZy3{8f<5RPBMb|pk3C^Bqd4$FhR}7piyYZSsqzP6~U2Pv2W^qxw8iO%` zQY*sJ0kUOm*ZI}()GFe5`OATn(W&QIEV`Gz8{tc6lVB=T-YR+55I0q1@%%%2_DrT^ zGD1-xaTHs6uLK`4(m4_bFU$N57ai>5H=y0r0(v}Mh4}ymAtb-g^Z~osqGd$fa_Hv# za68U+SMVhis0r$vbG~+;g*zMf@S~*kBbMSfR6+c$g3toY_Hca482V!flBcy5HNFn@ zE)(+t%L^78HH&vQqHYvbxtrb{1IGw7ouTQUm_S>G_5Jj~;~QNfKT+SZ=EdOt;?x-? zWl6PnmE}B-l=b}cDop;^#*6!_)A*XHQFi;#X-JdHgy@xJ{J;n(UVY z8iwlA8O+?Wj7;=G{}gCRhmS#UJx1ofwf|JpZ~} zJDVxVqpd^*M?UXgQMr%pzzt4&x)@)IGnCc2dsC_9C4v6 z6wO>VBLpg3PCG>4%GpnW9as;p-Dny!sy<*Yfgv+SiS&bL@o zs}_dSgDcNhn;SG#KQcUjkiBFAwXoI=Kk82pMqfA_JWa>du!*-Zwe{|s)<5-{PanR# zYr;5xf$B+!qP5orPj+iPVy>^}1zL8{uLy*VO+t;0i!w()XU3F?0p4I@u=~nt=*(hy zADQ(CiH{@WJC$=rUM-|5t!olNrX=?(YO8m@_3H-7tbV7?=T}KaI_WJ~j1Y`961G`2 z!0OtCk{n%g#Bq09|NWI-M`r-Wi5sRh7x92$%DEOXQu;0oof&20Y^m75i{!;M0W5z~U z0Me%~9NLP28@>F-j!lE3j&xOCCT2zIiqAyeE=AGNV6Airsg|nO`*3<> zdKa*^`X&^b6v3W zbRN$}A(7x>-FhL>X~N5(>%8P=${z-`%c@IA2d%#Hd4a!Zgra6^=t&H!&3IrVK2kqs zB`2V|<5vb@?9hdqfRQH{&cj(cSGU3?yq}pJh#Z!>zht~&xH+Q1j^y}=p^^?_6hDR# zi%2%~Z3Il$%19lwHW$dnXxJLCtO;_84p3{X$NA2D83v^3ONxoRwV9q(h5AYK5xxIK z46cdaMmFJI%A+`2i4>{}9sPu>ia*q9RtmwrmlgpVs1 z3Lyo4RQ7^?M?Z%X9anQrfARMSTp&*Z>{18!+ z*&_BP4!1EJ=p#&yp}mT<&;{nKE4If9yM5o9J*2*oPkl&Q2j(ysXR013^f@x2obK%Z zK*|Zo$cr$r8lB%BJpE z9>=UA64pY4$W(ZH*S)!E-FS6?zG>4?k3SMzl2;xH(+{d_;V5C#4}UFe)24EEr2}BX zUkdk-Pi|jl1$>qW9bGOBY-w6Zgx52H52z?A7VmS`07^(KD8*aR#Rf8bVE2Ufq}dDG z1b^DSHLK8ot6$}f(@p`oT~{{t*VMl8TQiE@hM@(=7p#Y-7}C-@?IY-<-PXG?t=6H$ ztc$mB=C)GO0=1mkb#`(L73adRytm&1jk5@mnma};EGn6dH>8P5VLZliP%ly8#^Mfl z=tUFc121&>25%DNb1`;gax$gU7)^XY(xM-(M%5W(~0MED_*a|H^(^PW7wnmT(=IWD%Zn~b@uyuVUP3-5jZ3$QCaEf?ut|MCaf{0png3{X+LHJ>Eu{F=@s?>eFr$k zl{afLv1t4C2v6o^|B_l-m%s^WmDx%M#R6$aDZ(G9PQI?2aeRg8yfctgMZ`Q#{-_E>;<1jqk`n#_-OS`O8{nh7sx~3b0|^k24HF&i`CgU`RDf8cUrDkg?5!y zzeEi(c^olVIHHl3cxhK02_yw&sB%1ubw*C5;YK&Bns=Zv*nDvfHH&SYgP&xtY@XAF zZU;Z{x2^?h9j(?OUJ^Q%f;**sWv>RX#c4v)p#N%j;7A<3V{0{o>vrrDrWIYK-NIsvlxB=TOe!E zYIEkWEailyCGkRqS7qEhAapp#xKd1P!YGBNdlZKTol4XKdu!V zOHGoYZDv)MH(Yb{UaJuS2o~>mHK6ptLefY*B5#vPG(DT^eWmVpm|7{P%U46>hjNSE zO?MTicd{nz)CjkI~(7dj0~0{aa1b_KXWL#$R>}owwh+Nm3m;8 z_L$7A&xe-?Ps~N@v6VzKL#1X%u$7{rggi#Qrja9&Rx>$BqPe4L-FwSZE|xkmH>|WZ z>Nzn>C{p*PK_9+cxwXQ$h7Xe9a#T>oL87A(1Z#3n(>@$b>Wtt$Z!u-b=l#xPk?Bx0 zyk>WYzVruaUHqbc+4`a%f;UFY<1Z{dKs^${>88JHzQrpV!WqwyE0I3(KWf2$%t-krn1VEoyR9_-x17u4BSuoKRiB@mJG=DABIHB(SoEy6oa++U6$%Fv==`+g};)JzS!|gd)aV1sWZFGwdz%Ez90YuE#g>S@UrMZ4LpmalRZ%R2u4t-`slwS zKBu%|ok{`+4&oQDOGhG`Vu>xfvu z#u-uG>*8q@83=tkZCd#V1$lE{7?OBXg(>j0QXyzKD2!#i^TZ6oCPyEVyzU*euI*N%KQtroSW1?mmPX{PaRLL%{#(2a}I0 zY6_^+6E)P%;m(NAf#W<*W+~^qc<9kBHmv9*5X7vofee`!b}X>*Ajry zf*t3|8K*Dk>vtwfBxEqa@Aeibz~nF9g2yAp&8rECq5euUKv<0hl0*nxuP^`pu*uPQ zU>EJ0N4q!FEzYtE$p7nxs?3oOqgPjeF#R_OiD0O$VEb66j)~>f;$BUh+?dJ9TD`AO zNQFI~KY1QeR*Dwv-PCt!)+#%qL7nl!K&v}nW3r(^zYGrT+jakz#I+>)-5f}rXKTQd zp&mcx?ze8`ppOKSi*(%$o0sD1eFx;Gf-usUkVP4nV>xAObb5v4FsgMM9gs5AOgkmW zw^@7&+E~J3dE>>Yb)bQ1ZcC4xC-79rr%Na0ssdfNc~XA&h1(?3f4>~}N5}N5U)a;p zOjxK8xi=AhZv~yU_jdH{ye1BxAAI=2frSK8EFzL$=7z`MlYh&3EDzDv@2+hNa2{-T zi|gz#Vj!c#0s%h@#*myjgmq0Q9)A!h(BebJ`i;k5Vm;@WY%wjz|Mw2j%e$2}+=^SG zDef$9son*~88Lh{38WjW+S=*JGOR_(s%dm8<-xF;DpzIg{CJgbPZ{}k60w(SdQ^Qj z>AWjchsWdBgK?k#xJhmv$pr-)R4^hzp5flwRMMMg#|5R&9(46F&~HYqA4)JyX3~WB z8ik!BUh}d}sxGjy4zM7^X=0cjODz~xMcq5uKA*rdA7BeVVEc}z6dbT&VkbeE70-qjww zq{1E0EN3;t$T+lg^ZN??91YF_RycCj;<(gI*y@GsX=$!#`N=422lr6*pdrP3$O}Zt z-8an)66sR{OSzz@j=2<16t|fC;jgGxiYW6F(C@l0t6ZmCd>LWZW^kCxFR^*4V=WQn zi8JLwC*mUhCGlW0EX?5=SLUJ)_Lnl52wR^=K-+=wNl7TNfqQ7_t{NH+d!03w-*m8p zY(-+e1pukfs)eS1=#TURf{C%_F<~7g^*-->m;>kJ$R+mH_)dB^;#p#o>+|9edOqnF!}NKe>vlOmcY=PA zdXA1H*T<(Tmi8);^6wf?YJLbC7%Mk<@g=&Jj>i2i5b&=P49{#B_-s= z+a{Lc9bvBm<9R$H+g9J^099@oi&I3*phC_Bk-80xRIq9+(xEc09tc+W0etoNs_0(YE@F47c94|~tH1tg6I=7#kCKOaA?~f=Drxr8XJ^V? z=p`Qh&N9WuPkJfW!+|2dXwS@q)QGOj8rurrpJLM-qi1hl@>rm}%g4StRc z`E3Q)AmK2u#FT|udq%Ayx?vdj-C2U*k8O}15}agvS}^sOxM6Y8id#bR zV+L}IySK+IA=PQ?DlL-;ELS8>52qjlL~4ILtEsh>fd<{K+lPli>58PZ4im~#!>dmS zq8OZELF|*%dt0ReW_9;921kU)aU9s-Lw`o9l#@~ zwj1Ub;+5nbKPqFyOb@R`na?9nxmi%I^*&=K5)Fr^6UJR}SFpxQrQ3g?!OQKuCUgM2 zL}ShC(e)~2B={Jznq@bPbHtX^vfftx8OurR(&b$H9A%V_7GruHTK3h6$L`c=ElatN zhxFwE!&M|2`}eM4#xqrE-xu44IMCb6)#sO&`&;l_$%Dql2J|a8-u?bTabWlKv`)ZZ zCtoqs6X;>5%P+ev-7{_eyu!+=(10wAd%DJMF1vieUNrGHNm&}4{=a{j{ZAwOrq-KU zceb@z#P1ZS!E=l-R<;qN>g`m4vF8(30{;cX?VJ_(RWg9E-x~h2Y z9m1y~QYH!itiTxw@+LNy%Yc<$tVVNTJ7crw=bA9An!na4vw4rsD1AP?%@^lXltu<4 zUd`^IZba3yvDMo^)xx$-4G-w|PQrR6)9QnkY?#bYSuEj?+e6Q;t^Fa(?dv%Nu1mkY zeM~RJ%d9&)rqywRb5i_YM7tYAHS zW8j3dJcwY$i$0-(vn<{^H8YzOu@2Sh7T8f`ABqE9x07OFOK8|aBv-9@ccl{P`wz9Q zXV+q!PWf1{&Sx#wVz@*)#$c_Ax6TlxEEs!Fz_k!;b+c2%ls}r6Mj<(gmShO&wP%oz zco3|!gi$=Rio+7ku)c|e$3#ovJ(DZn)?g-o7R{-Nc+a^uJy9RSwOfug!PhnXwJ`Cf z`QcT|#t>|{;V*T~eNcecFdX^EDD>NUmBxL-2M@EbrWK@ewi}9fTydI;pJgl$yHm@l;Zz$J`DFQe`A)fiMQpRdG zj{Zyen-Pcj$MNA_<3O~zBhqy7SXf#gk#gx%z#5OMdIg6TaG(rpyJH$|3yOo;zq*Ye zOcv@%j6mWmNZs$_7@=A5t6U<5J%X@tY>DtQ%yX*Tz!FX$+T5w1;Mx~9mWs|z_q$%Sty<=(9hb zz_tQ#CC)xMk>LfP%pwTFevNr(MXfxw z`{{l}D|`Iz-$2dy*2!G?lav|Rp?E7hNO^*Xgr+xA=bJYVk#U3GH*XIjV~Ili9WCML zJL>HXB3`2_fbC5-Ci~um8zxyKH#vsxbp%BVS)99ANE$y|V<)J#cKf}-G*QS1I&ur%!Z$WBBG5;ZX+)GgPx;qq+OD?2 zXu3;~O6Mr5&#cpK(8=KrxV9D>I?WtfjZ-i@_Hx>yk#c3b+DCM5zE^ip7w2B;S&hM3 zvXtMdsbQEEXpcmL`Ai6|;7hUCGdGQhbLol&emv5jPq=rN2*2enBRHo~YV?)ESWxNq z`7-M%f1P}vOQY%uY&AR|$CGSQ5ILptYos7VOymsps;|Bwl(;=N7*mLD|HGb5{IxUM z{W`3c{Z1MC0auKSU2u$XAyz(y%(BolCTq!}&{X`ZR454~asL~RI6juom+4u%IWHE+ z)CrB$>w7n!p0P*5j^vxI zFG6<3YJZez7uh&+S|?GCOS1rukO35-V;jHUa8VGzOQlb=6lK@YMD-nc@hXm|>q z&2z&cr+}U2P?`UYE@w|z`t&mA@-OX1HU%hIq}dj6kZ5RGAjV{3f|&LuQqm&pQCQ=$ zC3^4q8S%BpDaO#h#h|hU!$~v(RC|HY^MErNb)>0eVaUhbtt91~?phFQVr69qZNOK3 zu|O>wt=qvs0v@mq-YFtZc^BBi??nGPt8N)=W?Q zjrrCi1_aBW;UV=GdxYMzSKyV>X|5TEqm-cjJ?G(zBK3=!!>6p~{nnl?z*%lx(?P zW%PJ(o#Zo*r!YJ6-D-RnAy**P)W28UO>=!h z)ol@qZMIJ01Yv!;k>4eOC>M&olVdJVcZ9kuh_ zF}@5Y%)`-;l>gRzFVdt)FsMXUn?!ZSY^9Pbl_N56e#(FF;QfWJ8KVTc-&H8tN$=a>cl`{emWBCyQ{ znINenE)8J%XW&`$=AdX(kec$OB-)&$2*6eSu?#8UYi&=e|J?vMb7`wrdZ8>ebo0;z z`G{@BQ(#*`^;6)f!Ux5Onex44^TZH_+_Tx%lYBd7H@p+h)~QxC;YRWAZKkKv&03VN zr%IPw?yUWEsb-C^nc(u+Uos_(3#c&~zBS9)fExRHb$g_Z>RxxE#CyuA*0eZxvJMlR zgAVJ^Q-o?QB`L#RhbYI@A{$qwhtud;pIKl3i9NEVX1Y_#ejIZG6pn!Ygep3f9pWtZ zZQ?(HV`m3?P>^+dzv#19Pn3-*qQCBMH+irZf}NN)U51Y1O%qB)_!VwCVa1eDj@$D@ z&j0}erE#+G{w`7xzduH(?+nQX`-SQCY8DQye29`BK0*lgFRk4Jm6XJ{H{x=HL2U4W ztQe*yIk+rZ#}h*6GQO^gsMikwf$rjB%1g%~C31cO*zVK__F5%md_%vmNb$5cgj6w+ zt2UH#OsQ{irPLBe&GC#i&qZ4f(t0N2M#o^_F11*8uhY%#;caF-!s8GeI50zLcQA&n zOM$*}0;3(Nd`K-SpW`(8qQyG2o{08vSuM)vd-fvf9(NX1rMnL)9LEqcsr$~<8qp)2 zPZDHuBWrhHFwL}J@)4<2;mjyx@wr!*HU|s2100)aQ35m`C%55?j{pmRBatMzd4YWb zV(=EY2}##Y%T0<*A2yQUfg@TV&lT8lWj4^^Es!s<;EA>1w?91mCi2 zWHC#nj!K$$Hm&--h36bZ{A@QFm@#br-H~f4s6O$YmWe@0E_zH|!;o?eRm;ba3W2k% z$TRam+H^>uJBu71XUmRcGS7o3ah`h8a%`|JUgcBvEiF`k<1H_DVLuIPzeh)s`LNNx?<4S3pCd&j%+ED)CpF zY2AYVP?m_qIfi)w7yJkgXxqVEv22Ymi2)kUXu{Ldsq`q)|fTU2Vf zl~niBz9sj32Pu}_0Sju$Bxm#*=$zRrJf;NxQ$@a_i~;55(~?vcHn49fj-R%jG%9hH zknmYC;6N-0+w18{?;`3ZgwKpcJ$fzaIdg`b z?@U(yW9z$RhMKACbauM6N~Pqv+k!FIr`F9^s8#n*b4(|%3el9igvaqq%BE&RE6bQC z?w&qZ9uk>kA&81L?$^agJa(e+8@6i1Vj`jn&5;oCZG?G{e9q^IiKVV>=&9 z`kC6(tWKGsCLXYK-Ik`r56#(&1j&|mMtC&)y+i#=8)QPI(JiTuq2QbddDC`M!XRKS zZq(d;Cs{TO-ozxr;`izyfG=#+MB`u4&t=8CHkv6AP1O?$j_B&cp>{nXe~%NK)s$~l zCv-hvhRSz5aWV~_#b8+nYVrutkTPs}>^(?W-AkOV0L;6+^ch3hWq60+6ph5P;V5^r zqYWC5w0B<&q&ws-Z@M|lvywA*ACs3;+x`~8F%K;N*0r_3&d-_3MBMyaz~e}y?+A5Z zVRaI0OKc_VbFte1C^^y<2NIs{c7BAq-zxEqmbEp^Mn1n&BfmnJ|?gUg*OM(88AWL;lFt5G9+fPSL*gxvaTzp%_rPhv>1L0hST)X5!@4#qv=AR`yw&Zn+)_$DyjA-%b$^Jh}I;u4Kk9;S)Gk0Dr(5v z{EHKgM_urLi9g`_e~do>aPj_c;RlUcr)6v0n6F(*TlkSwU<)92irDO@$KNh_7+*8K zP=pnoV+hrGEG{f%cQm(}8*5v{AIM!hR{$h^*J*se0cXh#Spy8)Byh%q9Rm#fbKZ`6 zznTpi1hj8aaSAKEUat;WKVF_*@0||u`n@6804%d?X6PlStBz(Xe#*%jqy+GX&|7AN zDeOF1orHfUXG_sdS2Ov+;mq?}IfiM}0A4TSYW_=}m1My!!>`M`U8<0#EIZGe?k}_D z1-q*u^(Ds~`8ce=(~3Pd8pdN@!C#qdQqy)uA4$W^%yu9M@ZRt0cPWaLLHl9TajGK7 z)^xBFKlVTCpn|5Ar8T$(y^V~}bLG%Y{#A08Z+OXfQY+R`|NUHoD?y${G;?#ibT`bDe2`XILPX;8fus(dPo`|vPgBdC zoaZPqBw0y|kMTTM+fw|rb<7=&>}qB#DM_^`!_rsoFJ~*m)1N*P^_~!Sd@Ri4(P{-M z#tQp}ZSuu&Iof=4KAUd%S&63Tws>|G(_`eTE`Tl1=(?XEE5Hzm2H(_@;6X4J?e4g} z(kmUgd{a7Lg}z{}dZjm0#q4hLRYbXLwJ7#!iy)KO8$zM8Y%A(OB>TM<-MU{|;!okH z$&WUCm@(c>aI3+pWH*so^6}+KJRD)0^|9twQboCdVUu-Y)Z(mceLbc;S`i@_RiYso z%Q&@^V3WAEL*-C)bc?O%&o#Ra?|Znn1mm*xpKO{^$DoAr*p&(xV+@+N13e=|~O z5$0LyEhD{IB3&7Y9ndlfyeBTVltRVkCf=iT|4I3Qgu4vj2#}fJ>2>ukO2X1i+&*f> z`R1Bh+d#Fhjqzy~9#$3J^s)4_yfVE(>3Zoeudj^ieD^KxD}cN2VE{W5-27uF@zXi( zyYHolCe1(|ALU&I{tbrUGJ-kh_C~lo0&c0P7Y-{05 zd!6JuG?I8?@`>6 z4etwrjzSvk>eRwahI(CmV(M+$$#MME|~-8;TBM?jUgV4EfE{DcW}X z+INSSWp?xjK~9RHk8PHVVFH87cluV)7s;B_-rozSx*^zJSqw`=tqU$wrNrFRgj12r z%;M3S`*qXxecc-6Cj((E;#*qnoLSYteOt<*oC#hNvg>vTPS$UN*zvsH1z)$_KhvwX zGw`PCr-hv3IvK_)?Vsd-_7BA9t0LL3 zpOb$Lnm*wy`W--O<+>rq54d6dRsQ_v|D5{&TjZIVmy{WWawdF8`wmprZo_TFA`=^dBw$BSPyIGj=uqcZY&i)WzD^j#eh&cs2 z&p*-2X2}|H-+yjKsg9|El7E4wsCC znxmAAgo3ftfARVsWK}O$kh#5#gM}jnv}F}@OKWHVpBKevQRw)X(<8GgxtN)|{OjW9 z|J6k(QPs`K$-(^eH&Gp;QhI7Ito4 zeF^|Og)TQc8}tqzF9py4%EI-J%FWHf$H&0|?c0Cm=H{i)<>KOIVdvuDr1)RO^eK3t zSpXFOd4q@Rzv_hE}>lovLR|EhqKlaqywjfeL? z^XcpVcd7gv?*Bt7|0NG()_;Qxw6-(npkU|wAENn3Q8c!P>g)fmEI>PBOV@uYDf&-6 zKoopzTr3@e8Q3p#q^M9KEuURoMM^6e}W^N817CsI>9tvho9&Q!@H-MJ{T7r{> z8(KF7I~xEh94>b5|E-_@c>kaMls9*<1X-~|WyizL4XxMO4rJ~^!3t#`WG-R;PkW&c z+kfPN%9;zx$$t;}JniuEO_7}8HTXSwJ@!q8L*`TZ$bew78TPQKm^7jk4UTj^L6B2M zZX)VPDC#a1LgY`Fy3d#rm+x3pJSl^JB~XOR?f9{JZszCKF_f27v}}Jc-q-zoEsrkY zaqqXWA<~7xxu81#TmARE`uzNFPH{MxckgRZ;fAg3T)2fvY6VG*N{G9^>YkUlb{<}l ze6WgsADN|h(9)z#j_E=an5HaSWnTj74*o;;F1iSofV{NSwzbJ&`CGuiU80slb*VGI zm?5ecnwK&>b`8`@SoxkhVLgJ)#Nl-NXu{;xIZ(UnUD|dT@xg)+&C4tzTLJu+wgwmD zn3u^ukIoMX5j*8l8Vh_x2WLM#8NWOj2iE8Y9x84;y4%0g77*aQmr^KzUpmg4bTuXP ze@cpjwVb&5!HH|LPf`6b$O+!JY{>glk0n+qOMz z+qQQ8#nbLaROC4)D(*wwhl;vc_hgDuDvt_hJW@{)8w@P3vJ)6}A;K{2qcDQJ>C|sV zzru4*c*DP05c38z;u;YP6A!3CYEU!65fPZ_ORPrvPMLo>esWEz^und>Y6bv+`U4%y zSW*6(Z0J=gF@`N4XmXek+)3^PJ-t-54A@ET#9Wz0rwKa(F0N+20>@k=3pD}&aUBlI2!MJjo zj|714Q8ytZ%?fhCq^=O-1$bE@T_F7t4d6T+FG&l)al(!j(9htw^*rG9IthMv$)Sgx zDt-F`ay5S74!mU_==bZ$L9l@kWI=@-Z2wu!20IG1Iw z^5u^-D8=&|aa5B58~u_gh2>m1f%8?sBTT4;i1`c#-MR(4Tk>?ZHuPv2UFbH8}0S@ift}! z>H8i}iGBLO@7)7q+czd1_thKY&=AMk#qI$j%~9jj;Vc4{PJpJ#MM4b}SlVNjO=W|) zvXUQSoNKh0!1G2S20?&#{cJ|%2BUm$pwzwuqm%~>xl|O=PVB=*j4LO>9pW(nHNE;Q zl<0JFoYH&N^VJ^wT*MSV*0rh)&dv$)`oxV-^k7ItG_l1~K~hjZt!17p`x}m*K(e(i z#t8gt<9p-%2Wk$X*iJF#ab2-eQfI@EP>BaR5H2ULdCC zA#Hr`&iU4G^6Cb~hgx3`9=bT?_!;<@Peza4(y!=fdmp;NR;bSF{|fl05@^Mma3vmb zwY->@4APqEAqb5LN{;eBQ+b-!%^mT@*gVoKdvAB2^WNz1!QJRD z^KDO`ziqFm@^s@kZHsulEZG~-n_P_drZMx>`_A{qNk=Hw95#A$F~rWk%xCZ-1XwfG z_IffxeE3FPRoUV}<2Y><@YVMo?W32Y2?V*8djkMl8VvG5QnVD2FbN}GQ)ktDf|{cn zWsw4X!cLG2I*2OwhOqwNFOe3GGh3Xa8eBbbqbNyNkGVzlC!>&T%V7UK|0Msx>8*MT&wFE_7=cj|gapYGsB zR$(>Cu)2HiU;PIz&}MeA-@#WfK>pb6AyWB*yzFUf4__S&+N{1K^#nSilW_K)ee(W$ z=(T9BcpaRhs)Okoeox;K^6MFH#*9B&^qt))>xK3n-I$@Cw_*C=IbhAr6e5>6QN7(`z`5&*hqPH@W|gn&Zx z?8H~+K49~^N|32X>*HeopzZlO$Y+mfE@Iik?-e`sU-J)@>_8h&gm-9wvnINV5x@~S zGQ{BZyHR;T2>v9h&-ast5@}Mg2d8ofz8twrG7IUi32$Nc5?EugG_QqT&=;kkM1C~7 zw5mnmoj2MK0)6j9RT6G&|pLHZ2B)1oKe`ZIVmL@C3<*OOVt^LfRo2hrCn zuU0SRsAnu7sRQGNPCpC5yL7G@09TOX7kqMSuPMyFYh*@(ns5M#gXV!uf8YTzfJ%v+ z7Ow_L23gn!iZ7TQ`f3v_0*Qn4O!TQT6^c`TbAsgF;0*-wPJ2v0wE9gs;1{g$Y4;Tv z<67|kP52E0o?P#{*0hEF(gbua3D1#_7b*b{y!VMNvV^R zyB?Rc)n7s1Is8BHV?blO?2m!nKe%7tg1C1Cee-^r=9nuQ*c*92t=!+O0pEI&jAkJ0SG(ac_9I;B$>by@yx;KMAq(Sex{7YOus+AN?21bnRXkB~rTDs$(4 zx-9zV_uc5*+qaQd1Ow3yhKG4Bywb_R_N)5+a6o$Ae$>l0^Ca2m?7^81imwgG%EZmn z+Yd!w^8@%m6=dB%Ie7F*W?BhM&0G(JSsZ)bD*kBqDGnpLzIL@=7I_e-88M!a4$FKB z##^w(C`kTsh9le<#+xxS#>r2HtX|s^9ykSh(V;deE@q^uFun~Z>@;6%KKHWMu(@tC zvA5%SdJ&EU3;tq#lsq5A`#eZL;h*-&G$LJ~7pWvO1I|#wNU1yh_nM%QkCwqGkmVK# z_$FQpO5Z@iA(y<)f!?P-bl*82qyCY;c|d|(0xpF;Cp$#nCYLWKz5ORUY@gph?yLT3 zuW;&@0Tw+Rn2((wpL=+tar1e-)Q;GDJWkCoxVcozpSUp=T7JjoClX)}56?t?xug>o zk+_YGEIhpp{{iuK_^bHYuk%G%4S+ZQB8-TEB4H>4;Ql!!VVvF-s zgG&=W4PxBMxsX?NCO&Mk=t$SIT9=SxOy)@9aP*L(I+1M(+bE`yTsus=OsDRj{FHq} zkyB;cZrfbjZCg2~%w~LBa@*uSuG5SMwnKR9dY`zRS607yQ-&Q=3#d*d-DIjwRWn^< z4WOoB%eHgyIsXbuC(JI+F7JwLBdZ0Z1;Oq|a!Rg=B37navRbxUKKm!-G3BjlzU;KZ z$l^@v?J~Tv-C{I%T0qCq?S8GHaV1aB#qCSWVK&>v^W^*J?C#m4`x3>`@+8%UVq<*c z5LWvdt!dK;$cwHOn9e=w79cr9=oIM@($+S;VSO_`%iYoWHL zhKA9!Ss^qD-aP5WCCfg}tWA$amqrz*Ov`~l_rj=cfAMQlu2J7iZ^fn^u}Rl-lW^D> z%}H(3`Tj<;ws^()!uG1AraRRegw7q4j-zA6d1lDd0Lh-qS(;a7&UxHIqVVX_VlLjj`qAAh(DQ!h8FY>+gg8EYQfBi^A*{pTZ+=_zjO& z>T~s}*7m_U7W}OzSD%H1Z{9gUCz8?EP91m4{%W27j1`2~L>O)i54P`;M>6i)zrKvvf9%L>!fg6df<`Q_`{5c1=R`tU9Iuz@N5TNfLF^%c z6eqI#ke8{Vs{2w#c`r`xaME_;`Skh6GS2c##f{odQIF17$Gg;1(79Zje~w~I%3o z-9yOW`>G|ZXK|053d~Bln=%xiL}l0RI<=UJsJlj*KpfC3ec-PUo1m?I*lh&(VG_6D zbD`#6Fq=SSyWUd^?)}srK(Pbvx4&#d2zCt|vGM;N zkjpL+bs;(SZDP!!-~F$(;Pz9YLNab-!N)_Qd&W;tfh38sB2yO6Xj`JVq#9Ek&p*6U zxdYipC^7ZE-+k$;yF|s)f{*rP!4v z+==2K<++6dfdBtk|* z>;{qs+>F9*n*chi47?0ov|aRFF*Z+ld-1mgxAnK~du@)woK!v(J_@~MKZOEwi;oiT zC|?=h!u}-|r&{YIr@_C3;yPjarTB>XsqN7TP+G&|qI8Ds_EGke+Nx|Zw)WxcLEMjX z_80f*@31y4+@ic4^R_9jG4jyzP&$WChB1aPhSP@9hBbyXhF6AGhB<~fhFgbPhkGL- zhJyBa_B;2z_doWD?m+Gw+GKnw{YdeV;vxy6^1|~XM~8Z&@*?u048li;{lg6+4I(Ip z>V_SL9EOwj1%_{iRQ6x?dxr>y7Wd;0UG76(lU^HNqh6~##Cc(O@`G9uXg#pq`lCE} z&VzGqk!izD)=tz1OpPiPiDi+@Az4CbLX?d%8S(r`4HoR5xnMRyD4fjrp<9!9+fojb zLJrwC8LkPV@}oX8vPpkXF~a5c;fxy<$)}KOk!#T>hS7|x+9bP_bSbw<>CrVJtA>`1 zv)aVFw7S&0RCH;!iMJzp?(KXOdK5OpEixf)OA(Ob6skzDB;>B7J}e{5N0!3>3;RWI1}?p`lR7(PuEbu0LY2)X~yq zxh^jstxVNA_GM2{TRd%IuXMdNyFS6iz+`6XXk*ZKxZSz>zdt<5q|$d9y&0%)dMr+; zRc-v;?0QbsS=n;?z;mzNaa`GY+4SBS`5;(b`rZ{m5V$`*9&O?6am~6NN>Z?Mzd1eK ze0!|5=y1DP>vZ^OoOKoW8Hi%&{+y_)>;4|ea_IhA0fE@dao_R&nS=1>`P`F`i1qs% zFT9?D@caUEu(oiQ2p% zeyeW(+F@X}+hH|-+~V~?(~C25P&Q1vRfV%r*QH-4q5d^-V`(yRcMD;UtXtmUqo{n{ z!EeF^bsFfgkmF3?H96`-VfXog%$9*j;7fan0DGTLk1~2XG`CaWcv!&+yGVykAz2eB zit-A_M|kK=LDdO7Gy}!c*k}er>^3l+f}#mM=8`4=;kq@QjErm2UCN!yCZfO$UiN^N zAuqv54fc)tgOMpl7F>0Ld$C*V)}~ni7aG-jS9gUg$OY*{Zjrub!%^<&G>Ut#gdZgoN)_nNzqA707wx;?y zImdoHa~4!qii6+GtFXEXlY*Pg!RYWx({z>KifD7Y(Yf)zpN^A#TVF6m(I)H`We#et zyG~!3C4{EnNt)#JHnFVINFZ8@ORgD%Utv2*C^d;51HYgw9B|4wd5R*-p0!gMWA-Tw z1>26KvDm0$M!8~45l$p6t8gQ1U#Uqbk7|YE+BVnk(Ck9zt8>RY$#C$IHvEQ`^pX7U zHv-wgoM;&=%1Y=j3rY5I;FyV736k^e z@XU%9=RehHTy8Vzd8YdRc&{ZZ8)$`c&Epv=C{Vd9LjPU3a)8R~*G_%Y@fxPtXG(Ox zmoaG5Nm~6d01y#Z1I(MUL_dWZ7gzLo>3TdSs+HRTHNy3FcaK111|MVF>>dCRLdfx*e@2%Qr)6rD0|v`QPb{WVHuYR)K@j`)U) zInPm4ntsR0V`<{E}riQPFX>4;?&O?KMKciDzJ zY{(H5#Wk~`tH0u{A8j>~uj~DG!ICcwjg}Ak8t6uph1BfY8raGKH!1CwX)9ylq+QH) zK>RBL1q{~8h}3~Z$qL!puM8S_IgDpD$n4HisDhDfCHT!oNJ1-un2(nioOT5n?JsAc z33~C(fY(o?d#!+nrNv*kXgEVV{_ki3jFc_RHxvZMVEe=f=|r}#IplKt3vcYH$sHVr zIc?9wb1au4co5NcW;Dr-d>@zyx{yo9Mk3xJ`)_Rjc*YyIQJ3K{ZND&~DbZiZ^H|xa zE)76Yshg2n8GFE4YvmwfZaV=2f;A)Y3iBbe)}I#`Zb=D4mW=HRxGCMVx>31$yg*_N zKDYV^fh%EO2)Ez^G-xKoK)Z(!O-Kn{QrL6I*55a1*Sof@jrxvV^>~?ku&&_XW^N}o zA@!^0BP#wJz&Lh#7W-qeV+yF?BH+McMfCsdCO?q6=R#F+Ix67w7DH>TMCjH9LryN2 zR~xWB7eX%bmHxd9M0O%Nms?2)yp zocH{xt}kI0*?(@NJ~CzF*=qm|$zv6%D^>}Xz!CTwd^wBzH()T5bcAOZrU63x2u*vS ze?0Rf=pQf`#4Znf)4v5b^CDBdkH1o}{8aiKXkP31FRw}=c$`qZF?`eZOx~CtIo;T^ z{9v=<3x6~CA~7rOlB^FmF8DnKNPqFR+?6K6CSuxw$N6yN7|Q=Z zw-AX8Tj@V&7bPL+Mhlcu%@+le{twd11tXx7{{#B|$D03w(jR0p%Kt#O5R^<>@jqx2 zg_2Ir7lld#6b0jvT~ZpIAiE|1uUvN^_P^XE0ycR}@ITl^#-dCJ{15ucm=%dZ$e7!L zqmh}V$0&(hps>hHljD_ykN-~%B8{my5P%F$ZzKryKl45^*iC^NNZEZ%Xt?vQ5gpV`Tee$N!n@4#fPIyF|DpTT1)~C&*Wn^RfTI z4B5KpZVNGE#1B8^kIl*)j${Qmhek&PeAGb9?OoW@3tN#rt9hjVCO59R#H5h=m#|6q(22bb{xm&XK%*mw5r`0Xht@o9! zy-TH0SZUMW|3Re7Si;I_&if*Z^Ew$_ZO#LahKbJ8d{d#JcRJ;Q$O zsb+j3fpcI%?D=gD(HnZ-0m+TfR$MbSJ}5pSKGaYIaNA*e$k04WrFShNDGVSUvxN*d zBib0)N0d-YVWRKlZ=?#@Pe4-ihHuB2_l|rXCa}^X)ha~j)xhy<0!3dT* zJgPQ1G}J^5zeZ=%BO5EI=PB2z7?j|b;TOefj8;HYm@JFSm5<~@Eu6A($A}j#gkx6% z#nexFPvs8uurgP|!d8erqjK5hUyk<}3`~NNVMC_{@yJq^4lj-*7`wZZq0T033CKm= z5Oedo$=n@;^E6>bmc_Zm$x>Y_dzW>%C`WFcCss+jBRb1q{p~obqz7J<=?qU(+rpf_ zZsmd8Y-9K0Kt%RDSZ`ovl%SnrISGk{!%|aq43r4Dmp${&_(-b1h8td#-qJCd+{}o+ z9Z5y&bTDk)*k~_$LIvRu&)BwhV zjKZbSB`b}EK^2qMZY<&P~B=#&0EAovxxa2pBxG zk8wwDOB=!%;|cBd2t3BndH}rs!MioydtAU{SkD2&8@wEZKEMMr%kgY4c>@{I-r`ZW zx(4H(G|Ne8sK38wn9H8cd|1_dv#s`FSI^~(i&Hx_b}~@z5CJ)uxCNecqJWz-g4Gb! znH&+r#2#%3*ybCt)<^D^0BryRzU8iqBRGK&9+6uC9N7`a7q``s;4Gg3eeJpr&U@0CaN*n+QV0ZeYh#88!kS{V&8 zQHK=ggI#fVxmH+pR_e*EsTGv4P_s8030&{WAl>3iV7L}Pd@$=RVOf~Gq@~V0+7gU> zPz~b!ySBqEV1W?BYIy_jk-|l|wzyhdIIr7XSzU2aS8p|A(a>eBsjnyWXa%GuoA(mM z+fs>`6lvRbLGTzGzx7>3OG;76Sdnk)_p-6y;dHM6rlsqqyFSo+MSPy7cTJwzN}Jw8 zw9$~ZS0}{clb;1)!$t|kWrtu;J$i40MvuP!G&9Nejx$K%j+K%ekd}Yka}L zU51>j@6{a(^SOE0#+MAu8g^YUx>|I^xg?`@C|L;^&EvEzqS{92$`d9|?jO7bJx>7{`-goM+96(1nHVos-t*`13t5B~ z7Q9HFo>Qn{T=c!X{3Z6u4^*Sy-UIGra(ioEZQo=;EtmJ^Ya8R%`U`eG!S~RUN~Pmy z6K)%#TOknt2`dymBlut{u($8wmfHv6r%uIb8RrdSd6C*nuuWl=s*3^n&Z)_rcXdvj z?w0a$e`f)Y5cxSzNP3KS;UC%cl4P?a0~J&3TsMQvDcf*Dc)0}jV(d3DBb~7H=+QY6uc39GHmOV&1vAH z^C{UOrCsD!&c~O<=kAgF*mdAL@&P|0kNBJiDP}p3!$^%EaJ`6cp-$!~HO^c6?c zIL5{)&7viCO{$QX_K(4`$tJsX$HfzYaa;ml4|da*zsYysKZRHw#e5`rXZi~Lrj!)U zW)#*Wmt+;FB%fal#jxN5KLq_Nm!F=JFhj7Rj`-#LR2wFT1p)@9wDu1C=7L6Pi zbtY@iO88@7Ahrp86wM(IKAL6$FEk=-Y(SL>4RlJJq;r0OBjBU;}3 zAUfl!(D9NuPw-MzMJtI^DycfIXesAcsahggB0FC`e_0l=#Fm1U5}!W$Hb7mJ)C~$c zRkub>OH9&4iqeQQ%=UzSH^Q!sBp_6{%{Ta?$#ae_PcqC%UeOR`{~QRq2D5f(rde4C zoclgxGA9(fV6q?eh|DQUEm{1UuE@|WR0Du(oagR{yCyp@q?j&YITHsObK1O*HR&>d zaZh|J#%f7OGG%SgJ&Z58<&QxuO+bsk@C}a?l_8H;3Bc}yGe*;W7~QEzQgTF<_rOO4 z85S_{JO2ZBI#02ya?hDfl=z}EWe;pl0%4w!#kB1Jn))>1ltVZCgc*TVtnT{Dtzk#L z!bR0&_+l6J6Plu!4^!!YG0E@DDFPup&)cQTs%^kAr5|0_oz0P|HvIsT?4y24TH`=^ z#fW}IxI>W`Lx1q4=9Wk+=;C%k zAot{^n^$cP&H#m_K0#I&fuba2f!nI8NySUuONR}%rW^QrF#H>~NGpt`JlK8L<~74| zjw2x+TC4mD;s}t~>kTWS2d_>%_(j69z>_ zo~=X{Ti5RwIX}~w;LYN7%TSDBQ^{yUG04GrL&%Kmuu)++n#*qZ_gp+Dc0gcmUB6a? z`Zxrb8Rg?0t2US#*9bHj;2(0h6_R0%j0)*c(2WufG?Ku14>_T>3k~;26O+tWp0e1B z+*3k14M6aM$?&)J;2I4sEgq~~jB(}wH#liL{wgDq`&GaB2LkZ)17Cgz9B?83dbShv z8i}Af{^K&Yi)MfWou73G@>R_bX*#zEA;iWHnG0=*c@h*495C1aYftGHQqkJZWIH6r z9;vER^__qy2~&zEn<%fcBgq zI)}``6vQ=rt8(4GDHXcdjxu{BjK2_y?*lst2-Os^eI+Cmj-&nkvbrYFmN;zxj1$Xq zdj}dZRF8bQ_nY+8(wI?GFckJM7WIUlwahT>2xV4idnjp;In>0!9l$WkYe4-oq3I>R z9^`c{d2Fxfjj++oRn(vwbXQ<*&}I!`VNr4U+P7fj9l-{*amxgnIBaSKpw|;Xb}r`q z;!T&x5pGs3%BcE~hV0Q5D;Jjm%~7YRYz`jFX9q4<(^g6*OHoQx#m?cQcB_tsLqbH$ zDrr;_QjEAPlX@mh$&@aX6fEM%D9n%Iz63%a=q1$=2VD?DxzvtIHMLB$s&GS)`-Q~S z)F4_s_D>V{Ycb}t!l(c29_-j92|5StN6_UM+xfwnA{Q14FD2KTo{Y@Bgq79o=!u~Q z*VoOY!={ZN(bx zR<7FU49;r-~|;kGt{x$Nfq{5LWasZPMAb zIcSUupGh7$?PS&(bM%8%fpwVQ=C2Egq(kMVhOy0}F%5<{oMgyINu>*+>p{Ye!ci1q z35?!tTr|~Duq_l4#b7W2+cDWO%Tfg_6=JDi+0MG+xG_<()+-5yIbvxAD8Ka{kL52C z<$AOj(mfL8GUc`%-w%PW)y8<-xl2BylDm?Krbc0uo$I|c!IP8jofGZglp^k&pjdfm zSS^MewOFMh{*{%EBB!@gNL*{e_mr$cSV-?-sW&WNsUNiDz>p}WP{wDVrkB*F@gT$S3WLNCh z>@aE4rnQfOq9P);fkHxvc82)y;Bul1Zfr&})hajvQBm8rae#kBiTV)x2z^wFFb%}V=ap_Fp%HB5!W*XVy& zP%1l8~X}I~)7vRXX(Njx_us!!OVlaQrH_54!wjo;k=ywEt6bE5APCCN6dlozowDcg!J9nGf= z%L4G2e6S1gkhD-e&5T_SH;1y29pbu3tX-C_Pc@44qyAQ@2JRtMj)jvh$KN)Li9M2c zXgxKZqosVeS0li&vJ<&zN?IHK;WM*T|HEg`$WLt2h>h%pY$>cntCNEz)UTB`gdEN` z)M4TugJxo)GVTa4q1zCkoC2(y2;WyfTY~UUDQW$sU%ziE{|j-hthHo}GrBs(UE<@+LJCE4LM@=Y+%gh#Qpk}1lOca8|;veGP5ank}7%yEb z1y3@J#J!%-NOfNvN-D$1rnRu)9oBgNV;4!mbY{G2@-5G$mwONPVmKj{8Z&Ppj7ZL- z3w6rTf|)2DjXC2(fk`5({-~BHjYg1`p2KD=3Sav|+wuyfL%uf>zL%lNMx4djO?896 zx}ruay1{Dzfw?qb7MCp0;D7Af+Up}GC35%Mc{;of#aSArmJ-(tKdDf%D zzUMA7JzG*vNollF%O=Xi)_E#f~nu3+2X zau8VjNYnsZPQ;?p{M=l8iZyNcvs&FO$^>{c*h~T=ayW33y}z|1~tF%8ysl{5J@N;-wpQ|*Mj zLu*3YH^tergs)gr`BzOsHA|04*%K?d3?Kr$PR}~@%d)=smazy*j&;DIF@@CGa})-D zwtY+E3J_*4r>Xm$hUSvPA>}=j)3Nwaa-uOyH7lW!VplvKCvU&mWVaIZ(uijcKEsD_ zz>EBCpQntii>{5XgmMY(DyHk8kL!uiE%Ot(hxV`oOOUQmE^jFxoCEnIhQ`ipYx;Ur zkS2keG5JW}?e*JIUNlxqlavlxpK1%Wi`!Y|)C8$h#9hoX$m3JYg6Z>jITMYg2VqqN zZt}2`qon#SKJor|`7k~<@-H`2FrNGVTUX2wZ^+y#fE0TLR27-JB!a1EzN<4LRAi*O zPE3MQ3BO8%-bvYg1ci!xo~+RZ+%-Fu?TVDprA)u2ikXR#5gW0Du@Z!(RJ52=`bk#) z#CYvR+6+SlJjF=zqC?)Tj|j1Rk7Kr97MI<-oz9CxS0&Nc8iph>Qbgt|zKK&(T#JnG z1{P5$Zx;?MCWjbM(e?Z`PFp`f%uBXaqmu(bBVELlIC)x9NxXF^qyy|;;U?By=`HsY zcn$c*c$y6iWa?+ydp=#QcJx-1RCj!J*@GAeakS47^2CM6KwNBglbtRg6G9VW9mADg zjk_Q&Y8<;&Y$#I)36n6wg?MtAb5-bDlkW{fdaAy zBtAOkDJ?=w<1aB=85eQ5rXu#@)f>v*Z+*r|nx(%=dq}lhZ5`$q!hg5ogX&@NUrzIgb zX3Po!!~vX;CQJqD8K+pr_)f;1Cs{apL$BLX$d%j{7^yrX)6&SbIi9Ze^h7F)H@>%L z8~o(hiTj?*jX&>fX$k8Wsj&n*Q-9~LJVq!7agl-Wo|6rYiHgP?xi=YOqha2HKc$e@BJO zjA>igb1aHKP_cZ?M7JWF7>jbc)7`g(C8_L@@-8H&T};x*QBCZjROBTi3#0(xo)62F zh{$u3*k~zCq{(S3U<>!s^%`E0m6#H3b((kTot&#rWIQLx#GVHE+H7)pdd?a~)LL^p zW8l4Q)!q8fvu1pwl1B3PTk%NOrmg4g=L*d|i8^>YTSR5jthUzeuY|7LwZ{u<=IEDO zOhQP@mprbJ{{*h?5#oxBJ5Lo+OO>Fzs>WPI?>JXOX&D!bXK3v3-X;f7fck~AQ}W2D z{A+4kcaOREks=<}6McNl?$(v%{Yd!XA7RUG!6Ci$hFnyjPB>*Ar*oBoD7hgWCsvb` zDRHZgSH$W-7L>zxFbmWq?OMUcuQ znNiN6g61(wQeVF#x+>;N6CesYQYK;oZ9WNht&ZQc>tn>1kCLp!!}Df5i3j9n4|Mw& zf?~>J(<6d@MyI$db4h4Hv~6Nv!+_RLJ(vw7E1_sB&&4lfzD(TY*cJrkI>yp!&F|HR zN8q@})@j{Q9KTeggWL>ffbYXQr+qcljiqJg4f?@j7?c?7&2uY9E-f$*P60BI%#J%x zw<_{r{tgndnKQSl+xFnTEdCczVT|@xI;zOoKmeX7BO?-wYE-(RzT3Z0XZ8gep;`hg>4n5URiX z>DEYR;$I4zR4pyH^8X|40=$sBENDSX}}~jZ+h+&ZqNF4{u?i{mr(@ zp_Bre7>?mDGNG&_&2nn!HeN?(#{^g1Y;JlYDkh0MyXn};*jahSs^7}tf34wV7eSX%eC*H)sH8!wQcy@VW6aKxkO2wG!qm5hL!5)e@kxl@B;lql z9@53e^Sc3%JZ^YQo2H|Fa&DJPrn58S%=J9+EKu7kSL*?DUAz6Sf(SmHG>*yJ)xp^YyY3!zBOc94~4GW7qz88skpw7tOW*JJv+Vs>;4b56e8e1*u19IVTYUeW<|0<$pz!g2dn#%HxS4UMghe^i zg$JXDP^DvrXf2nyo42QM@nB2ck~T@&gYMn85hp#HWnY;Fn5#m zdA_yPf$Xm3A*3QdbAHGcFmT!`|8)h&5l_i`33m;H4}kLY{uq0*x85(vi~%C9mtLW62$E-MG=!rQa!Ys z5Q#7FcQ3pC#J7P|HhMp0chwwndPyCsHkf&eR>AUZBDF{XTYR~G>72GD%x2*n_Z=8N za}kK4yke>FMFIDzdUqDPbG;sq9VOnJzXj%ID9l1l6lVNGW0H`%hjRaB*m? zqGgwF=rL6j==Mqvc%--xg=9&ul;Zc2h%e`Eup>4cLb95cTNC|yfQL(qu^P^YcX{#MDRQ8qyq zlp1BLv-GBu)3Ck|u;wmeSFdDIqiKUSo=MBwp#8jC=K1?J&Gz!IpBNBaJ=DcLzpY4o zd|nj!em~9sb7lO0PfuFnwJztOX0BBRE&^#1SZmkK5yLXq95>}|6sr_u$l)LSM*J8g zEpB&En>zMCX-c>_P_rzZMaYI9^(x|0E)a{K=Km3A z^n0KT5%+V!E+gUkVGS|=OmpVW91dvLDqsSia^b?8e9EYvlO@Qjqs#Vh7H0Xr;L7PU z?(35JZ`$3FJ7aD1Gg`RFcWQ$(XXL|Er9ewymPvbY_B+$>dF3%Ht#qgB@$2>Jzwuf4 z@?pkJ8!GN7%*L6ITn`&ZA{&M+uMiAgE@7$EW{MeCLSDSK82Cg#Lu4XHH-R8l8L-0h zoyJ%4Vog5CYFlySA2vU_810IJ_%VV5UUZuSWCfivZu6j2h<>M{N zB?M<}FzWr=NCxJlcF%;Vs+E$2^#M!Wc@2GAf7aV5{cNev)sf1@>d==ZhPqvS+K2V; zz-txf^V!aXMA94NSF`iow+^5?Hx1*i$0m6`2igJTW|E_&w%DTGMwaK$U*{`&T$JyW#Z4ZJzv`nO7Z*iPqZxIU7mAk# z<0jhb``ZqU)cnu7`WHzC!F2S)VbNvizBf?`;65|MEg_N3zI=~0W)Lqa4BV#ngVJW- z^w-VWGmB#@84TDL?Zpn{Uqu_sI9kjt22~!Vu{MLoizlt-W@IbR=Srh8zVjQ-`aM?s z4t^;Um@Xa&{}!VLV!Q|i1K4)ZLf=T~W~n}|r+dI`OWkP67}bM$8QIjYW7+b($f+q= z+Fo9&7S%G~ziA{zcw{ z>_cNI*{i?c<(&~;zoqem8uvtQcC~W%nz3_n@?*9A@@8I}2$v%rdoiijqi?aBODF=B zmPV8i$AOfW=R1DzD_zCxCnU&bX^d|2nI6Ny;b;4#0XgdF>%QDyx8=R6 ztwit3t-92&taeHNy425U!(IN>M26|=+~eoF%^GuVhxq#EZ8PhuF+u?2#fhD;j$yXR zIjZs7#_Qx@vC9AV3f0<%h5t^awd#Hm!z4mkFy~`@`sej_f%l6C5iB6LIQ5Tb=#`DXIj=+2M zvO9O}q3^dF-Ur_?=j`9*pOh9~_3tZ4kXdvPHfL&zWAsg$FFt@CIf2b$z*Qm%5sb5= zMe-dNtc;b%$(ZoiFSprTpkZhW@D}*NRN9c4z`aWZ3gYMh+pauD!|s(^7b%k1wAdT5 zPrl@v8xRd2vRR+rRt3h&fVvs6Zl}EsFg@uOGj6#fto7C~dF2^xxCyG<8igJC^&uVI zO_0UgwN?i}xp6CvoAK+-qx{L|W!-t%L1TyoO9mr_Xqcn*k!!+1w92|Rfl(QaxrjnvFI8i-J6;^T&nbr zO`P9wD%=;cxqlTmCag1$pq?l>Y?sQTS$$05Qyy$=jw))~+32S4RV^ADtqK~geMEet z8e?4-nl$a*@)7@bRvuAXG}_qvR%Y)UIxkJ6j2$@|ndpfmsnlB4xGZ&M>KsmBOC%LF zfPrP|=%!3N%fuAvkF*Rsi!@&jC}C%1tIbGTxVVU@G+vh1CE~66+nB{%Dt9<9Ptw&q z$$KVLm4FZDXl^JD7h~yN^f4)J2TuHxph4%l(9xAYtQ9S6wz^Zy$SO7}XdDk*w6SvL z);Ab3D>7cK0f9U+-D$9u3 zx2<$pc8M(weOs*od8)G=4m@u+SEC=@1pnJpJ`%!G?ev$uPz?L9y*5OCrmy=l-ew-x zIS0i~z@pn0clKD@S+oz1(Hl`JsiMg^XF{tR`No6RlTDfslRfg--Doz&mIkty` z$X7l0I!Ch;T(eq{MIj>0+Ef7v%TYgyUWy#tl#mFg47N!IXU?L|;hEg&7q`Ts5BrHg zq;K}LF02TJIBp!ZX%3Sdy;IXz#QkH${o(m}fi9bcU89QltHDClQ*YYE!1j_2$1CcdvuxeewGVF-m&_9^!7Jo8)Cl!#9fUu*254 zn7nWni zG#Bi1<@NnMyLRE(#WTXqbPmnPoE!W)YhB{wgm}B+>xmVM|Gijs?C2)_C!*ZEG99;* zol5UNi8vR1;Z@GF!wK8gl$c%BHqHq8^6bo6Q{J^(LnT&o$DVWDr}BOFOJln)ON2kW zg~lhnnXA0&=;gvx@A>^VkH~agyDI)}+Uh&n3sSh3D$5+dXf(_4%#3OJ%TG4Dr#GkjJXP*fD%?55XR-AwGz%-$!${k^K1vwAG zC*QwZ*TA;ORYto{@OZcKk%^09_qgbYx${n#Q|A~hX6)0J`1e56efQGO?0t?-4~!M% z`lFcFUISip`RhMJAIpWkm`4PGkI{jiTY~SP5@Q3@yXj*mp3gdLz;o<<9hdxWXU{pu zV%Zfhq-sCeV6StVdC9g*l48d#-YMqWM`u~jP;8vP#DD&TP0MPt)^gfZf1mtH;lsg* zw~6!4#WC6+IvJtHwXT;-{Lp@>>f@U<-c7SSaN|~DQs~jjZ-SePKXAvt{aVsuJ#B9# z@4Xws@wKM0Pu{(`!{3v@d_FR^nAiEumrJ{^*Rjl7m~;2l4D-ZD_p|XwpFe64Khd-- zaB|ztuUs|x{s&gs?3)$d{v(Sm?EmR}&r+EyeKQ|4yJNtP2; zv?iRAy%Z)S+adL3Ps#M~^|v#QGS7T_GS&OmC$Ae#Yfig!nTgy<4Cs2zBC%wm>PgkJ z#rk@}6Vjs`tghtle)Xm{Mkh|#d{$Ye&K2@jmqJr#WC%QM5LwL7$ie`4Y!QkW z&@CoL=;j$)8kiYir~{q6gQC~a)X)TYND-Qtu@So849$V+(EJMwD>G9xJAh&q7U=#3 znvU)dkcDPw<{21RnpvQQJy6UX(>&nOOlbBR7#bLuqMK)EUO2*qzej;R^&Y#|VU z#W_$f@X#eRF(X4W4D*bP3^439v9Pd2_pgbiu@QP4nOd5o$2l;93@p*&7np{O(BlZ` zK41)^xzErL)4xW*gQn2UGXkbHbhnsTS{R`F*VMqk7+sy2fw2L)y=Dd`mPqPKiV}fm xO%;I-2@B4wN(E&>{h<8(65#nx;KQeYXK@vmBo>u`b0{zo0W%?&s;aBM8vrN_85{rr From aec24cf1a72b03a4afaa1963b36d6ecf2f74e69e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 16:16:07 -0700 Subject: [PATCH 378/688] readme --- uncore/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/README.md b/uncore/README.md index 003f9caa..6e67a691 100644 --- a/uncore/README.md +++ b/uncore/README.md @@ -8,4 +8,5 @@ These components are only dependent on Chisel, i.e. lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) -Documentation about the uncore library will come in the near future. +Documentation for the uncore library is available here +and an overview of the TileLink Protocol is available here From fc883b5049553d81fd932d4838c53aace5cd74ba Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 7 May 2015 16:17:40 -0700 Subject: [PATCH 379/688] rm index.html --- uncore/index.html | 1 - 1 file changed, 1 deletion(-) delete mode 100644 uncore/index.html diff --git a/uncore/index.html b/uncore/index.html deleted file mode 100644 index 03f98016..00000000 --- a/uncore/index.html +++ /dev/null @@ -1 +0,0 @@ -My GitHub Page From 172c372d3efdaa304774f7691cd74bf5377b6a80 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 12 May 2015 17:14:06 -0700 Subject: [PATCH 380/688] L2 alloc cleanup --- uncore/src/main/scala/cache.scala | 4 ++-- uncore/src/main/scala/uncore.scala | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 7fcd1408..21f0cd42 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -836,7 +836,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // We write data to the cache at this level if it was Put here with allocate flag, // written back dirty, or refilled from outer memory. pending_writes := (pending_writes & dropPendingBit(io.data.write)) | - addPendingBitWhenBeatHasData(io.inner.acquire) | + addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) val curr_write_beat = PriorityEncoder(pending_writes) @@ -887,7 +887,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { SInt(-1, width = innerDataBeats), (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) - pending_writes := addPendingBitWhenBeatHasData(io.inner.acquire) + pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) pending_meta_write := UInt(0) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d7573aec..b7d06141 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -114,6 +114,9 @@ abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) + def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData() && in.bits.allocate(), in.bits) + def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { val a = in.bits val isGetOrAtomic = a.isBuiltInType() && From a7fa77c7fc49da7957a4e2d85b23a458d12e106b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 13 May 2015 23:28:18 -0700 Subject: [PATCH 381/688] track operand size for Gets --- uncore/src/main/scala/consts.scala | 1 + uncore/src/main/scala/tilelink.scala | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 69cdc41f..54000208 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -16,6 +16,7 @@ trait MemoryOpConstants { val MT_BU = Bits("b100") val MT_HU = Bits("b101") val MT_WU = Bits("b110") + val MT_Q = Bits("b111") val NUM_XA_OPS = 9 val M_SZ = 5 diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ca80f3c3..f8161783 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -283,6 +283,8 @@ object Acquire { * @param client_xact_id client's transaction id * @param addr_block address of the cache block * @param addr_beat sub-block address (which beat) + * @param addr_byte sub-block address (which byte) + * @param operand_size {byte, half, word, double} from [[uncore.MemoryOpConstants]] * @param alloc hint whether the block should be allocated in intervening caches */ object Get { @@ -297,7 +299,22 @@ object Get { client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - union = Cat(M_XRD, alloc)) + union = Cat(MT_Q, M_XRD, alloc)) + } + def apply( + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + operand_size: UInt, + alloc: Bool): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = Acquire.getType, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + union = Cat(addr_byte, operand_size, M_XRD, alloc)) } } From 90c9ee7b04e402c1c08b4aa509d5f46056fb1b1b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 14 May 2015 12:37:35 -0700 Subject: [PATCH 382/688] fix unalloc putblocks --- uncore/src/main/scala/cache.scala | 18 ++++++++++-------- uncore/src/main/scala/htif.scala | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 21f0cd42..ab287033 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -711,7 +711,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { xact.conflicts(io.iacq()) && state != s_idle && state != s_meta_write && !all_pending_done && - xact.allocate() && + (xact.allocate() || xact.isBuiltInType(Acquire.putBlockType)) && !io.inner.release.fire() && !io.outer.grant.fire() && !io.data.resp.valid && @@ -766,14 +766,16 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := state === s_outer_acquire + io.outer.acquire.valid := state === s_outer_acquire && + (xact.allocate() || !pending_puts(oacq_data_idx)) io.outer.acquire.bits := Mux( - xact.allocate(), - xact_old_meta.coh.outer.makeAcquire( - client_xact_id = UInt(0), - addr_block = xact.addr_block, - op_code = xact.op_code()), - Bundle(Acquire(xact))(outerTLParams)) + xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( + client_xact_id = UInt(0), + addr_block = xact.addr_block, + op_code = xact.op_code()), + Bundle(Acquire(xact))(outerTLParams)) + io.oacq().data := data_buffer(oacq_data_idx) // Handle the response from outer memory io.outer.grant.ready := state === s_busy diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8a79cda1..5c1bc4ef 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -180,7 +180,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { addr_block = init_addr, addr_beat = cnt, client_xact_id = UInt(0), - data = mem_req_data), + data = mem_req_data), GetBlock(addr_block = init_addr)) io.mem.grant.ready := Bool(true) From c202449e34ff593ac653b3a2368e8c9a457bfcca Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 14 May 2015 15:29:49 -0700 Subject: [PATCH 383/688] first version NASTI IOs --- uncore/src/main/scala/nasti.scala | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 uncore/src/main/scala/nasti.scala diff --git a/uncore/src/main/scala/nasti.scala b/uncore/src/main/scala/nasti.scala new file mode 100644 index 00000000..d0ce1657 --- /dev/null +++ b/uncore/src/main/scala/nasti.scala @@ -0,0 +1,89 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +case object NASTIDataBits extends Field[Int] +case object NASTIAddrBits extends Field[Int] +case object NASTIReadIdBits extends Field[Int] +case object NASTIWriteIdBits extends Field[Int] + +trait NASTIParameters extends UsesParameters { + val dataBits = params(NASTIDataBits) + val strobeBits = dataBits / 8 + val addrBits = params(NASTIAddrBits) + val widBits = params(NASTIWriteIdBits) + val ridBits = params(NASTIReadIdBits) + val userBits = 32 + val awUserBits = userBits + val wUserBits = userBits + val bUserBits = userBits + val arUserBits = userBits + val rUserBits = userBits +} + +abstract class NASTIBundle extends Bundle with NASTIParameters +abstract class NASTIModule extends Module with NASTIParameters + +trait NASTIChannel extends NASTIBundle +trait NASTIMasterToSlaveChannel extends NASTIChannel +trait NASTISlaveToMasterChannel extends NASTIChannel + +class NASTIMasterIO extends Bundle { + val writeAddress = Decoupled(new NASTIaw) + val writeData = Decoupled(new NASTIw) + val writeResponse = Decoupled(new NASTIb).flip + val readAddress = Decoupled(new NASTIar) + val readData = Decoupled(new NASTIr).flip +} + +class NASTISlaveIO extends NASTIMasterIO { flip() } + +class NASTIaw extends NASTIMasterToSlaveChannel { + val awID = UInt(width = widBits) + val awAddr = UInt(width = addrBits) + val awLen = UInt(width = 8) + val awSize = UInt(width = 3) + val awBurst = UInt(width = 2) + val awCache = UInt(width = 4) + val awProt = UInt(width = 3) + val awQOS = UInt(width = 4) + val awRegion = UInt(width = 4) + val awLock = Bool() + val awUser = UInt(width = awUserBits) +} + +class NASTIw extends NASTIMasterToSlaveChannel { + val wData = UInt(width = dataBits) + val wStrb = UInt(width = strobeBits) + val wUser = UInt(width = wUserBits) + val wLast = Bool() +} + +class NASTIb extends NASTISlaveToMasterChannel { + val bID = UInt(width = widBits) + val bResp = UInt(width = 2) + val bUser = UInt(width = bUserBits) +} + +class NASTIar extends NASTIMasterToSlaveChannel { + val arID = UInt(width = ridBits) + val arAddr = UInt(width = addrBits) + val arLen = UInt(width = 8) + val arSize = UInt(width = 3) + val arBurst = UInt(width = 2) + val arCache = UInt(width = 4) + val arQOS = UInt(width = 3) + val arRegion = UInt(width = 4) + val arProt = UInt(width = 4) + val arLock = Bool() + val arUser = UInt(width = arUserBits) +} + +class NASTIr extends NASTISlaveToMasterChannel { + val rID = UInt(width = ridBits) + val rData = UInt(width = dataBits) + val rResp = UInt(width = 2) + val rUser = UInt(width = rUserBits) + val rLast = Bool() +} From f3a838cedf51994bc186227e85905a0e41a59ba1 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 19 May 2015 18:40:34 -0700 Subject: [PATCH 384/688] nasti converters, hub bugfix --- uncore/src/main/scala/broadcast.scala | 2 + uncore/src/main/scala/cache.scala | 2 +- uncore/src/main/scala/nasti.scala | 320 +++++++++++++++++++++----- uncore/src/main/scala/tilelink.scala | 7 +- 4 files changed, 270 insertions(+), 61 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index b554b231..4dcc339a 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -111,6 +111,8 @@ class L2BroadcastHub extends ManagerCoherenceAgent io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + io.outer.acquire.bits.union := Cat(Fill(outer_arb.io.out.acquire.bits.union(1), io.outer.acquire.bits.tlWriteMaskBits), + outer_arb.io.out.acquire.bits.union(0)) io.outer <> outer_arb.io.out // Update SDQ valid bits diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ab287033..4c2b1a88 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -641,7 +641,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // Provide a single ALU per tracker to merge Puts and AMOs with data being // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU) - amoalu.io.addr := xact.addr() + amoalu.io.addr := xact.full_addr() amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData diff --git a/uncore/src/main/scala/nasti.scala b/uncore/src/main/scala/nasti.scala index d0ce1657..904a596e 100644 --- a/uncore/src/main/scala/nasti.scala +++ b/uncore/src/main/scala/nasti.scala @@ -2,24 +2,43 @@ package uncore import Chisel._ +import scala.math.max case object NASTIDataBits extends Field[Int] case object NASTIAddrBits extends Field[Int] -case object NASTIReadIdBits extends Field[Int] -case object NASTIWriteIdBits extends Field[Int] +case object NASTIIdBits extends Field[Int] trait NASTIParameters extends UsesParameters { - val dataBits = params(NASTIDataBits) - val strobeBits = dataBits / 8 - val addrBits = params(NASTIAddrBits) - val widBits = params(NASTIWriteIdBits) - val ridBits = params(NASTIReadIdBits) - val userBits = 32 - val awUserBits = userBits - val wUserBits = userBits - val bUserBits = userBits - val arUserBits = userBits - val rUserBits = userBits + val nastiXDataBits = params(NASTIDataBits) + val nastiWStrobeBits = nastiXDataBits / 8 + val nastiXAddrBits = params(NASTIAddrBits) + val nastiWIdBits = params(NASTIIdBits) + val nastiRIdBits = params(NASTIIdBits) + val nastiXIdBits = max(nastiWIdBits, nastiRIdBits) + val nastiXUserBits = 1 + val nastiAWUserBits = nastiXUserBits + val nastiWUserBits = nastiXUserBits + val nastiBUserBits = nastiXUserBits + val nastiARUserBits = nastiXUserBits + val nastiRUserBits = nastiXUserBits + val nastiXLenBits = 8 + val nastiXSizeBits = 3 + val nastiXBurstBits = 2 + val nastiXCacheBits = 4 + val nastiXProtBits = 3 + val nastiXQosBits = 4 + val nastiXRegionBits = 4 + val nastiXRespBits = 2 + + def bytesToXSize(bytes: UInt) = MuxLookup(bytes, UInt("b111"), Array( + UInt(1) -> UInt(0), + UInt(2) -> UInt(1), + UInt(4) -> UInt(2), + UInt(8) -> UInt(3), + UInt(16) -> UInt(4), + UInt(32) -> UInt(5), + UInt(64) -> UInt(6), + UInt(128) -> UInt(7))) } abstract class NASTIBundle extends Bundle with NASTIParameters @@ -30,60 +49,247 @@ trait NASTIMasterToSlaveChannel extends NASTIChannel trait NASTISlaveToMasterChannel extends NASTIChannel class NASTIMasterIO extends Bundle { - val writeAddress = Decoupled(new NASTIaw) - val writeData = Decoupled(new NASTIw) - val writeResponse = Decoupled(new NASTIb).flip - val readAddress = Decoupled(new NASTIar) - val readData = Decoupled(new NASTIr).flip + val aw = Decoupled(new NASTIWriteAddressChannel) + val w = Decoupled(new NASTIWriteDataChannel) + val b = Decoupled(new NASTIWriteResponseChannel).flip + val ar = Decoupled(new NASTIReadAddressChannel) + val r = Decoupled(new NASTIReadDataChannel).flip } class NASTISlaveIO extends NASTIMasterIO { flip() } -class NASTIaw extends NASTIMasterToSlaveChannel { - val awID = UInt(width = widBits) - val awAddr = UInt(width = addrBits) - val awLen = UInt(width = 8) - val awSize = UInt(width = 3) - val awBurst = UInt(width = 2) - val awCache = UInt(width = 4) - val awProt = UInt(width = 3) - val awQOS = UInt(width = 4) - val awRegion = UInt(width = 4) - val awLock = Bool() - val awUser = UInt(width = awUserBits) +trait HasNASTIMetadata extends NASTIBundle { + val addr = UInt(width = nastiXAddrBits) + val len = UInt(width = nastiXLenBits) + val size = UInt(width = nastiXSizeBits) + val burst = UInt(width = nastiXBurstBits) + val lock = Bool() + val cache = UInt(width = nastiXCacheBits) + val prot = UInt(width = nastiXProtBits) + val qos = UInt(width = nastiXQosBits) + val region = UInt(width = nastiXRegionBits) } -class NASTIw extends NASTIMasterToSlaveChannel { - val wData = UInt(width = dataBits) - val wStrb = UInt(width = strobeBits) - val wUser = UInt(width = wUserBits) - val wLast = Bool() +trait HasNASTIData extends NASTIBundle { + val data = UInt(width = nastiXDataBits) + val last = Bool() } -class NASTIb extends NASTISlaveToMasterChannel { - val bID = UInt(width = widBits) - val bResp = UInt(width = 2) - val bUser = UInt(width = bUserBits) +class NASTIAddressChannel extends NASTIMasterToSlaveChannel with HasNASTIMetadata + +class NASTIResponseChannel extends NASTISlaveToMasterChannel { + val resp = UInt(width = nastiXRespBits) } -class NASTIar extends NASTIMasterToSlaveChannel { - val arID = UInt(width = ridBits) - val arAddr = UInt(width = addrBits) - val arLen = UInt(width = 8) - val arSize = UInt(width = 3) - val arBurst = UInt(width = 2) - val arCache = UInt(width = 4) - val arQOS = UInt(width = 3) - val arRegion = UInt(width = 4) - val arProt = UInt(width = 4) - val arLock = Bool() - val arUser = UInt(width = arUserBits) +class NASTIWriteAddressChannel extends NASTIAddressChannel { + val id = UInt(width = nastiWIdBits) + val user = UInt(width = nastiAWUserBits) } -class NASTIr extends NASTISlaveToMasterChannel { - val rID = UInt(width = ridBits) - val rData = UInt(width = dataBits) - val rResp = UInt(width = 2) - val rUser = UInt(width = rUserBits) - val rLast = Bool() +class NASTIWriteDataChannel extends NASTIMasterToSlaveChannel with HasNASTIData { + val strb = UInt(width = nastiWStrobeBits) + val user = UInt(width = nastiWUserBits) +} + +class NASTIWriteResponseChannel extends NASTIResponseChannel { + val id = UInt(width = nastiWIdBits) + val user = UInt(width = nastiBUserBits) +} + +class NASTIReadAddressChannel extends NASTIAddressChannel { + val id = UInt(width = nastiRIdBits) + val user = UInt(width = nastiARUserBits) +} + +class NASTIReadDataChannel extends NASTIResponseChannel with HasNASTIData { + val id = UInt(width = nastiRIdBits) + val user = UInt(width = nastiRUserBits) +} + +class MemIONASTISlaveIOConverter extends MIFModule with NASTIParameters { + val io = new Bundle { + val nasti = new NASTISlaveIO + val mem = new MemIO + } + + require(mifDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") + val (mif_cnt_out, mif_wrap_out) = Counter(io.mem.resp.fire(), mifDataBeats) + + io.mem.req_cmd.bits.addr := Mux(io.nasti.aw.valid, io.nasti.aw.bits.addr, io.nasti.ar.bits.addr) >> + UInt(params(CacheBlockOffsetBits)) + io.mem.req_cmd.bits.tag := Mux(io.nasti.aw.valid, io.nasti.aw.bits.id, io.nasti.ar.bits.id) + io.mem.req_cmd.bits.rw := io.nasti.aw.valid + io.mem.req_cmd.valid := (io.nasti.aw.valid && io.nasti.b.ready) || io.nasti.ar.valid + io.nasti.ar.ready := io.mem.req_cmd.ready && !io.nasti.aw.valid + io.nasti.aw.ready := io.mem.req_cmd.ready && io.nasti.b.ready + + io.nasti.b.valid := io.nasti.aw.valid && io.mem.req_cmd.ready + io.nasti.b.bits.id := io.nasti.aw.bits.id + io.nasti.b.bits.resp := UInt(0) + + io.nasti.w.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.nasti.w.valid + io.mem.req_data.bits.data := io.nasti.w.bits.data + assert(!io.nasti.w.valid || io.nasti.w.bits.strb.andR, "MemIO must write full cache line") + + io.nasti.r.valid := io.mem.resp.valid + io.nasti.r.bits.data := io.mem.resp.bits.data + io.nasti.r.bits.last := mif_wrap_out + io.nasti.r.bits.id := io.mem.resp.bits.tag + io.nasti.r.bits.resp := UInt(0) + io.mem.resp.ready := io.nasti.r.ready +} + +class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val nasti = new NASTIMasterIO + } + + val dataBits = tlDataBits*tlDataBeats + val dstIdBits = params(LNHeaderBits) + require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction + require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") + require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIMasterIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + + io.nasti.b.ready := Bool(false) + io.nasti.r.ready := Bool(false) + io.nasti.ar.valid := Bool(false) + io.nasti.aw.valid := Bool(false) + io.nasti.w.valid := Bool(false) + + val dst_off = dstIdBits + tlClientXactIdBits + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() + val is_write = io.tl.release.valid || (io.tl.acquire.valid && acq_has_data) + + // Decompose outgoing TL Acquires into NASTI address and data channels + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val tag_out = Reg(UInt(width = nastiXIdBits)) + val addr_out = Reg(UInt(width = nastiXAddrBits)) + val has_data = Reg(init=Bool(false)) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + + io.nasti.ar.bits.id := tag_out + io.nasti.ar.bits.addr := addr_out + io.nasti.ar.bits.len := Mux(has_data, UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.size := UInt(log2Ceil(tlDataBits)) + io.nasti.ar.bits.burst := UInt("b01") + io.nasti.ar.bits.lock := Bool(false) + io.nasti.ar.bits.cache := UInt("b0000") + io.nasti.ar.bits.prot := UInt("b000") + io.nasti.ar.bits.qos := UInt("b0000") + io.nasti.ar.bits.region := UInt("b0000") + io.nasti.ar.bits.user := UInt(0) + io.nasti.aw.bits := io.nasti.ar.bits + io.nasti.w.bits.strb := Mux(data_from_rel, SInt(-1), io.tl.acquire.bits.wmask()) + io.nasti.w.bits.data := Mux(data_from_rel, io.tl.release.bits.data, io.tl.acquire.bits.data) + io.nasti.w.bits.last := tl_wrap_out + + when(!active_out){ + io.tl.release.ready := io.nasti.w.ready + io.tl.acquire.ready := io.nasti.w.ready && !io.tl.release.valid + io.nasti.w.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.nasti.w.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := (!is_write && !io.nasti.ar.ready) || + (is_write && !(io.nasti.aw.ready && io.nasti.w.ready)) || + (io.nasti.w.valid && Bool(tlDataBeats > 1)) + io.nasti.aw.valid := is_write + io.nasti.ar.valid := !is_write + cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + io.nasti.w.bits.data := io.tl.release.bits.data + io.nasti.w.bits.strb := SInt(-1) + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.full_addr() + io.nasti.aw.bits.id := tag + io.nasti.aw.bits.addr := addr + io.nasti.aw.bits.len := UInt(tlDataBeats-1) + io.nasti.aw.bits.size := MT_Q + tag_out := tag + addr_out := addr + has_data := rel_has_data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + io.nasti.w.bits.data := io.tl.acquire.bits.data + io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.full_addr() + when(is_write) { + io.nasti.aw.bits.id := tag + io.nasti.aw.bits.addr := addr + io.nasti.aw.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.putBlockType), + UInt(tlDataBeats-1), UInt(0)) + io.nasti.aw.bits.size := bytesToXSize(PopCount(io.tl.acquire.bits.wmask())) + } .otherwise { + io.nasti.ar.bits.id := tag + io.nasti.ar.bits.addr := addr + io.nasti.ar.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.getBlockType), + UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() + } + tag_out := tag + addr_out := addr + has_data := acq_has_data + } + } + } + when(active_out) { + io.nasti.ar.valid := !cmd_sent_out && !has_data + io.nasti.aw.valid := !cmd_sent_out && has_data + cmd_sent_out := cmd_sent_out || io.nasti.ar.fire() || io.nasti.aw.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.nasti.w.ready + io.nasti.w.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.nasti.w.ready + io.nasti.w.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(cmd_sent_out && (!has_data || tl_done_out)) { active_out := Bool(false) } + } + + // Aggregate incoming NASTI responses into TL Grants + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.nasti.r.bits.id(0), + g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.nasti.r.bits.id >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.nasti.r.bits.data) + + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) io.nasti.b.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(io.nasti.b.bits.id(0), Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = io.nasti.b.bits.id >> UInt(1), + manager_xact_id = UInt(0)) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index f8161783..31f0b136 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -172,7 +172,7 @@ class Acquire extends ClientToManagerChannel /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) /** Complete physical address for block, beat or operand */ - def addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) + def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) // Other helper functions: /** Message type equality */ @@ -337,7 +337,7 @@ object GetBlock { a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - union = Cat(M_XRD, alloc)) + union = Cat(MT_Q, M_XRD, alloc)) } } @@ -357,7 +357,7 @@ object GetPrefetch { client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = UInt(0), - union = Cat(M_XRD, Bool(true))) + union = Cat(MT_Q, M_XRD, Bool(true))) } } @@ -548,6 +548,7 @@ class Release extends ClientToManagerChannel def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) def isVoluntary(dummy: Int = 0) = voluntary def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) + def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, UInt(0, width = tlByteAddrBits)) } /** [[uncore.Release]] with an extra field stating its source id */ From ea76800d1ae15e2f2fc1e388670f5808c38946dc Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 11 Jun 2015 15:28:23 -0700 Subject: [PATCH 385/688] Fix data array reset bug io.resp.valid could have been valid the cycle after reset, causing the write mask in the acquire tracker to have an erroneous value after reset. This caused the L1 I$ to be refilled with the wrong data. This probably only affects programs loaded with +loadmem and so shouldn't matter for the EOS24 silicon. It should only affect the first L2 xact, which, in practice, would be an HTIF write to load the program. --- uncore/src/main/scala/cache.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4c2b1a88..925d0648 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -329,10 +329,9 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { reg_raddr := raddr } - io.resp.valid := ShiftRegister(io.read.fire(), delay+1) - io.resp.bits.id := ShiftRegister(io.read.bits.id, delay+1) - io.resp.bits.addr_beat := ShiftRegister(io.read.bits.addr_beat, delay+1) - io.resp.bits.data := ShiftRegister(array(reg_raddr), delay) + val r_req = Pipe(io.read.fire(), io.read.bits) + io.resp := Pipe(r_req.valid, r_req.bits, delay) + io.resp.bits.data := Pipe(r_req.valid, array(reg_raddr), delay).bits io.read.ready := !io.write.valid io.write.ready := Bool(true) } From b4e38192a1805cce3c2fbe75509680af01663e7d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 24 Jun 2015 18:01:56 -0700 Subject: [PATCH 386/688] Fix (?) L2$ miss bug The victim's metadata was incorrectly used for the new line. --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 925d0648..d9d5c6da 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -874,6 +874,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) + val pending_coh_on_miss = HierarchicalMetadata.onReset // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { @@ -910,7 +911,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } - pending_coh := Mux(is_hit, pending_coh_on_hit, coh) + pending_coh := Mux(is_hit, pending_coh_on_hit, Mux(tag_match, coh, pending_coh_on_miss)) when(needs_inner_probes) { val full_sharers = coh.inner.full() val mask_self = Mux( From d7cb60e8fa3ee415445f8f11c617f53af0f8d0d6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 2 Jul 2015 13:52:40 -0700 Subject: [PATCH 387/688] L2 WritebackUnit bug fix --- uncore/src/main/scala/cache.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d9d5c6da..608fe5da 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1012,9 +1012,11 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { io.inner.release.ready := state === s_inner_probe || state === s_busy when(io.inner.release.fire()) { xact.coh.inner := inner_coh_on_irel - when(io.irel().hasData()) { xact.coh.outer := outer_coh_on_irel } // WB is a write data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data } + when(io.inner.release.valid && io.irel().conflicts(xact_addr_block) && io.irel().hasData()) { + xact.coh.outer := outer_coh_on_irel // must writeback dirty data supplied by any matching release, even voluntary ones + } // If a release didn't write back data, have to read it from data array pending_reads := (pending_reads & From 55059632c46496eca73d558f67b6184a752f856e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 5 Jul 2015 16:19:39 -0700 Subject: [PATCH 388/688] Temporarily use HTIF to push RTC value to cores --- uncore/src/main/scala/htif.scala | 24 ++++++++++++++++++++++-- uncore/src/main/scala/uncore.scala | 1 + 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5c1bc4ef..5e270703 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -184,6 +184,11 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { GetBlock(addr_block = init_addr)) io.mem.grant.ready := Bool(true) + // real-time counter (which doesn't really belong here...) + val rtc = Reg(init=UInt(0,64)) + val rtc_tick = Counter(params(RTCPeriod)).inc() + when (rtc_tick) { rtc := rtc + UInt(1) } + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) @@ -197,6 +202,21 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { cpu.pcr_req.bits.data := pcr_wdata cpu.reset := my_reset + // use pcr port to update core's rtc value periodically + val rtc_sent = Reg(init=Bool(false)) + val rtc_outstanding = Reg(init=Bool(false)) + when (rtc_tick) { rtc_sent := Bool(false) } + when (cpu.pcr_rep.valid) { rtc_outstanding := Bool(false) } + when (rtc_outstanding) { cpu.pcr_req.valid := Bool(false) } + when (state != state_pcr_req && state != state_pcr_resp && !rtc_sent && !rtc_outstanding) { + cpu.pcr_req.valid := Bool(true) + cpu.pcr_req.bits.rw := Bool(true) + cpu.pcr_req.bits.addr := UInt(pcr_RESET) /* XXX this means write mtime */ + cpu.pcr_req.bits.data := rtc + rtc_sent := cpu.pcr_req.ready + rtc_outstanding := cpu.pcr_req.ready + } + when (cpu.ipi_rep.ready) { my_ipi := Bool(false) } @@ -208,7 +228,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - when (cpu.pcr_req.valid && cpu.pcr_req.ready) { + when (state === state_pcr_req && cpu.pcr_req.fire()) { state := state_pcr_resp } when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { @@ -220,7 +240,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } cpu.pcr_rep.ready := Bool(true) - when (cpu.pcr_rep.valid) { + when (state === state_pcr_resp && cpu.pcr_rep.valid) { pcrReadData := cpu.pcr_rep.bits state := state_tx } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index b7d06141..a6ca291f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -6,6 +6,7 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +case object RTCPeriod extends Field[Int] trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 From 80ad1eac70b6787c7a5497293358f25436cd6377 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 8 Jul 2015 19:05:18 -0700 Subject: [PATCH 389/688] Update README.md --- uncore/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/README.md b/uncore/README.md index 6e67a691..63ab16f7 100644 --- a/uncore/README.md +++ b/uncore/README.md @@ -7,6 +7,5 @@ the your chip repository and add it as Project in your chip's build.scala. These components are only dependent on Chisel, i.e. lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) - -Documentation for the uncore library is available here -and an overview of the TileLink Protocol is available here +ScalaDoc for the uncore library is available here +and an overview of the TileLink Protocol is available here, with associated CoherencePolicy documentation here. From fb91e3e1aba9627987caf98ffe2c770fc1588e38 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 9 Jul 2015 14:35:39 -0700 Subject: [PATCH 390/688] minor metadata API update (0.3.3) --- uncore/src/main/scala/metadata.scala | 38 +++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index b8d4446c..829a2a7f 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -45,9 +45,9 @@ class ClientMetadata extends CoherenceMetadata { * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] */ def makeAcquire( + op_code: UInt, client_xact_id: UInt, - addr_block: UInt, - op_code: UInt): Acquire = { + addr_block: UInt): Acquire = { Bundle(Acquire( is_builtin_type = Bool(false), a_type = co.getAcquireType(op_code, this), @@ -57,6 +57,28 @@ class ClientMetadata extends CoherenceMetadata { { case TLId => id }) } + /** Constructs a Release message based on this metadata on cache control op + * + * @param client_xact_id client's transaction id + * @param addr_block address of the cache block + * @param addr_beat sub-block address (which beat) + * @param data data being written back + */ + def makeVoluntaryRelease( + op_code: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = { + Bundle(Release( + voluntary = Bool(true), + r_type = co.getReleaseType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data), { case TLId => id }) + } + /** Constructs a Release message based on this metadata on an eviction * * @param client_xact_id client's transaction id @@ -68,15 +90,13 @@ class ClientMetadata extends CoherenceMetadata { client_xact_id: UInt, addr_block: UInt, addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Bundle(Release( - voluntary = Bool(true), - r_type = co.getReleaseType(M_FLUSH, this), + data: UInt = UInt(0)): Release = + makeVoluntaryRelease( + op_code = M_FLUSH, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - data = data), { case TLId => id }) - } + data = data) /** Constructs a Release message based on this metadata and a [[uncore.Probe]] * @@ -119,7 +139,7 @@ class ClientMetadata extends CoherenceMetadata { def onHit(op_code: UInt): ClientMetadata = Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) - /** New metadata after receiving a [[uncore.Probe]] + /** New metadata after op_code releases permissions on this block * * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] */ From 5dc3da008e7b3c4f1e29ed8a6329d4596ad05f3f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 11 Jul 2015 13:36:14 -0700 Subject: [PATCH 391/688] Use Chisel3 SeqMem construct --- uncore/src/main/scala/cache.scala | 21 ++++++++------------- uncore/src/main/scala/memserdes.scala | 17 +++++++---------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 608fe5da..0f92eaa0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -154,12 +154,12 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth - val tag_arr = Mem(UInt(width = metabits*nWays), nSets, seqRead = true) + val tag_arr = SeqMem(UInt(width = metabits*nWays), nSets) when (rst || io.write.valid) { tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) } - val tags = tag_arr(RegEnable(io.read.bits.idx, io.read.valid)) + val tags = tag_arr.read(io.read.bits.idx, io.read.valid) io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst @@ -317,21 +317,16 @@ class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataW class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip - val wmask = FillInterleaved(8, io.write.bits.wmask) - val reg_raddr = Reg(UInt()) - val array = Mem(Bits(width=rowBits), nWays*nSets*refillCycles, seqRead = true) - val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val array = SeqMem(Bits(width=rowBits), nWays*nSets*refillCycles) + val ren = !io.write.valid && io.read.valid val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) - - when (io.write.bits.way_en.orR && io.write.valid) { - array.write(waddr, io.write.bits.data, wmask) - }.elsewhen (io.read.bits.way_en.orR && io.read.valid) { - reg_raddr := raddr - } + val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) + val wmask = FillInterleaved(8, io.write.bits.wmask) + when (io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } val r_req = Pipe(io.read.fire(), io.read.bits) io.resp := Pipe(r_req.valid, r_req.bits, delay) - io.resp.bits.data := Pipe(r_req.valid, array(reg_raddr), delay).bits + io.resp.bits.data := Pipe(r_req.valid, array.read(raddr, ren), delay).bits io.read.ready := !io.write.valid io.write.ready := Bool(true) } diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index a4944e1e..e124b2be 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -482,19 +482,16 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) do_flow := empty && io.deq.ready - val ram = Mem(data, entries, seqRead = true) - val ram_addr = Reg(Bits()) - val ram_out_valid = Reg(Bool()) - ram_out_valid := Bool(false) - when (do_enq) { ram(enq_ptr) := io.enq.bits } - when (io.deq.ready && (atLeastTwo || !io.deq.valid && !empty)) { - ram_out_valid := Bool(true) - ram_addr := Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) - } + val ram = SeqMem(data, entries) + when (do_enq) { ram.write(enq_ptr, io.enq.bits) } + + val ren = io.deq.ready && (atLeastTwo || !io.deq.valid && !empty) + val raddr = Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) + val ram_out_valid = Reg(next = ren) io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, ram(ram_addr)) + io.deq.bits := Mux(empty, io.enq.bits, ram.read(raddr, ren)) } class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module From e76a9d34939aa90f6cd5cee99951f01d4476b470 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 11 Jul 2015 14:05:39 -0700 Subject: [PATCH 392/688] Chisel3: Don't mix Mux types --- uncore/src/main/scala/cache.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0f92eaa0..4a2ecfca 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -150,7 +150,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en).toUInt + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en.toSInt).toUInt when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth @@ -881,9 +881,9 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { UInt(0)) pending_reads := Mux( // GetBlocks and custom types read all beats io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), - SInt(-1, width = innerDataBeats), + SInt(-1), (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | - addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toUInt) + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toSInt).toUInt pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) From 15cec0eab7ae1c6052c89933eb8f6e19316a0eeb Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 15 Jul 2015 12:44:54 -0700 Subject: [PATCH 393/688] Vec(Reg) -> Reg(Vec) --- uncore/src/main/scala/broadcast.scala | 8 ++++---- uncore/src/main/scala/cache.scala | 8 ++++---- uncore/src/main/scala/memserdes.scala | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 4dcc339a..35112d42 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -40,7 +40,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent trackerList.map(_.io.incoherent := io.incoherent.toBits) // Queue to store impending Put data - val sdq = Vec.fill(sdqDepth){ Reg(io.iacq().data) } + val sdq = Reg(Vec.fill(sdqDepth){io.iacq().data}) val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR @@ -69,7 +69,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent val voluntary = io.irel().isVoluntary() val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Vec.fill(innerDataBeats){ Reg(io.irel().data) } //TODO Assumes nReleaseTransactors == 1 + val vwbdq = Reg(Vec.fill(innerDataBeats){io.irel().data}) //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data @@ -131,7 +131,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTrac val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.irel().data.clone) } + val data_buffer = Reg(Vec.fill(innerDataBeats){io.irel().data}) val coh = ManagerMetadata.onReset val collect_irel_data = Reg(init=Bool(false)) @@ -210,7 +210,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(io.iacq().data.clone) } + val data_buffer = Reg(Vec.fill(innerDataBeats){io.iacq().data}) val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4a2ecfca..721ff5a1 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -491,7 +491,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -586,8 +586,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding transaction metadata val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) - val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } - val wmask_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0,width = innerDataBits/8)) } + val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) + val wmask_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits/8)}) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -979,7 +979,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(new L2WritebackReq) - val data_buffer = Vec.fill(innerDataBeats){ Reg(init=UInt(0, width = innerDataBits)) } + val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) val xact_addr_block = Cat(xact.tag, xact.idx) val pending_irels = diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index e124b2be..029541ff 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -264,7 +264,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters mem_data_q.io.enq.valid := Bool(false) val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Vec.fill(tlDataBeats){ Reg(io.tl.acquire.bits.data.clone) } + val tl_buf_out = Reg(Vec(io.tl.acquire.bits.data, tlDataBeats)) val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) @@ -414,7 +414,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) - val mif_buf_in = Vec.fill(mifDataBeats){ Reg(new MemData) } + val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) From 276f53b6524d0a2350cf3b6f50e4faeec0553a4d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 15 Jul 2015 17:41:47 -0700 Subject: [PATCH 394/688] Delete BigMem; it's not used anymore --- uncore/src/main/scala/bigmem.scala | 80 ------------------------------ 1 file changed, 80 deletions(-) delete mode 100644 uncore/src/main/scala/bigmem.scala diff --git a/uncore/src/main/scala/bigmem.scala b/uncore/src/main/scala/bigmem.scala deleted file mode 100644 index fbbc4c07..00000000 --- a/uncore/src/main/scala/bigmem.scala +++ /dev/null @@ -1,80 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ - -class BigMem[T <: Data](n: Int, preLatency: Int, postLatency: Int, leaf: Mem[UInt], noMask: Boolean = false)(gen: => T) extends Module -{ - class Inputs extends Bundle { - val addr = UInt(INPUT, log2Up(n)) - val rw = Bool(INPUT) - val wdata = gen.asInput - val wmask = gen.asInput - override def clone = new Inputs().asInstanceOf[this.type] - } - val io = new Bundle { - val in = Valid(new Inputs).flip - val rdata = gen.asOutput - } - val data = gen - val colMux = if (2*data.getWidth <= leaf.data.getWidth && n > leaf.n) 1 << math.floor(math.log(leaf.data.getWidth/data.getWidth)/math.log(2)).toInt else 1 - val nWide = if (data.getWidth > leaf.data.getWidth) 1+(data.getWidth-1)/leaf.data.getWidth else 1 - val nDeep = if (n > colMux*leaf.n) 1+(n-1)/(colMux*leaf.n) else 1 - if (nDeep > 1 || colMux > 1) - require(isPow2(n) && isPow2(leaf.n)) - - val rdataDeep = Vec.fill(nDeep){Bits()} - val rdataSel = Vec.fill(nDeep){Bool()} - for (i <- 0 until nDeep) { - val in = Pipe(io.in.valid && (if (nDeep == 1) Bool(true) else UInt(i) === io.in.bits.addr(log2Up(n)-1, log2Up(n/nDeep))), io.in.bits, preLatency) - val idx = in.bits.addr(log2Up(n/nDeep/colMux)-1, 0) - val wdata = in.bits.wdata.toBits - val wmask = in.bits.wmask.toBits - val ren = in.valid && !in.bits.rw - val reg_ren = Reg(next=ren) - val rdata = Vec.fill(nWide){Bits()} - - val r = Pipe(ren, in.bits.addr, postLatency) - - for (j <- 0 until nWide) { - val mem = leaf.clone - var dout: Bits = null - val ridx = if (postLatency > 0) Reg(Bits()) else null - - var wmask0 = Fill(colMux, wmask(math.min(wmask.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - if (colMux > 1) - wmask0 = wmask0 & FillInterleaved(gen.getWidth, UIntToOH(in.bits.addr(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux)), log2Up(colMux))) - val wdata0 = Fill(colMux, wdata(math.min(wdata.getWidth, leaf.data.getWidth*(j+1))-1, leaf.data.getWidth*j)) - when (in.valid) { - when (in.bits.rw) { - if (noMask) - mem.write(idx, wdata0) - else - mem.write(idx, wdata0, wmask0) - } - .otherwise { if (postLatency > 0) ridx := idx } - } - - if (postLatency == 0) { - dout = mem(idx) - } else if (postLatency == 1) { - dout = mem(ridx) - } else - dout = Pipe(reg_ren, mem(ridx), postLatency-1).bits - - rdata(j) := dout - } - val rdataWide = rdata.reduceLeft((x, y) => Cat(y, x)) - - var colMuxOut = rdataWide - if (colMux > 1) { - val colMuxIn = Vec((0 until colMux).map(k => rdataWide(gen.getWidth*(k+1)-1, gen.getWidth*k))) - colMuxOut = colMuxIn(r.bits(log2Up(n/nDeep)-1, log2Up(n/nDeep/colMux))) - } - - rdataDeep(i) := colMuxOut - rdataSel(i) := r.valid - } - - io.rdata := Mux1H(rdataSel, rdataDeep) -} From 2d6b3b23313b9a74e35fd0c48b54615052b13395 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 15 Jul 2015 18:06:27 -0700 Subject: [PATCH 395/688] Don't use clone --- uncore/src/main/scala/cache.scala | 14 +++++++------- uncore/src/main/scala/memserdes.scala | 6 +++--- uncore/src/main/scala/network.scala | 12 ++++++------ uncore/src/main/scala/tilelink.scala | 10 +++++----- uncore/src/main/scala/uncore.scala | 2 +- uncore/src/main/scala/util.scala | 6 +++--- 6 files changed, 25 insertions(+), 25 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 721ff5a1..012156c5 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -135,16 +135,16 @@ class MetaReadReq extends CacheBundle { class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { val way_en = Bits(width = nWays) - val data = gen.clone - override def clone = new MetaWriteReq(gen).asInstanceOf[this.type] + val data = gen.cloneType + override def cloneType = new MetaWriteReq(gen).asInstanceOf[this.type] } class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val rstVal = makeRstVal() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip - val write = Decoupled(new MetaWriteReq(rstVal.clone)).flip - val resp = Vec.fill(nWays){rstVal.clone.asOutput} + val write = Decoupled(new MetaWriteReq(rstVal)).flip + val resp = Vec.fill(nWays){rstVal.cloneType.asOutput} } val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) @@ -186,7 +186,7 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { def doInternalOutputArbitration[T <: Data : ClassTag]( out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { - val arb = Module(new RRArbiter(out.bits.clone, ins.size)) + val arb = Module(new RRArbiter(out.bits, ins.size)) out <> arb.io.out arb.io.in <> ins } @@ -237,7 +237,7 @@ class L2MetaReadReq extends MetaReadReq with HasL2Id { class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) with HasL2Id { - override def clone = new L2MetaWriteReq().asInstanceOf[this.type] + override def cloneType = new L2MetaWriteReq().asInstanceOf[this.type] } class L2MetaResp extends L2HellaCacheBundle @@ -591,7 +591,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } - val pending_coh = Reg{ xact_old_meta.coh.clone } + val pending_coh = Reg{ xact_old_meta.coh } // Secondary miss queue val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index 029541ff..b58553bb 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -415,7 +415,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data.clone } + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data } tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) @@ -506,7 +506,7 @@ class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module object HellaQueue { def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { - val q = Module((new HellaQueue(entries)) { enq.bits.clone }) + val q = Module((new HellaQueue(entries)) { enq.bits }) q.io.enq.valid := enq.valid // not using <> so that override is allowed q.io.enq.bits := enq.bits enq.ready := q.io.enq.ready @@ -522,7 +522,7 @@ class MemIOArbiter(val arbN: Int) extends MIFModule { if(arbN > 1) { val cmd_arb = Module(new RRArbiter(new MemReqCmd, arbN)) - val choice_q = Module(new Queue(cmd_arb.io.chosen.clone, 4)) + val choice_q = Module(new Queue(cmd_arb.io.chosen, 4)) val (data_cnt, data_done) = Counter(io.outer.req_data.fire(), mifDataBeats) io.inner.map(_.req_cmd).zipWithIndex.zip(cmd_arb.io.in).map{ case ((req, id), arb) => { diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 4b00091d..9170982f 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -13,8 +13,8 @@ class PhysicalHeader(n: Int) extends Bundle { class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { val header = new PhysicalHeader(n) - val payload = dType.clone - override def clone = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] + val payload = dType + override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] } class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { @@ -52,8 +52,8 @@ class LogicalHeader extends Bundle { class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { val header = new LogicalHeader - val payload = dType.clone - override def clone = { new LogicalNetworkIO(dType).asInstanceOf[this.type] } + val payload = dType.cloneType + override def cloneType = new LogicalNetworkIO(dType).asInstanceOf[this.type] } object DecoupledLogicalNetworkIOWrapper { @@ -61,7 +61,7 @@ object DecoupledLogicalNetworkIOWrapper { in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Decoupled(new LogicalNetworkIO(in.bits.clone)).asDirectionless + val out = Decoupled(new LogicalNetworkIO(in.bits)).asDirectionless out.valid := in.valid out.bits.payload := in.bits out.bits.header.dst := dst @@ -73,7 +73,7 @@ object DecoupledLogicalNetworkIOWrapper { object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { - val out = Decoupled(in.bits.payload.clone).asDirectionless + val out = Decoupled(in.bits.payload).asDirectionless out.valid := in.valid out.bits := in.bits.payload in.ready := out.ready diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 31f0b136..c8f6951c 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -904,7 +904,7 @@ object ClientTileLinkHeaderCreator { in: DecoupledIO[T], clientId: Int, addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + val out = new DecoupledIO(new LogicalNetworkIO(in.bits)).asDirectionless out.bits.payload := in.bits out.bits.header.src := UInt(clientId) out.bits.header.dst := addrConvert(in.bits.addr_block) @@ -943,7 +943,7 @@ object ManagerTileLinkHeaderCreator { in: DecoupledIO[T], managerId: Int, idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits.clone)).asDirectionless + val out = new DecoupledIO(new LogicalNetworkIO(in.bits)).asDirectionless out.bits.payload := in.bits out.bits.header.src := UInt(managerId) out.bits.header.dst := idConvert(in.bits.client_id) @@ -1001,7 +1001,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], mngr: DecoupledIO[LogicalNetworkIO[M]]) { def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -1015,7 +1015,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits.clone, arbN, tlDataBeats, Some(hasData _))) + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { arb.valid := req.valid arb.bits := req.bits @@ -1076,7 +1076,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { } def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits.clone, arbN)) + val arb = Module(new RRArbiter(mngr.bits, arbN)) arb.io.in <> clts arb.io.out <> mngr } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index a6ca291f..9b53832b 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -33,7 +33,7 @@ trait HasCoherenceAgentWiringHelpers { out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { def lock(o: T) = o.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits.clone, ins.size, out.bits.tlDataBeats, lock _)) + val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, lock _)) out <> arb.io.out arb.io.in <> ins } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 65c5d6cd..ff982e3f 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -44,8 +44,8 @@ object ZCounter { class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { - val in = Decoupled(gen.clone).flip - val out = Decoupled(gen.clone) + val in = Decoupled(gen).flip + val out = Decoupled(gen) val cnt = UInt(OUTPUT, log2Up(n)) val done = Bool(OUTPUT) } @@ -59,7 +59,7 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module } else { val cnt = Reg(init=UInt(0, width = log2Up(n))) val wrap = cnt === UInt(n-1) - val rbits = Reg(io.in.bits.clone) + val rbits = Reg{io.in.bits} val active = Reg(init=Bool(false)) val shifter = Vec.fill(n){Bits(width = narrowWidth)} From 3c0475e08bb4adbd37aa572c2fdc65937576ac81 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 15 Jul 2015 20:24:03 -0700 Subject: [PATCH 396/688] Add Wire() wrap --- uncore/src/main/scala/cache.scala | 4 ++-- uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/memserdes.scala | 8 ++++---- uncore/src/main/scala/metadata.scala | 8 ++++---- uncore/src/main/scala/network.scala | 2 +- uncore/src/main/scala/tilelink.scala | 18 +++++++++--------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 012156c5..38bf1930 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -115,7 +115,7 @@ abstract class ReplacementPolicy { } class RandomReplacement(ways: Int) extends ReplacementPolicy { - private val replace = Bool() + private val replace = Wire(Bool()) replace := Bool(false) val lfsr = LFSR16(replace) @@ -224,7 +224,7 @@ class L2Metadata extends Metadata with L2HellaCacheParameters { object L2Metadata { def apply(tag: Bits, coh: HierarchicalMetadata) = { - val meta = new L2Metadata + val meta = Wire(new L2Metadata) meta.tag := tag meta.coh := coh meta diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5e270703..ea09ed1f 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -247,7 +247,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } val scr_addr = addr(log2Up(nSCR)-1, 0) - val scr_rdata = Vec.fill(io.scr.rdata.size){Bits(width = 64)} + val scr_rdata = Wire(Vec(Bits(width=64), io.scr.rdata.size)) for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala index b58553bb..82ee61ae 100644 --- a/uncore/src/main/scala/memserdes.scala +++ b/uncore/src/main/scala/memserdes.scala @@ -467,7 +467,7 @@ class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module val io = new QueueIO(data, entries) require(entries > 1) - val do_flow = Bool() + val do_flow = Wire(Bool()) val do_enq = io.enq.fire() && !do_flow val do_deq = io.deq.fire() && !do_flow @@ -559,7 +559,7 @@ class MemIOArbiter(val arbN: Int) extends MIFModule { object MemIOMemPipeIOConverter { def apply(in: MemPipeIO): MemIO = { - val out = new MemIO().asDirectionless + val out = Wire(new MemIO()) in.resp.valid := out.resp.valid in.resp.bits := out.resp.bits out.resp.ready := Bool(true) @@ -582,8 +582,8 @@ class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { val numEntries = numRequests * mifDataBeats val size = log2Down(numEntries) + 1 - val inc = Bool() - val dec = Bool() + val inc = Wire(Bool()) + val dec = Wire(Bool()) val count = Reg(init=UInt(numEntries, size)) val watermark = count >= UInt(mifDataBeats) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 829a2a7f..68b2981c 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -150,7 +150,7 @@ class ClientMetadata extends CoherenceMetadata { /** Factories for ClientMetadata, including on reset */ object ClientMetadata { def apply(state: UInt) = { - val meta = new ClientMetadata + val meta = Wire(new ClientMetadata) meta.state := state meta } @@ -288,13 +288,13 @@ class ManagerMetadata extends CoherenceMetadata { /** Factories for ManagerMetadata, including on reset */ object ManagerMetadata { def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { - val meta = new ManagerMetadata + val meta = Wire(new ManagerMetadata) //meta.state := state TODO: Fix 0-width wires in Chisel meta.sharers := sharers meta } def apply() = { - val meta = new ManagerMetadata + val meta = Wire(new ManagerMetadata) //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel meta.sharers := meta.co.dir.flush meta @@ -321,7 +321,7 @@ class HierarchicalMetadata extends CoherenceMetadata { /** Factories for HierarchicalMetadata, including on reset */ object HierarchicalMetadata { def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { - val m = new HierarchicalMetadata + val m = Wire(new HierarchicalMetadata) m.inner := inner m.outer := outer m diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 9170982f..93acf64e 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -73,7 +73,7 @@ object DecoupledLogicalNetworkIOWrapper { object DecoupledLogicalNetworkIOUnwrapper { def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { - val out = Decoupled(in.bits.payload).asDirectionless + val out = Wire(Decoupled(in.bits.payload)) out.valid := in.valid out.bits := in.bits.payload in.ready := out.ready diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c8f6951c..6c7aeb6e 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -257,7 +257,7 @@ object Acquire { addr_beat: UInt = UInt(0), data: UInt = UInt(0), union: UInt = UInt(0)): Acquire = { - val acq = new Acquire + val acq = Wire(new Acquire) acq.is_builtin_type := is_builtin_type acq.a_type := a_type acq.client_xact_id := client_xact_id @@ -269,7 +269,7 @@ object Acquire { } // Copy constructor def apply(a: Acquire): Acquire = { - val acq = new Acquire + val acq = Wire(new Acquire) acq := a acq } @@ -513,13 +513,13 @@ class ProbeToDst extends Probe with HasClientId */ object Probe { def apply(p_type: UInt, addr_block: UInt): Probe = { - val prb = new Probe + val prb = Wire(new Probe) prb.p_type := p_type prb.addr_block := addr_block prb } def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { - val prb = new ProbeToDst + val prb = Wire(new ProbeToDst) prb.client_id := dst prb.p_type := p_type prb.addr_block := addr_block @@ -574,7 +574,7 @@ object Release { addr_block: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): Release = { - val rel = new Release + val rel = Wire(new Release) rel.r_type := r_type rel.client_xact_id := client_xact_id rel.addr_block := addr_block @@ -613,7 +613,7 @@ class Grant extends ManagerToClientChannel def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts }) + val f = Wire(Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts })) f.manager_xact_id := this.manager_xact_id f } @@ -671,7 +671,7 @@ object Grant { manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): GrantToDst = { - val gnt = new GrantToDst + val gnt = Wire(new GrantToDst) gnt.client_id := dst gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type @@ -904,7 +904,7 @@ object ClientTileLinkHeaderCreator { in: DecoupledIO[T], clientId: Int, addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits)).asDirectionless + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) out.bits.payload := in.bits out.bits.header.src := UInt(clientId) out.bits.header.dst := addrConvert(in.bits.addr_block) @@ -943,7 +943,7 @@ object ManagerTileLinkHeaderCreator { in: DecoupledIO[T], managerId: Int, idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { - val out = new DecoupledIO(new LogicalNetworkIO(in.bits)).asDirectionless + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) out.bits.payload := in.bits out.bits.header.src := UInt(managerId) out.bits.header.dst := idConvert(in.bits.client_id) From 0e06c941dfbb2e79a9bd5230b199ade8ce3e9802 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 23 Jul 2015 14:58:46 -0700 Subject: [PATCH 397/688] Chisel3 compatibility fixes --- uncore/src/main/scala/broadcast.scala | 8 ++------ uncore/src/main/scala/htif.scala | 2 +- uncore/src/main/scala/network.scala | 6 +++--- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 35112d42..67d3c983 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -111,7 +111,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer.acquire.bits.union := Cat(Fill(outer_arb.io.out.acquire.bits.union(1), io.outer.acquire.bits.tlWriteMaskBits), + io.outer.acquire.bits.union := Cat(Fill(io.outer.acquire.bits.tlWriteMaskBits, outer_arb.io.out.acquire.bits.union(1)), outer_arb.io.out.acquire.bits.union(0)) io.outer <> outer_arb.io.out @@ -221,11 +221,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) val curr_p_id = PriorityEncoder(pending_probes) - val full_sharers = coh.full() - val probe_self = io.inner.acquire.bits.requiresSelfProbe() - val mask_self_true = UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) - val mask_self_false = ~UInt(UInt(1) << io.inner.acquire.bits.client_id, width = io.inner.tlNCachingClients) - val mask_self = Mux(probe_self, full_sharers | mask_self_true, full_sharers & mask_self_false) + val mask_self = coh.full().bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe()) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index ea09ed1f..daedf32d 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -3,7 +3,7 @@ package uncore import Chisel._ -import Node._ +import Chisel.ImplicitConversions._ import uncore._ case object HTIFWidth extends Field[Int] diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 93acf64e..929f1e43 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -61,7 +61,7 @@ object DecoupledLogicalNetworkIOWrapper { in: DecoupledIO[T], src: UInt = UInt(0), dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Decoupled(new LogicalNetworkIO(in.bits)).asDirectionless + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) out.valid := in.valid out.bits.payload := in.bits out.bits.header.dst := dst @@ -83,7 +83,7 @@ object DecoupledLogicalNetworkIOUnwrapper { object DefaultFromPhysicalShim { def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Decoupled(new LogicalNetworkIO(in.bits.payload)).asDirectionless + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) out.bits.header := in.bits.header out.bits.payload := in.bits.payload out.valid := in.valid @@ -94,7 +94,7 @@ object DefaultFromPhysicalShim { object DefaultToPhysicalShim { def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { - val out = Decoupled(new PhysicalNetworkIO(n, in.bits.payload)).asDirectionless + val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) out.bits.header := in.bits.header out.bits.payload := in.bits.payload out.valid := in.valid From f8ec6d63938d386211ca7bc5fa54a947a55695c0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 28 Jul 2015 02:46:23 -0700 Subject: [PATCH 398/688] Chisel3 compatibility: use BitPat for don't-cares Also, call the UInt factory instead of the Bits one, for good measure. --- uncore/src/main/scala/consts.scala | 68 +++++++++++++++--------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 54000208..56d02f68 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -8,42 +8,42 @@ import Chisel._ object MemoryOpConstants extends MemoryOpConstants trait MemoryOpConstants { val MT_SZ = 3 - val MT_X = Bits("b???") - val MT_B = Bits("b000") - val MT_H = Bits("b001") - val MT_W = Bits("b010") - val MT_D = Bits("b011") - val MT_BU = Bits("b100") - val MT_HU = Bits("b101") - val MT_WU = Bits("b110") - val MT_Q = Bits("b111") + val MT_X = BitPat("b???") + val MT_B = UInt("b000") + val MT_H = UInt("b001") + val MT_W = UInt("b010") + val MT_D = UInt("b011") + val MT_BU = UInt("b100") + val MT_HU = UInt("b101") + val MT_WU = UInt("b110") + val MT_Q = UInt("b111") val NUM_XA_OPS = 9 val M_SZ = 5 - val M_X = Bits("b?????"); - val M_XRD = Bits("b00000"); // int load - val M_XWR = Bits("b00001"); // int store - val M_PFR = Bits("b00010"); // prefetch with intent to read - val M_PFW = Bits("b00011"); // prefetch with intent to write - val M_XA_SWAP = Bits("b00100"); - val M_NOP = Bits("b00101"); - val M_XLR = Bits("b00110"); - val M_XSC = Bits("b00111"); - val M_XA_ADD = Bits("b01000"); - val M_XA_XOR = Bits("b01001"); - val M_XA_OR = Bits("b01010"); - val M_XA_AND = Bits("b01011"); - val M_XA_MIN = Bits("b01100"); - val M_XA_MAX = Bits("b01101"); - val M_XA_MINU = Bits("b01110"); - val M_XA_MAXU = Bits("b01111"); - val M_FLUSH = Bits("b10000") // write back dirty data and cede R/W permissions - val M_PRODUCE = Bits("b10001") // write back dirty data and cede W permissions - val M_CLEAN = Bits("b10011") // write back dirty data and retain R/W permissions + val M_X = BitPat("b?????"); + val M_XRD = UInt("b00000"); // int load + val M_XWR = UInt("b00001"); // int store + val M_PFR = UInt("b00010"); // prefetch with intent to read + val M_PFW = UInt("b00011"); // prefetch with intent to write + val M_XA_SWAP = UInt("b00100"); + val M_NOP = UInt("b00101"); + val M_XLR = UInt("b00110"); + val M_XSC = UInt("b00111"); + val M_XA_ADD = UInt("b01000"); + val M_XA_XOR = UInt("b01001"); + val M_XA_OR = UInt("b01010"); + val M_XA_AND = UInt("b01011"); + val M_XA_MIN = UInt("b01100"); + val M_XA_MAX = UInt("b01101"); + val M_XA_MINU = UInt("b01110"); + val M_XA_MAXU = UInt("b01111"); + val M_FLUSH = UInt("b10000") // write back dirty data and cede R/W permissions + val M_PRODUCE = UInt("b10001") // write back dirty data and cede W permissions + val M_CLEAN = UInt("b10011") // write back dirty data and retain R/W permissions - def isAMO(cmd: Bits) = cmd(3) || cmd === M_XA_SWAP - def isPrefetch(cmd: Bits) = cmd === M_PFR || cmd === M_PFW - def isRead(cmd: Bits) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) - def isWrite(cmd: Bits) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) - def isWriteIntent(cmd: Bits) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR + def isAMO(cmd: UInt) = cmd(3) || cmd === M_XA_SWAP + def isPrefetch(cmd: UInt) = cmd === M_PFR || cmd === M_PFW + def isRead(cmd: UInt) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isWrite(cmd: UInt) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) + def isWriteIntent(cmd: UInt) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } From a69c74924971adc97ff050cca1e79d9fb8db2975 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 28 Jul 2015 16:24:45 -0700 Subject: [PATCH 399/688] Fix compilation with scala 2.11.6 We forgot to specify return types on overloaded methods, and a previous version of the scala compiler failed to flag this as an error. --- uncore/src/main/scala/cache.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 38bf1930..dcfc188d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -441,13 +441,13 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( in: DecoupledIO[T], beat: UInt = UInt(0), - full_block: Bool = Bool(true)) = { + full_block: Bool = Bool(true)): (UInt, Bool) = { connectDataBeatCounter(in.fire(), in.bits, beat, full_block) } def connectInternalDataBeatCounter[T <: HasL2Data]( in: ValidIO[T], - full_block: Bool = Bool(true)) = { + full_block: Bool): Bool = { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } From 4daa20b5fe96005426ab500671361acbeec81acf Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 28 Jul 2015 15:15:34 -0700 Subject: [PATCH 400/688] simplify .sbt files --- uncore/build.sbt | 4 ++++ uncore/chisel-dependent.sbt | 8 -------- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 uncore/chisel-dependent.sbt diff --git a/uncore/build.sbt b/uncore/build.sbt index 35b7dcac..3287e088 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -6,6 +6,10 @@ name := "uncore" scalaVersion := "2.10.2" +// Provide a managed dependency on chisel if -DchiselVersion="" is supplied on the command line. +libraryDependencies ++= (Seq("chisel").map { + dep: String => sys.props.get(dep + "Version") map { "edu.berkeley.cs" %% dep % _ }}).flatten + site.settings site.includeScaladoc() diff --git a/uncore/chisel-dependent.sbt b/uncore/chisel-dependent.sbt deleted file mode 100644 index 88eb615c..00000000 --- a/uncore/chisel-dependent.sbt +++ /dev/null @@ -1,8 +0,0 @@ -// Provide a managed dependency on chisel if -DchiselVersion="" is -// supplied on the command line. - -val chiselVersion_u = System.getProperty("chiselVersion", "None") - -// _u a temporary fix until sbt 13.6 https://github.com/sbt/sbt/issues/1465 - -libraryDependencies ++= ( if (chiselVersion_u != "None" ) ("edu.berkeley.cs" %% "chisel" % chiselVersion_u) :: Nil; else Nil) From 8b1ab23347c94c07019290f6c255656e74ea6f88 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 28 Jul 2015 16:12:17 -0700 Subject: [PATCH 401/688] update README.md --- uncore/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/uncore/README.md b/uncore/README.md index 63ab16f7..3a628a24 100644 --- a/uncore/README.md +++ b/uncore/README.md @@ -3,9 +3,10 @@ Uncore Library This is the repository for uncore components assosciated with Rocket chip project. To uses these modules, include this repo as a git submodule within -the your chip repository and add it as Project in your chip's build.scala. -These components are only dependent on Chisel, i.e. +the your chip repository and add it as a project in your chip's build.scala. +These components are only dependent on the ucb-bar/chisel repo, i.e. + + lazy val uncore = project.dependsOn(chisel) - lazy val uncore = Project("uncore", file("uncore"), settings = buildSettings) dependsOn(chisel) ScalaDoc for the uncore library is available here and an overview of the TileLink Protocol is available here, with associated CoherencePolicy documentation here. From c70b495f6d66df45a154dfa7c1864db604dbc47b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 29 Jul 2015 18:04:30 -0700 Subject: [PATCH 402/688] moved buses to junctions repo --- uncore/build.sbt | 4 +- uncore/src/main/scala/cache.scala | 1 + uncore/src/main/scala/memserdes.scala | 640 -------------------------- uncore/src/main/scala/nasti.scala | 295 ------------ uncore/src/main/scala/slowio.scala | 70 --- uncore/src/main/scala/tilelink.scala | 420 +++++++++++++++++ 6 files changed, 423 insertions(+), 1007 deletions(-) delete mode 100644 uncore/src/main/scala/memserdes.scala delete mode 100644 uncore/src/main/scala/nasti.scala delete mode 100644 uncore/src/main/scala/slowio.scala diff --git a/uncore/build.sbt b/uncore/build.sbt index 3287e088..a34b2b9b 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -6,8 +6,8 @@ name := "uncore" scalaVersion := "2.10.2" -// Provide a managed dependency on chisel if -DchiselVersion="" is supplied on the command line. -libraryDependencies ++= (Seq("chisel").map { +// Provide a managed dependency on X if -DXVersion="" is supplied on the command line. +libraryDependencies ++= (Seq("chisel","junction").map { dep: String => sys.props.get(dep + "Version") map { "edu.berkeley.cs" %% dep % _ }}).flatten site.settings diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dcfc188d..b956a2e0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import scala.reflect.ClassTag +import junctions._ case object CacheName extends Field[String] case object NSets extends Field[Int] diff --git a/uncore/src/main/scala/memserdes.scala b/uncore/src/main/scala/memserdes.scala deleted file mode 100644 index 82ee61ae..00000000 --- a/uncore/src/main/scala/memserdes.scala +++ /dev/null @@ -1,640 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ -import scala.math._ - -case object PAddrBits extends Field[Int] -case object VAddrBits extends Field[Int] -case object PgIdxBits extends Field[Int] -case object PgLevels extends Field[Int] -case object PgLevelBits extends Field[Int] -case object ASIdBits extends Field[Int] -case object PPNBits extends Field[Int] -case object VPNBits extends Field[Int] - -case object MIFAddrBits extends Field[Int] -case object MIFDataBits extends Field[Int] -case object MIFTagBits extends Field[Int] -case object MIFDataBeats extends Field[Int] - -trait MIFParameters extends UsesParameters { - val mifTagBits = params(MIFTagBits) - val mifAddrBits = params(MIFAddrBits) - val mifDataBits = params(MIFDataBits) - val mifDataBeats = params(MIFDataBeats) -} - -abstract class MIFBundle extends Bundle with MIFParameters -abstract class MIFModule extends Module with MIFParameters - -trait HasMemData extends MIFBundle { - val data = Bits(width = mifDataBits) -} - -trait HasMemAddr extends MIFBundle { - val addr = UInt(width = mifAddrBits) -} - -trait HasMemTag extends MIFBundle { - val tag = UInt(width = mifTagBits) -} - -class MemReqCmd extends HasMemAddr with HasMemTag { - val rw = Bool() -} - -class MemTag extends HasMemTag -class MemData extends HasMemData -class MemResp extends HasMemData with HasMemTag - -class MemIO extends Bundle { - val req_cmd = Decoupled(new MemReqCmd) - val req_data = Decoupled(new MemData) - val resp = Decoupled(new MemResp).flip -} - -class MemPipeIO extends Bundle { - val req_cmd = Decoupled(new MemReqCmd) - val req_data = Decoupled(new MemData) - val resp = Valid(new MemResp).flip -} - -class MemSerializedIO(w: Int) extends Bundle { - val req = Decoupled(Bits(width = w)) - val resp = Valid(Bits(width = w)).flip -} - -class MemSerdes extends MIFModule -{ - val w = params(HTIFWidth) - val io = new Bundle { - val wide = new MemIO().flip - val narrow = new MemSerializedIO(w) - } - val abits = io.wide.req_cmd.bits.toBits.getWidth - val dbits = io.wide.req_data.bits.toBits.getWidth - val rbits = io.wide.resp.bits.getWidth - - val out_buf = Reg(Bits()) - val in_buf = Reg(Bits()) - - val s_idle :: s_read_addr :: s_write_addr :: s_write_idle :: s_write_data :: Nil = Enum(UInt(), 5) - val state = Reg(init=s_idle) - val send_cnt = Reg(init=UInt(0, log2Up((max(abits, dbits)+w-1)/w))) - val data_send_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) - val adone = io.narrow.req.ready && send_cnt === UInt((abits-1)/w) - val ddone = io.narrow.req.ready && send_cnt === UInt((dbits-1)/w) - - when (io.narrow.req.valid && io.narrow.req.ready) { - send_cnt := send_cnt + UInt(1) - out_buf := out_buf >> UInt(w) - } - when (io.wide.req_cmd.valid && io.wide.req_cmd.ready) { - out_buf := io.wide.req_cmd.bits.toBits - } - when (io.wide.req_data.valid && io.wide.req_data.ready) { - out_buf := io.wide.req_data.bits.toBits - } - - io.wide.req_cmd.ready := state === s_idle - io.wide.req_data.ready := state === s_write_idle - io.narrow.req.valid := state === s_read_addr || state === s_write_addr || state === s_write_data - io.narrow.req.bits := out_buf - - when (state === s_idle && io.wide.req_cmd.valid) { - state := Mux(io.wide.req_cmd.bits.rw, s_write_addr, s_read_addr) - } - when (state === s_read_addr && adone) { - state := s_idle - send_cnt := UInt(0) - } - when (state === s_write_addr && adone) { - state := s_write_idle - send_cnt := UInt(0) - } - when (state === s_write_idle && io.wide.req_data.valid) { - state := s_write_data - } - when (state === s_write_data && ddone) { - data_send_cnt := data_send_cnt + UInt(1) - state := Mux(data_send_cnt === UInt(mifDataBeats-1), s_idle, s_write_idle) - send_cnt := UInt(0) - } - - val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) - val resp_val = Reg(init=Bool(false)) - - resp_val := Bool(false) - when (io.narrow.resp.valid) { - recv_cnt := recv_cnt + UInt(1) - when (recv_cnt === UInt((rbits-1)/w)) { - recv_cnt := UInt(0) - data_recv_cnt := data_recv_cnt + UInt(1) - resp_val := Bool(true) - } - in_buf := Cat(io.narrow.resp.bits, in_buf((rbits+w-1)/w*w-1,w)) - } - - io.wide.resp.valid := resp_val - io.wide.resp.bits := io.wide.resp.bits.fromBits(in_buf) -} - -class MemDesserIO(w: Int) extends Bundle { - val narrow = new MemSerializedIO(w).flip - val wide = new MemIO -} - -class MemDesser(w: Int) extends Module // test rig side -{ - val io = new MemDesserIO(w) - val abits = io.wide.req_cmd.bits.toBits.getWidth - val dbits = io.wide.req_data.bits.toBits.getWidth - val rbits = io.wide.resp.bits.getWidth - val mifDataBeats = params(MIFDataBeats) - - require(dbits >= abits && rbits >= dbits) - val recv_cnt = Reg(init=UInt(0, log2Up((rbits+w-1)/w))) - val data_recv_cnt = Reg(init=UInt(0, log2Up(mifDataBeats))) - val adone = io.narrow.req.valid && recv_cnt === UInt((abits-1)/w) - val ddone = io.narrow.req.valid && recv_cnt === UInt((dbits-1)/w) - val rdone = io.narrow.resp.valid && recv_cnt === UInt((rbits-1)/w) - - val s_cmd_recv :: s_cmd :: s_data_recv :: s_data :: s_reply :: Nil = Enum(UInt(), 5) - val state = Reg(init=s_cmd_recv) - - val in_buf = Reg(Bits()) - when (io.narrow.req.valid && io.narrow.req.ready || io.narrow.resp.valid) { - recv_cnt := recv_cnt + UInt(1) - in_buf := Cat(io.narrow.req.bits, in_buf((rbits+w-1)/w*w-1,w)) - } - io.narrow.req.ready := state === s_cmd_recv || state === s_data_recv - - when (state === s_cmd_recv && adone) { - state := s_cmd - recv_cnt := UInt(0) - } - when (state === s_cmd && io.wide.req_cmd.ready) { - state := Mux(io.wide.req_cmd.bits.rw, s_data_recv, s_reply) - } - when (state === s_data_recv && ddone) { - state := s_data - recv_cnt := UInt(0) - } - when (state === s_data && io.wide.req_data.ready) { - state := s_data_recv - when (data_recv_cnt === UInt(mifDataBeats-1)) { - state := s_cmd_recv - } - data_recv_cnt := data_recv_cnt + UInt(1) - } - when (rdone) { // state === s_reply - when (data_recv_cnt === UInt(mifDataBeats-1)) { - state := s_cmd_recv - } - recv_cnt := UInt(0) - data_recv_cnt := data_recv_cnt + UInt(1) - } - - val req_cmd = in_buf >> UInt(((rbits+w-1)/w - (abits+w-1)/w)*w) - io.wide.req_cmd.valid := state === s_cmd - io.wide.req_cmd.bits := io.wide.req_cmd.bits.fromBits(req_cmd) - - io.wide.req_data.valid := state === s_data - io.wide.req_data.bits.data := in_buf >> UInt(((rbits+w-1)/w - (dbits+w-1)/w)*w) - - val dataq = Module(new Queue(new MemResp, mifDataBeats)) - dataq.io.enq <> io.wide.resp - dataq.io.deq.ready := recv_cnt === UInt((rbits-1)/w) - - io.narrow.resp.valid := dataq.io.deq.valid - io.narrow.resp.bits := dataq.io.deq.bits.toBits >> (recv_cnt * UInt(w)) -} - -//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { - val io = new Bundle { - val tl = new ManagerTileLinkIO - val mem = new MemIO - } - val dataBits = tlDataBits*tlDataBeats - val dstIdBits = params(LNHeaderBits) - require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") - require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) - - io.tl.acquire.ready := Bool(false) - io.tl.probe.valid := Bool(false) - io.tl.release.ready := Bool(false) - io.tl.finish.ready := Bool(true) - io.mem.resp.ready := Bool(false) - - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.hasData() - val rel_has_data = io.tl.release.bits.hasData() - - // Decompose outgoing TL Acquires into MemIO cmd and data - val active_out = Reg(init=Bool(false)) - val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(UInt(width = mifTagBits)) - val addr_out = Reg(UInt(width = mifAddrBits)) - val has_data = Reg(init=Bool(false)) - val data_from_rel = Reg(init=Bool(false)) - val (tl_cnt_out, tl_wrap_out) = - Counter((io.tl.acquire.fire() && acq_has_data) || - (io.tl.release.fire() && rel_has_data), tlDataBeats) - val tl_done_out = Reg(init=Bool(false)) - val make_grant_ack = Reg(init=Bool(false)) - - gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits := Grant( - dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> UInt(1), - manager_xact_id = UInt(0)) - - if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) - val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Reg(Vec(io.tl.acquire.bits.data, tlDataBeats)) - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) - - when(!active_out){ - io.tl.release.ready := Bool(true) - io.tl.acquire.ready := !io.tl.release.valid - when(io.tl.release.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - addr_out := io.tl.release.bits.addr_block - has_data := rel_has_data - data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.requiresAck() - tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.release.bits.data - } .elsewhen(io.tl.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.client_id, - io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) - addr_out := io.tl.acquire.bits.addr_block - has_data := acq_has_data - data_from_rel := Bool(false) - make_grant_ack := acq_has_data - tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data - } - } - when(active_out) { - mem_cmd_q.io.enq.valid := !cmd_sent_out - cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() - when(has_data) { - when(!tl_done_out) { - io.tl.acquire.ready := Bool(true) - when(io.tl.acquire.valid) { - tl_buf_out(tl_cnt_out) := Mux(data_from_rel, - io.tl.release.bits.data, - io.tl.acquire.bits.data) - } - } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out - } - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) - when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } - } - when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { - active_out := Bool(false) - } - } - - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - } else { // Don't make the data buffers and try to flow cmd and data - io.mem.req_cmd.valid := Bool(false) - io.mem.req_data.valid := Bool(false) - io.mem.req_cmd.bits.rw := has_data - io.mem.req_cmd.bits.tag := tag_out - io.mem.req_cmd.bits.addr := addr_out - io.mem.req_data.bits.data := Mux(data_from_rel, - io.tl.release.bits.data, - io.tl.acquire.bits.data) - when(!active_out){ - io.tl.release.ready := io.mem.req_data.ready - io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid - io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || - (io.tl.acquire.valid && acq_has_data) - when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { - active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid - io.mem.req_cmd.valid := Bool(true) - cmd_sent_out := io.mem.req_cmd.ready - tl_done_out := tl_wrap_out - when(io.tl.release.valid) { - data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.requiresAck() - io.mem.req_data.bits.data := io.tl.release.bits.data - val tag = Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - val addr = io.tl.release.bits.addr_block - io.mem.req_cmd.bits.tag := tag - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := rel_has_data - tag_out := tag - addr_out := addr - has_data := rel_has_data - } .elsewhen(io.tl.acquire.valid) { - data_from_rel := Bool(false) - make_grant_ack := acq_has_data // i.e. is it a Put - io.mem.req_data.bits.data := io.tl.acquire.bits.data - io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.client_id, - io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) - val addr = io.tl.acquire.bits.addr_block - io.mem.req_cmd.bits.tag := tag - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := acq_has_data - tag_out := tag - addr_out := addr - has_data := acq_has_data - } - } - } - when(active_out) { - io.mem.req_cmd.valid := !cmd_sent_out - cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() - when(has_data && !tl_done_out) { - when(data_from_rel) { - io.tl.release.ready := io.mem.req_data.ready - io.mem.req_data.valid := io.tl.release.valid - } .otherwise { - io.tl.acquire.ready := io.mem.req_data.ready - io.mem.req_data.valid := io.tl.acquire.valid - } - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? - when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } - } - when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { - active_out := Bool(false) - } - } - } - - // Aggregate incoming MemIO responses into TL Grants - val active_in = Reg(init=Bool(false)) - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val tag_in = Reg(UInt(width = mifTagBits)) - - if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val mif_done_in = Reg(init=Bool(false)) - val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data } - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = tag_in(0), - g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) - - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - when(active_in) { - gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data - } - } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } - } - } else { // Don't generate all the uneeded data buffers and flow resp - gnt_arb.io.in(0).valid := io.mem.resp.valid - io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = io.mem.resp.bits.tag(0), - g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) - } -} - -class HellaFlowQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - require(entries > 1) - - val do_flow = Wire(Bool()) - val do_enq = io.enq.fire() && !do_flow - val do_deq = io.deq.fire() && !do_flow - - val maybe_full = Reg(init=Bool(false)) - val enq_ptr = Counter(do_enq, entries)._1 - val (deq_ptr, deq_done) = Counter(do_deq, entries) - when (do_enq != do_deq) { maybe_full := do_enq } - - val ptr_match = enq_ptr === deq_ptr - val empty = ptr_match && !maybe_full - val full = ptr_match && maybe_full - val atLeastTwo = full || enq_ptr - deq_ptr >= UInt(2) - do_flow := empty && io.deq.ready - - val ram = SeqMem(data, entries) - when (do_enq) { ram.write(enq_ptr, io.enq.bits) } - - val ren = io.deq.ready && (atLeastTwo || !io.deq.valid && !empty) - val raddr = Mux(io.deq.valid, Mux(deq_done, UInt(0), deq_ptr + UInt(1)), deq_ptr) - val ram_out_valid = Reg(next = ren) - - io.deq.valid := Mux(empty, io.enq.valid, ram_out_valid) - io.enq.ready := !full - io.deq.bits := Mux(empty, io.enq.bits, ram.read(raddr, ren)) -} - -class HellaQueue[T <: Data](val entries: Int)(data: => T) extends Module -{ - val io = new QueueIO(data, entries) - - val fq = Module(new HellaFlowQueue(entries)(data)) - io.enq <> fq.io.enq - io.deq <> Queue(fq.io.deq, 1, pipe = true) -} - -object HellaQueue -{ - def apply[T <: Data](enq: DecoupledIO[T], entries: Int) = { - val q = Module((new HellaQueue(entries)) { enq.bits }) - q.io.enq.valid := enq.valid // not using <> so that override is allowed - q.io.enq.bits := enq.bits - enq.ready := q.io.enq.ready - q.io.deq - } -} - -class MemIOArbiter(val arbN: Int) extends MIFModule { - val io = new Bundle { - val inner = Vec.fill(arbN){new MemIO}.flip - val outer = new MemIO - } - - if(arbN > 1) { - val cmd_arb = Module(new RRArbiter(new MemReqCmd, arbN)) - val choice_q = Module(new Queue(cmd_arb.io.chosen, 4)) - val (data_cnt, data_done) = Counter(io.outer.req_data.fire(), mifDataBeats) - - io.inner.map(_.req_cmd).zipWithIndex.zip(cmd_arb.io.in).map{ case ((req, id), arb) => { - arb.valid := req.valid - arb.bits := req.bits - arb.bits.tag := Cat(req.bits.tag, UInt(id)) - req.ready := arb.ready - }} - io.outer.req_cmd.bits := cmd_arb.io.out.bits - io.outer.req_cmd.valid := cmd_arb.io.out.valid && choice_q.io.enq.ready - cmd_arb.io.out.ready := io.outer.req_cmd.ready && choice_q.io.enq.ready - choice_q.io.enq.bits := cmd_arb.io.chosen - choice_q.io.enq.valid := cmd_arb.io.out.fire() && cmd_arb.io.out.bits.rw - - io.outer.req_data.bits := io.inner(choice_q.io.deq.bits).req_data.bits - io.outer.req_data.valid := io.inner(choice_q.io.deq.bits).req_data.valid && choice_q.io.deq.valid - io.inner.map(_.req_data.ready).zipWithIndex.foreach { - case(r, i) => r := UInt(i) === choice_q.io.deq.bits && choice_q.io.deq.valid - } - choice_q.io.deq.ready := data_done - - io.outer.resp.ready := Bool(false) - for (i <- 0 until arbN) { - io.inner(i).resp.valid := Bool(false) - when(io.outer.resp.bits.tag(log2Up(arbN)-1,0).toUInt === UInt(i)) { - io.inner(i).resp.valid := io.outer.resp.valid - io.outer.resp.ready := io.inner(i).resp.ready - } - io.inner(i).resp.bits := io.outer.resp.bits - io.inner(i).resp.bits.tag := io.outer.resp.bits.tag >> UInt(log2Up(arbN)) - } - } else { io.inner.head <> io.outer } -} - -object MemIOMemPipeIOConverter { - def apply(in: MemPipeIO): MemIO = { - val out = Wire(new MemIO()) - in.resp.valid := out.resp.valid - in.resp.bits := out.resp.bits - out.resp.ready := Bool(true) - out.req_cmd.valid := in.req_cmd.valid - out.req_cmd.bits := in.req_cmd.bits - in.req_cmd.ready := out.req_cmd.ready - out.req_data.valid := in.req_data.valid - out.req_data.bits := in.req_data.bits - in.req_data.ready := out.req_data.ready - out - } -} - -class MemPipeIOMemIOConverter(numRequests: Int) extends MIFModule { - val io = new Bundle { - val cpu = new MemIO().flip - val mem = new MemPipeIO - } - - val numEntries = numRequests * mifDataBeats - val size = log2Down(numEntries) + 1 - - val inc = Wire(Bool()) - val dec = Wire(Bool()) - val count = Reg(init=UInt(numEntries, size)) - val watermark = count >= UInt(mifDataBeats) - - when (inc && !dec) { - count := count + UInt(1) - } - when (!inc && dec) { - count := count - UInt(mifDataBeats) - } - when (inc && dec) { - count := count - UInt(mifDataBeats-1) - } - - val cmdq_mask = io.cpu.req_cmd.bits.rw || watermark - - io.mem.req_cmd.valid := io.cpu.req_cmd.valid && cmdq_mask - io.cpu.req_cmd.ready := io.mem.req_cmd.ready && cmdq_mask - io.mem.req_cmd.bits := io.cpu.req_cmd.bits - - io.mem.req_data <> io.cpu.req_data - - // Have separate queues to allow for different mem implementations - val resp_data_q = Module((new HellaQueue(numEntries)) { new MemData }) - resp_data_q.io.enq.valid := io.mem.resp.valid - resp_data_q.io.enq.bits.data := io.mem.resp.bits.data - - val resp_tag_q = Module((new HellaQueue(numEntries)) { new MemTag }) - resp_tag_q.io.enq.valid := io.mem.resp.valid - resp_tag_q.io.enq.bits.tag := io.mem.resp.bits.tag - - io.cpu.resp.valid := resp_data_q.io.deq.valid && resp_tag_q.io.deq.valid - io.cpu.resp.bits.data := resp_data_q.io.deq.bits.data - io.cpu.resp.bits.tag := resp_tag_q.io.deq.bits.tag - resp_data_q.io.deq.ready := io.cpu.resp.ready - resp_tag_q.io.deq.ready := io.cpu.resp.ready - - inc := resp_data_q.io.deq.fire() && resp_tag_q.io.deq.fire() - dec := io.mem.req_cmd.fire() && !io.mem.req_cmd.bits.rw -} - -class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { - val io = new Bundle { - val tl = new ManagerTileLinkIO - val mem = new MemPipeIO - } - - val a = Module(new MemIOTileLinkIOConverter(1)) - val b = Module(new MemPipeIOMemIOConverter(outstanding)) - a.io.tl <> io.tl - b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) - a.io.mem.resp <> b.io.cpu.resp - b.io.mem <> io.mem -} diff --git a/uncore/src/main/scala/nasti.scala b/uncore/src/main/scala/nasti.scala deleted file mode 100644 index 904a596e..00000000 --- a/uncore/src/main/scala/nasti.scala +++ /dev/null @@ -1,295 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ -import scala.math.max - -case object NASTIDataBits extends Field[Int] -case object NASTIAddrBits extends Field[Int] -case object NASTIIdBits extends Field[Int] - -trait NASTIParameters extends UsesParameters { - val nastiXDataBits = params(NASTIDataBits) - val nastiWStrobeBits = nastiXDataBits / 8 - val nastiXAddrBits = params(NASTIAddrBits) - val nastiWIdBits = params(NASTIIdBits) - val nastiRIdBits = params(NASTIIdBits) - val nastiXIdBits = max(nastiWIdBits, nastiRIdBits) - val nastiXUserBits = 1 - val nastiAWUserBits = nastiXUserBits - val nastiWUserBits = nastiXUserBits - val nastiBUserBits = nastiXUserBits - val nastiARUserBits = nastiXUserBits - val nastiRUserBits = nastiXUserBits - val nastiXLenBits = 8 - val nastiXSizeBits = 3 - val nastiXBurstBits = 2 - val nastiXCacheBits = 4 - val nastiXProtBits = 3 - val nastiXQosBits = 4 - val nastiXRegionBits = 4 - val nastiXRespBits = 2 - - def bytesToXSize(bytes: UInt) = MuxLookup(bytes, UInt("b111"), Array( - UInt(1) -> UInt(0), - UInt(2) -> UInt(1), - UInt(4) -> UInt(2), - UInt(8) -> UInt(3), - UInt(16) -> UInt(4), - UInt(32) -> UInt(5), - UInt(64) -> UInt(6), - UInt(128) -> UInt(7))) -} - -abstract class NASTIBundle extends Bundle with NASTIParameters -abstract class NASTIModule extends Module with NASTIParameters - -trait NASTIChannel extends NASTIBundle -trait NASTIMasterToSlaveChannel extends NASTIChannel -trait NASTISlaveToMasterChannel extends NASTIChannel - -class NASTIMasterIO extends Bundle { - val aw = Decoupled(new NASTIWriteAddressChannel) - val w = Decoupled(new NASTIWriteDataChannel) - val b = Decoupled(new NASTIWriteResponseChannel).flip - val ar = Decoupled(new NASTIReadAddressChannel) - val r = Decoupled(new NASTIReadDataChannel).flip -} - -class NASTISlaveIO extends NASTIMasterIO { flip() } - -trait HasNASTIMetadata extends NASTIBundle { - val addr = UInt(width = nastiXAddrBits) - val len = UInt(width = nastiXLenBits) - val size = UInt(width = nastiXSizeBits) - val burst = UInt(width = nastiXBurstBits) - val lock = Bool() - val cache = UInt(width = nastiXCacheBits) - val prot = UInt(width = nastiXProtBits) - val qos = UInt(width = nastiXQosBits) - val region = UInt(width = nastiXRegionBits) -} - -trait HasNASTIData extends NASTIBundle { - val data = UInt(width = nastiXDataBits) - val last = Bool() -} - -class NASTIAddressChannel extends NASTIMasterToSlaveChannel with HasNASTIMetadata - -class NASTIResponseChannel extends NASTISlaveToMasterChannel { - val resp = UInt(width = nastiXRespBits) -} - -class NASTIWriteAddressChannel extends NASTIAddressChannel { - val id = UInt(width = nastiWIdBits) - val user = UInt(width = nastiAWUserBits) -} - -class NASTIWriteDataChannel extends NASTIMasterToSlaveChannel with HasNASTIData { - val strb = UInt(width = nastiWStrobeBits) - val user = UInt(width = nastiWUserBits) -} - -class NASTIWriteResponseChannel extends NASTIResponseChannel { - val id = UInt(width = nastiWIdBits) - val user = UInt(width = nastiBUserBits) -} - -class NASTIReadAddressChannel extends NASTIAddressChannel { - val id = UInt(width = nastiRIdBits) - val user = UInt(width = nastiARUserBits) -} - -class NASTIReadDataChannel extends NASTIResponseChannel with HasNASTIData { - val id = UInt(width = nastiRIdBits) - val user = UInt(width = nastiRUserBits) -} - -class MemIONASTISlaveIOConverter extends MIFModule with NASTIParameters { - val io = new Bundle { - val nasti = new NASTISlaveIO - val mem = new MemIO - } - - require(mifDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") - val (mif_cnt_out, mif_wrap_out) = Counter(io.mem.resp.fire(), mifDataBeats) - - io.mem.req_cmd.bits.addr := Mux(io.nasti.aw.valid, io.nasti.aw.bits.addr, io.nasti.ar.bits.addr) >> - UInt(params(CacheBlockOffsetBits)) - io.mem.req_cmd.bits.tag := Mux(io.nasti.aw.valid, io.nasti.aw.bits.id, io.nasti.ar.bits.id) - io.mem.req_cmd.bits.rw := io.nasti.aw.valid - io.mem.req_cmd.valid := (io.nasti.aw.valid && io.nasti.b.ready) || io.nasti.ar.valid - io.nasti.ar.ready := io.mem.req_cmd.ready && !io.nasti.aw.valid - io.nasti.aw.ready := io.mem.req_cmd.ready && io.nasti.b.ready - - io.nasti.b.valid := io.nasti.aw.valid && io.mem.req_cmd.ready - io.nasti.b.bits.id := io.nasti.aw.bits.id - io.nasti.b.bits.resp := UInt(0) - - io.nasti.w.ready := io.mem.req_data.ready - io.mem.req_data.valid := io.nasti.w.valid - io.mem.req_data.bits.data := io.nasti.w.bits.data - assert(!io.nasti.w.valid || io.nasti.w.bits.strb.andR, "MemIO must write full cache line") - - io.nasti.r.valid := io.mem.resp.valid - io.nasti.r.bits.data := io.mem.resp.bits.data - io.nasti.r.bits.last := mif_wrap_out - io.nasti.r.bits.id := io.mem.resp.bits.tag - io.nasti.r.bits.resp := UInt(0) - io.mem.resp.ready := io.nasti.r.ready -} - -class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { - val io = new Bundle { - val tl = new ManagerTileLinkIO - val nasti = new NASTIMasterIO - } - - val dataBits = tlDataBits*tlDataBeats - val dstIdBits = params(LNHeaderBits) - require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction - require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIMasterIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) - - io.tl.acquire.ready := Bool(false) - io.tl.probe.valid := Bool(false) - io.tl.release.ready := Bool(false) - io.tl.finish.ready := Bool(true) - - io.nasti.b.ready := Bool(false) - io.nasti.r.ready := Bool(false) - io.nasti.ar.valid := Bool(false) - io.nasti.aw.valid := Bool(false) - io.nasti.w.valid := Bool(false) - - val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.hasData() - val rel_has_data = io.tl.release.bits.hasData() - val is_write = io.tl.release.valid || (io.tl.acquire.valid && acq_has_data) - - // Decompose outgoing TL Acquires into NASTI address and data channels - val active_out = Reg(init=Bool(false)) - val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(UInt(width = nastiXIdBits)) - val addr_out = Reg(UInt(width = nastiXAddrBits)) - val has_data = Reg(init=Bool(false)) - val data_from_rel = Reg(init=Bool(false)) - val (tl_cnt_out, tl_wrap_out) = - Counter((io.tl.acquire.fire() && acq_has_data) || - (io.tl.release.fire() && rel_has_data), tlDataBeats) - val tl_done_out = Reg(init=Bool(false)) - - io.nasti.ar.bits.id := tag_out - io.nasti.ar.bits.addr := addr_out - io.nasti.ar.bits.len := Mux(has_data, UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := UInt(log2Ceil(tlDataBits)) - io.nasti.ar.bits.burst := UInt("b01") - io.nasti.ar.bits.lock := Bool(false) - io.nasti.ar.bits.cache := UInt("b0000") - io.nasti.ar.bits.prot := UInt("b000") - io.nasti.ar.bits.qos := UInt("b0000") - io.nasti.ar.bits.region := UInt("b0000") - io.nasti.ar.bits.user := UInt(0) - io.nasti.aw.bits := io.nasti.ar.bits - io.nasti.w.bits.strb := Mux(data_from_rel, SInt(-1), io.tl.acquire.bits.wmask()) - io.nasti.w.bits.data := Mux(data_from_rel, io.tl.release.bits.data, io.tl.acquire.bits.data) - io.nasti.w.bits.last := tl_wrap_out - - when(!active_out){ - io.tl.release.ready := io.nasti.w.ready - io.tl.acquire.ready := io.nasti.w.ready && !io.tl.release.valid - io.nasti.w.valid := (io.tl.release.valid && rel_has_data) || - (io.tl.acquire.valid && acq_has_data) - when(io.nasti.w.ready && (io.tl.release.valid || io.tl.acquire.valid)) { - active_out := (!is_write && !io.nasti.ar.ready) || - (is_write && !(io.nasti.aw.ready && io.nasti.w.ready)) || - (io.nasti.w.valid && Bool(tlDataBeats > 1)) - io.nasti.aw.valid := is_write - io.nasti.ar.valid := !is_write - cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) - tl_done_out := tl_wrap_out - when(io.tl.release.valid) { - data_from_rel := Bool(true) - io.nasti.w.bits.data := io.tl.release.bits.data - io.nasti.w.bits.strb := SInt(-1) - val tag = Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - val addr = io.tl.release.bits.full_addr() - io.nasti.aw.bits.id := tag - io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := UInt(tlDataBeats-1) - io.nasti.aw.bits.size := MT_Q - tag_out := tag - addr_out := addr - has_data := rel_has_data - } .elsewhen(io.tl.acquire.valid) { - data_from_rel := Bool(false) - io.nasti.w.bits.data := io.tl.acquire.bits.data - io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() - val tag = Cat(io.tl.acquire.bits.client_id, - io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) - val addr = io.tl.acquire.bits.full_addr() - when(is_write) { - io.nasti.aw.bits.id := tag - io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.putBlockType), - UInt(tlDataBeats-1), UInt(0)) - io.nasti.aw.bits.size := bytesToXSize(PopCount(io.tl.acquire.bits.wmask())) - } .otherwise { - io.nasti.ar.bits.id := tag - io.nasti.ar.bits.addr := addr - io.nasti.ar.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.getBlockType), - UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() - } - tag_out := tag - addr_out := addr - has_data := acq_has_data - } - } - } - when(active_out) { - io.nasti.ar.valid := !cmd_sent_out && !has_data - io.nasti.aw.valid := !cmd_sent_out && has_data - cmd_sent_out := cmd_sent_out || io.nasti.ar.fire() || io.nasti.aw.fire() - when(has_data && !tl_done_out) { - when(data_from_rel) { - io.tl.release.ready := io.nasti.w.ready - io.nasti.w.valid := io.tl.release.valid - } .otherwise { - io.tl.acquire.ready := io.nasti.w.ready - io.nasti.w.valid := io.tl.acquire.valid - } - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || tl_done_out)) { active_out := Bool(false) } - } - - // Aggregate incoming NASTI responses into TL Grants - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - gnt_arb.io.in(0).valid := io.nasti.r.valid - io.nasti.r.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = io.nasti.r.bits.id(0), - g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.nasti.r.bits.id >> UInt(1), - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.nasti.r.bits.data) - - gnt_arb.io.in(1).valid := io.nasti.b.valid - io.nasti.b.ready := gnt_arb.io.in(1).ready - gnt_arb.io.in(1).bits := Grant( - dst = (if(dstIdBits > 0) io.nasti.b.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = Bool(true), - g_type = Mux(io.nasti.b.bits.id(0), Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = io.nasti.b.bits.id >> UInt(1), - manager_xact_id = UInt(0)) -} diff --git a/uncore/src/main/scala/slowio.scala b/uncore/src/main/scala/slowio.scala deleted file mode 100644 index 95ca34e6..00000000 --- a/uncore/src/main/scala/slowio.scala +++ /dev/null @@ -1,70 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ - -class SlowIO[T <: Data](val divisor_max: Int)(data: => T) extends Module -{ - val io = new Bundle { - val out_fast = Decoupled(data).flip - val out_slow = Decoupled(data) - val in_fast = Decoupled(data) - val in_slow = Decoupled(data).flip - val clk_slow = Bool(OUTPUT) - val set_divisor = Valid(Bits(width = 32)).flip - val divisor = Bits(OUTPUT, 32) - } - - require(divisor_max >= 8 && divisor_max <= 65536 && isPow2(divisor_max)) - val divisor = Reg(init=UInt(divisor_max-1)) - val d_shadow = Reg(init=UInt(divisor_max-1)) - val hold = Reg(init=UInt(divisor_max/4-1)) - val h_shadow = Reg(init=UInt(divisor_max/4-1)) - when (io.set_divisor.valid) { - d_shadow := io.set_divisor.bits(log2Up(divisor_max)-1, 0).toUInt - h_shadow := io.set_divisor.bits(log2Up(divisor_max)-1+16, 16).toUInt - } - io.divisor := hold << UInt(16) | divisor - - val count = Reg{UInt(width = log2Up(divisor_max))} - val myclock = Reg{Bool()} - count := count + UInt(1) - - val rising = count === (divisor >> UInt(1)) - val falling = count === divisor - val held = count === (divisor >> UInt(1)) + hold - - when (falling) { - divisor := d_shadow - hold := h_shadow - count := UInt(0) - myclock := Bool(false) - } - when (rising) { - myclock := Bool(true) - } - - val in_slow_rdy = Reg(init=Bool(false)) - val out_slow_val = Reg(init=Bool(false)) - val out_slow_bits = Reg(data) - - val fromhost_q = Module(new Queue(data,1)) - fromhost_q.io.enq.valid := rising && (io.in_slow.valid && in_slow_rdy || this.reset) - fromhost_q.io.enq.bits := io.in_slow.bits - fromhost_q.io.deq <> io.in_fast - - val tohost_q = Module(new Queue(data,1)) - tohost_q.io.enq <> io.out_fast - tohost_q.io.deq.ready := rising && io.out_slow.ready && out_slow_val - - when (held) { - in_slow_rdy := fromhost_q.io.enq.ready - out_slow_val := tohost_q.io.deq.valid - out_slow_bits := Mux(this.reset, fromhost_q.io.deq.bits, tohost_q.io.deq.bits) - } - - io.in_slow.ready := in_slow_rdy - io.out_slow.valid := out_slow_val - io.out_slow.bits := out_slow_bits - io.clk_slow := myclock -} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6c7aeb6e..71c8d909 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import junctions._ import scala.math.max /** Parameters exposed to the top-level design, set based on @@ -1237,3 +1238,422 @@ trait HasDataBeatCounters { (cnt > UInt(0), up_idx, up_done, down_idx, down_done) } } + +class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val nasti = new NASTIMasterIO + } + + val dataBits = tlDataBits*tlDataBeats + val dstIdBits = params(LNHeaderBits) + require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction + require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") + require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIMasterIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + + io.nasti.b.ready := Bool(false) + io.nasti.r.ready := Bool(false) + io.nasti.ar.valid := Bool(false) + io.nasti.aw.valid := Bool(false) + io.nasti.w.valid := Bool(false) + + val dst_off = dstIdBits + tlClientXactIdBits + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() + val is_write = io.tl.release.valid || (io.tl.acquire.valid && acq_has_data) + + // Decompose outgoing TL Acquires into NASTI address and data channels + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val tag_out = Reg(UInt(width = nastiXIdBits)) + val addr_out = Reg(UInt(width = nastiXAddrBits)) + val has_data = Reg(init=Bool(false)) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + + io.nasti.ar.bits.id := tag_out + io.nasti.ar.bits.addr := addr_out + io.nasti.ar.bits.len := Mux(has_data, UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.size := UInt(log2Ceil(tlDataBits)) + io.nasti.ar.bits.burst := UInt("b01") + io.nasti.ar.bits.lock := Bool(false) + io.nasti.ar.bits.cache := UInt("b0000") + io.nasti.ar.bits.prot := UInt("b000") + io.nasti.ar.bits.qos := UInt("b0000") + io.nasti.ar.bits.region := UInt("b0000") + io.nasti.ar.bits.user := UInt(0) + io.nasti.aw.bits := io.nasti.ar.bits + io.nasti.w.bits.strb := Mux(data_from_rel, SInt(-1), io.tl.acquire.bits.wmask()) + io.nasti.w.bits.data := Mux(data_from_rel, io.tl.release.bits.data, io.tl.acquire.bits.data) + io.nasti.w.bits.last := tl_wrap_out + + when(!active_out){ + io.tl.release.ready := io.nasti.w.ready + io.tl.acquire.ready := io.nasti.w.ready && !io.tl.release.valid + io.nasti.w.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.nasti.w.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := (!is_write && !io.nasti.ar.ready) || + (is_write && !(io.nasti.aw.ready && io.nasti.w.ready)) || + (io.nasti.w.valid && Bool(tlDataBeats > 1)) + io.nasti.aw.valid := is_write + io.nasti.ar.valid := !is_write + cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + io.nasti.w.bits.data := io.tl.release.bits.data + io.nasti.w.bits.strb := SInt(-1) + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.full_addr() + io.nasti.aw.bits.id := tag + io.nasti.aw.bits.addr := addr + io.nasti.aw.bits.len := UInt(tlDataBeats-1) + io.nasti.aw.bits.size := MT_Q + tag_out := tag + addr_out := addr + has_data := rel_has_data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + io.nasti.w.bits.data := io.tl.acquire.bits.data + io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.full_addr() + when(is_write) { + io.nasti.aw.bits.id := tag + io.nasti.aw.bits.addr := addr + io.nasti.aw.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.putBlockType), + UInt(tlDataBeats-1), UInt(0)) + io.nasti.aw.bits.size := bytesToXSize(PopCount(io.tl.acquire.bits.wmask())) + } .otherwise { + io.nasti.ar.bits.id := tag + io.nasti.ar.bits.addr := addr + io.nasti.ar.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.getBlockType), + UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() + } + tag_out := tag + addr_out := addr + has_data := acq_has_data + } + } + } + when(active_out) { + io.nasti.ar.valid := !cmd_sent_out && !has_data + io.nasti.aw.valid := !cmd_sent_out && has_data + cmd_sent_out := cmd_sent_out || io.nasti.ar.fire() || io.nasti.aw.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.nasti.w.ready + io.nasti.w.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.nasti.w.ready + io.nasti.w.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(cmd_sent_out && (!has_data || tl_done_out)) { active_out := Bool(false) } + } + + // Aggregate incoming NASTI responses into TL Grants + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.nasti.r.bits.id(0), + g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.nasti.r.bits.id >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.nasti.r.bits.data) + + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) io.nasti.b.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(io.nasti.b.bits.id(0), Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = io.nasti.b.bits.id >> UInt(1), + manager_xact_id = UInt(0)) +} + +class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val mem = new MemPipeIO + } + + val a = Module(new MemIOTileLinkIOConverter(1)) + val b = Module(new MemPipeIOMemIOConverter(outstanding)) + a.io.tl <> io.tl + b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) + b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) + a.io.mem.resp <> b.io.cpu.resp + b.io.mem <> io.mem +} + +//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO +class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { + val io = new Bundle { + val tl = new ManagerTileLinkIO + val mem = new MemIO + } + val dataBits = tlDataBits*tlDataBeats + val dstIdBits = params(LNHeaderBits) + require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") + require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) + + io.tl.acquire.ready := Bool(false) + io.tl.probe.valid := Bool(false) + io.tl.release.ready := Bool(false) + io.tl.finish.ready := Bool(true) + io.mem.resp.ready := Bool(false) + + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + val dst_off = dstIdBits + tlClientXactIdBits + val acq_has_data = io.tl.acquire.bits.hasData() + val rel_has_data = io.tl.release.bits.hasData() + + // Decompose outgoing TL Acquires into MemIO cmd and data + val active_out = Reg(init=Bool(false)) + val cmd_sent_out = Reg(init=Bool(false)) + val tag_out = Reg(UInt(width = mifTagBits)) + val addr_out = Reg(UInt(width = mifAddrBits)) + val has_data = Reg(init=Bool(false)) + val data_from_rel = Reg(init=Bool(false)) + val (tl_cnt_out, tl_wrap_out) = + Counter((io.tl.acquire.fire() && acq_has_data) || + (io.tl.release.fire() && rel_has_data), tlDataBeats) + val tl_done_out = Reg(init=Bool(false)) + val make_grant_ack = Reg(init=Bool(false)) + + gnt_arb.io.in(1).valid := Bool(false) + gnt_arb.io.in(1).bits := Grant( + dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = Bool(true), + g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), + client_xact_id = tag_out >> UInt(1), + manager_xact_id = UInt(0)) + + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) + val mem_data_q = Module(new Queue(new MemData, qDepth)) + mem_cmd_q.io.enq.valid := Bool(false) + mem_data_q.io.enq.valid := Bool(false) + val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) + val mif_done_out = Reg(init=Bool(false)) + val tl_buf_out = Reg(Vec(io.tl.acquire.bits.data, tlDataBeats)) + val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) + val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) + val tl_prog_out = tl_cnt_out*UInt(tlDataBits) + + when(!active_out){ + io.tl.release.ready := Bool(true) + io.tl.acquire.ready := !io.tl.release.valid + when(io.tl.release.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + addr_out := io.tl.release.bits.addr_block + has_data := rel_has_data + data_from_rel := Bool(true) + make_grant_ack := io.tl.release.bits.requiresAck() + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.release.bits.data + } .elsewhen(io.tl.acquire.valid) { + active_out := Bool(true) + cmd_sent_out := Bool(false) + tag_out := Cat(io.tl.release.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + addr_out := io.tl.acquire.bits.addr_block + has_data := acq_has_data + data_from_rel := Bool(false) + make_grant_ack := acq_has_data + tl_done_out := tl_wrap_out + tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data + } + } + when(active_out) { + mem_cmd_q.io.enq.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() + when(has_data) { + when(!tl_done_out) { + io.tl.acquire.ready := Bool(true) + when(io.tl.acquire.valid) { + tl_buf_out(tl_cnt_out) := Mux(data_from_rel, + io.tl.release.bits.data, + io.tl.acquire.bits.data) + } + } + when(!mif_done_out) { + mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(mif_wrap_out) { mif_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { + active_out := Bool(false) + } + } + + mem_cmd_q.io.enq.bits.rw := has_data + mem_cmd_q.io.enq.bits.tag := tag_out + mem_cmd_q.io.enq.bits.addr := addr_out + mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data + io.mem.req_cmd <> mem_cmd_q.io.deq + io.mem.req_data <> mem_data_q.io.deq + } else { // Don't make the data buffers and try to flow cmd and data + io.mem.req_cmd.valid := Bool(false) + io.mem.req_data.valid := Bool(false) + io.mem.req_cmd.bits.rw := has_data + io.mem.req_cmd.bits.tag := tag_out + io.mem.req_cmd.bits.addr := addr_out + io.mem.req_data.bits.data := Mux(data_from_rel, + io.tl.release.bits.data, + io.tl.acquire.bits.data) + when(!active_out){ + io.tl.release.ready := io.mem.req_data.ready + io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid + io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || + (io.tl.acquire.valid && acq_has_data) + when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid + io.mem.req_cmd.valid := Bool(true) + cmd_sent_out := io.mem.req_cmd.ready + tl_done_out := tl_wrap_out + when(io.tl.release.valid) { + data_from_rel := Bool(true) + make_grant_ack := io.tl.release.bits.requiresAck() + io.mem.req_data.bits.data := io.tl.release.bits.data + val tag = Cat(io.tl.release.bits.client_id, + io.tl.release.bits.client_xact_id, + io.tl.release.bits.isVoluntary()) + val addr = io.tl.release.bits.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := rel_has_data + tag_out := tag + addr_out := addr + has_data := rel_has_data + } .elsewhen(io.tl.acquire.valid) { + data_from_rel := Bool(false) + make_grant_ack := acq_has_data // i.e. is it a Put + io.mem.req_data.bits.data := io.tl.acquire.bits.data + io.mem.req_cmd.bits.rw := acq_has_data + val tag = Cat(io.tl.acquire.bits.client_id, + io.tl.acquire.bits.client_xact_id, + io.tl.acquire.bits.isBuiltInType()) + val addr = io.tl.acquire.bits.addr_block + io.mem.req_cmd.bits.tag := tag + io.mem.req_cmd.bits.addr := addr + io.mem.req_cmd.bits.rw := acq_has_data + tag_out := tag + addr_out := addr + has_data := acq_has_data + } + } + } + when(active_out) { + io.mem.req_cmd.valid := !cmd_sent_out + cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() + when(has_data && !tl_done_out) { + when(data_from_rel) { + io.tl.release.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.release.valid + } .otherwise { + io.tl.acquire.ready := io.mem.req_data.ready + io.mem.req_data.valid := io.tl.acquire.valid + } + } + when(tl_wrap_out) { tl_done_out := Bool(true) } + when(tl_done_out && make_grant_ack) { + gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? + when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } + } + when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { + active_out := Bool(false) + } + } + } + + // Aggregate incoming MemIO responses into TL Grants + val active_in = Reg(init=Bool(false)) + val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val tag_in = Reg(UInt(width = mifTagBits)) + + if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { + val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data + val mif_done_in = Reg(init=Bool(false)) + val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) + val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data } + tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) + val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) + val mif_prog_in = mif_cnt_in*UInt(mifDataBits) + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = tag_in(0), + g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = tag_in >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = tl_buf_in(tl_cnt_in)) + + when(!active_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + active_in := Bool(true) + mif_done_in := mif_wrap_in + tag_in := io.mem.resp.bits.tag + mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data + } + } + when(active_in) { + gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in + when(!mif_done_in) { + io.mem.resp.ready := Bool(true) + when(io.mem.resp.valid) { + mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data + } + } + when(mif_wrap_in) { mif_done_in := Bool(true) } + when(tl_wrap_in) { active_in := Bool(false) } + } + } else { // Don't generate all the uneeded data buffers and flow resp + gnt_arb.io.in(0).valid := io.mem.resp.valid + io.mem.resp.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), + is_builtin_type = io.mem.resp.bits.tag(0), + g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + client_xact_id = io.mem.resp.bits.tag >> UInt(1), + manager_xact_id = UInt(0), + addr_beat = tl_cnt_in, + data = io.mem.resp.bits.data) + } +} From 4c0f996808c6c1bea15a63af830e8e4ac047ad67 Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 30 Jul 2015 14:50:28 -0700 Subject: [PATCH 403/688] Fix typo (juntion -> junctions). --- uncore/build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index a34b2b9b..6e53ae86 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -7,7 +7,7 @@ name := "uncore" scalaVersion := "2.10.2" // Provide a managed dependency on X if -DXVersion="" is supplied on the command line. -libraryDependencies ++= (Seq("chisel","junction").map { +libraryDependencies ++= (Seq("chisel","junctions").map { dep: String => sys.props.get(dep + "Version") map { "edu.berkeley.cs" %% dep % _ }}).flatten site.settings From 6c391e3b372242a24dc2e250a4f5e36bd2bafb64 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 30 Jul 2015 23:45:48 -0700 Subject: [PATCH 404/688] Use UInt(0), not UInt(width=0), for constant 0 --- uncore/src/main/scala/util.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index ff982e3f..58305e52 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -54,7 +54,7 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module if(n == 1) { io.in <> io.out - io.cnt := UInt(width = 0) + io.cnt := UInt(0) io.done := Bool(true) } else { val cnt = Reg(init=UInt(0, width = log2Up(n))) From 8f7b3903538748e5b93f911be3b0aff737645990 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 30 Jul 2015 23:46:32 -0700 Subject: [PATCH 405/688] UInt-> Bits; avoid mixed UInt/SInt code --- uncore/src/main/scala/cache.scala | 6 +++--- uncore/src/main/scala/ecc.scala | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b956a2e0..6e5872c2 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -36,7 +36,7 @@ abstract trait CacheParameters extends UsesParameters { abstract class CacheBundle extends Bundle with CacheParameters abstract class CacheModule extends Module with CacheParameters -class StoreGen(typ: Bits, addr: Bits, dat: Bits) { +class StoreGen(typ: UInt, addr: UInt, dat: UInt) { val byte = typ === MT_B || typ === MT_BU val half = typ === MT_H || typ === MT_HU val word = typ === MT_W || typ === MT_WU @@ -54,7 +54,7 @@ class StoreGen(typ: Bits, addr: Bits, dat: Bits) { dat) } -class LoadGen(typ: Bits, addr: Bits, dat: Bits, zero: Bool) { +class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool) { val t = new StoreGen(typ, addr, dat) val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D @@ -87,7 +87,7 @@ class AMOALU extends CacheModule { val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: io.typ === MT_B || io.typ === MT_BU - val mask = SInt(-1,64) ^ (io.addr(2) << UInt(31)) + val mask = ~UInt(0,64) ^ (io.addr(2) << UInt(31)) val adder_out = (io.lhs & mask).toUInt + (rhs & mask) val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala index b5864b2c..609e5fca 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/ecc.scala @@ -107,11 +107,11 @@ class SECDEDCode extends Code object ErrGen { // generate a 1-bit error with approximate probability 2^-f - def apply(width: Int, f: Int): Bits = { + def apply(width: Int, f: Int): UInt = { require(width > 0 && f >= 0 && log2Up(width) + f <= 16) UIntToOH(LFSR16()(log2Up(width)+f-1,0))(width-1,0) } - def apply(x: Bits, f: Int): Bits = x ^ apply(x.getWidth, f) + def apply(x: UInt, f: Int): UInt = x ^ apply(x.getWidth, f) } class SECDEDTest extends Module From 0686bdbe28aafc33bad5c4a2f01db361f4b0c8b7 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 30 Jul 2015 23:47:13 -0700 Subject: [PATCH 406/688] Avoid cross-module references You can't instantiate a Vec in one module and use it in another. An idiosyncrasy of the Chisel2 implementation let this one slip by. In this case, it's just a matter of using def instead of val. --- uncore/src/main/scala/coherence.scala | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 862eb484..f24bae17 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -29,8 +29,8 @@ trait HasCustomTileLinkMessageTypes { def grantTypeWidth = log2Up(nGrantTypes) val acquireTypesWithData = Nil // Only built-in Acquire types have data for now - val releaseTypesWithData: Vec[UInt] - val grantTypesWithData: Vec[UInt] + def releaseTypesWithData: Vec[UInt] + def grantTypesWithData: Vec[UInt] } /** This API contains all functions required for client coherence agents. @@ -129,8 +129,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) - val grantTypesWithData = Vec(grantExclusive) + def releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) + def grantTypesWithData = Vec(grantExclusive) // Client states and functions val nClientStates = 2 @@ -220,8 +220,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantExclusive) + def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Vec(grantExclusive) // Client states and functions val nClientStates = 3 @@ -322,8 +322,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) + def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Vec(grantShared, grantExclusive) // Client states and functions val nClientStates = 3 @@ -440,8 +440,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - val grantTypesWithData = Vec(grantShared, grantExclusive) + def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Vec(grantShared, grantExclusive) // Client states and functions val nClientStates = 4 @@ -555,8 +555,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - val releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - val grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + def grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) // Client states and functions val nClientStates = 7 From 377e17e811095708fe94ead60855a01182e90ea7 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 31 Jul 2015 00:32:02 -0700 Subject: [PATCH 407/688] Add Wire() wrap --- uncore/src/main/scala/broadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 67d3c983..087d16db 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -18,7 +18,7 @@ class DataQueueLocation extends Bundle with BroadcastHubParameters { object DataQueueLocation { def apply(idx: UInt, loc: UInt) = { - val d = new DataQueueLocation + val d = Wire(new DataQueueLocation) d.idx := idx d.loc := loc d From 6d574f8c1b2061077db52c3548a1a0778262c35c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 31 Jul 2015 00:59:34 -0700 Subject: [PATCH 408/688] Fix incompatible assignment --- uncore/src/main/scala/broadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 087d16db..cd69651c 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -37,7 +37,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) // Propagate incoherence flags - trackerList.map(_.io.incoherent := io.incoherent.toBits) + trackerList.map(_.io.incoherent := io.incoherent) // Queue to store impending Put data val sdq = Reg(Vec.fill(sdqDepth){io.iacq().data}) From 6fc807f06914247e58a317380150b4a14d784d86 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 1 Aug 2015 21:08:35 -0700 Subject: [PATCH 409/688] Chisel3: Avoid subword assignment --- uncore/src/main/scala/broadcast.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index cd69651c..74f7baee 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -165,7 +165,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTrac io.inner.release.ready := Bool(true) when(io.inner.release.valid) { data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid(io.irel().addr_beat) := Bool(true) + irel_data_valid := irel_data_valid.bitSet(io.irel().addr_beat, Bool(true)) } when(irel_data_done) { collect_irel_data := Bool(false) } } @@ -290,7 +290,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { data_buffer(io.iacq().addr_beat) := io.iacq().data - iacq_data_valid(io.iacq().addr_beat) := Bool(true) + iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true)) } when(iacq_data_done) { collect_iacq_data := Bool(false) } } From 9c7a41e8d33d3c7dfbf7cce2254f647394546d8d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 1 Aug 2015 21:09:00 -0700 Subject: [PATCH 410/688] Chisel3: bulk connect is not commutative We haven't decided if this is a FIRRTL limitation that we should relax, or a backwards incompatibility we're forced to live with. Should make for lively debate. --- uncore/src/main/scala/tilelink.scala | 10 +++++----- uncore/src/main/scala/util.scala | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 71c8d909..ccdcba09 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1009,7 +1009,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) req.ready := arb.ready }} - arb.io.out <> mngr + mngr <> arb.io.out } def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( @@ -1023,7 +1023,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) req.ready := arb.ready }} - arb.io.out <> mngr + mngr <> arb.io.out } def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( @@ -1079,7 +1079,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { val arb = Module(new RRArbiter(mngr.bits, arbN)) arb.io.in <> clts - arb.io.out <> mngr + mngr <> arb.io.out } } @@ -1401,11 +1401,11 @@ class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { val a = Module(new MemIOTileLinkIOConverter(1)) val b = Module(new MemPipeIOMemIOConverter(outstanding)) - a.io.tl <> io.tl + io.tl <> a.io.tl b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) a.io.mem.resp <> b.io.cpu.resp - b.io.mem <> io.mem + io.mem <> b.io.mem } //Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 58305e52..4c0b27c0 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -53,7 +53,7 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module require(io.in.bits.data.getWidth % narrowWidth == 0) if(n == 1) { - io.in <> io.out + io.out <> io.in io.cnt := UInt(0) io.done := Bool(true) } else { From a21979a2fa13b1e91f84222286380e10a3bf877c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 3 Aug 2015 18:01:06 -0700 Subject: [PATCH 411/688] Bits -> UInt --- uncore/src/main/scala/ecc.scala | 26 +++++++++++++------------- uncore/src/main/scala/htif.scala | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala index 609e5fca..eb11c7b8 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/ecc.scala @@ -6,8 +6,8 @@ import Chisel._ abstract class Decoding { - def uncorrected: Bits - def corrected: Bits + def uncorrected: UInt + def corrected: UInt def correctable: Bool def uncorrectable: Bool def error = correctable || uncorrectable @@ -16,15 +16,15 @@ abstract class Decoding abstract class Code { def width(w0: Int): Int - def encode(x: Bits): Bits - def decode(x: Bits): Decoding + def encode(x: UInt): UInt + def decode(x: UInt): Decoding } class IdentityCode extends Code { def width(w0: Int) = w0 - def encode(x: Bits) = x - def decode(y: Bits) = new Decoding { + def encode(x: UInt) = x + def decode(y: UInt) = new Decoding { def uncorrected = y def corrected = y def correctable = Bool(false) @@ -35,8 +35,8 @@ class IdentityCode extends Code class ParityCode extends Code { def width(w0: Int) = w0+1 - def encode(x: Bits) = Cat(x.xorR, x) - def decode(y: Bits) = new Decoding { + def encode(x: UInt) = Cat(x.xorR, x) + def decode(y: UInt) = new Decoding { def uncorrected = y(y.getWidth-2,0) def corrected = uncorrected def correctable = Bool(false) @@ -50,7 +50,7 @@ class SECCode extends Code val m = new Unsigned(k).log2 + 1 k + m + (if((1 << m) < m+k+1) 1 else 0) } - def encode(x: Bits) = { + def encode(x: UInt) = { val k = x.getWidth require(k > 0) val n = width(k) @@ -65,7 +65,7 @@ class SECCode extends Code } Vec(y).toBits } - def decode(y: Bits) = new Decoding { + def decode(y: UInt) = new Decoding { val n = y.getWidth require(n > 0 && !isPow2(n)) @@ -77,7 +77,7 @@ class SECCode extends Code } val s = Vec(syndrome).toBits - private def swizzle(z: Bits) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits + private def swizzle(z: UInt) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits def uncorrected = swizzle(y) def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) def correctable = s.orR @@ -92,8 +92,8 @@ class SECDEDCode extends Code private val par = new ParityCode def width(k: Int) = sec.width(k)+1 - def encode(x: Bits) = par.encode(sec.encode(x)) - def decode(x: Bits) = new Decoding { + def encode(x: UInt) = par.encode(sec.encode(x)) + def decode(x: UInt) = new Decoding { val secdec = sec.decode(x(x.getWidth-2,0)) val pardec = par.decode(x) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index daedf32d..cb49d8d3 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -256,7 +256,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata io.scr.waddr := scr_addr.toUInt - when (state === state_pcr_req && pcr_coreid === SInt(-1)) { + when (state === state_pcr_req && pcr_coreid.andR) { io.scr.wen := cmd === cmd_writecr pcrReadData := scr_rdata(scr_addr) state := state_tx From 121e4fb511b986b3709749a7a3094619710bedaa Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 3 Aug 2015 18:01:14 -0700 Subject: [PATCH 412/688] Flip direction of some bulk connects --- uncore/src/main/scala/tilelink.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ccdcba09..3c6df452 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -787,7 +787,7 @@ class TileLinkIOWrapper extends TLModule { val out = new TileLinkIO } io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant + io.in.grant <> io.out.grant io.out.finish <> io.in.finish io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) @@ -799,7 +799,7 @@ class ClientTileLinkIOWrapper extends TLModule { val out = new ClientTileLinkIO } io.out.acquire <> io.in.acquire - io.out.grant <> io.in.grant + io.in.grant <> io.out.grant io.out.probe.ready := Bool(true) io.out.release.valid := Bool(false) } From 77cf26aebabb3026b4b102a44624220858fdbbd9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 3 Aug 2015 18:53:31 -0700 Subject: [PATCH 413/688] Chisel3: Flip order of := and <> --- uncore/src/main/scala/broadcast.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 74f7baee..620c4bd2 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -91,9 +91,9 @@ class L2BroadcastHub extends ManagerCoherenceAgent // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) io.inner.grant.bits.data := io.outer.grant.bits.data io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) @@ -108,12 +108,12 @@ class L2BroadcastHub extends ManagerCoherenceAgent val free_sdq = io.outer.acquire.fire() && io.outer.acquire.bits.hasData() && outer_data_ptr.loc === inStoreQueue + io.outer <> outer_arb.io.out io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) io.outer.acquire.bits.union := Cat(Fill(io.outer.acquire.bits.tlWriteMaskBits, outer_arb.io.out.acquire.bits.union(1)), outer_arb.io.out.acquire.bits.union(0)) - io.outer <> outer_arb.io.out // Update SDQ valid bits when (io.outer.acquire.valid || sdq_enq) { From fb718f03c14471f522a5f9e1f8f1742b8a2b98f1 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 3 Aug 2015 19:50:58 -0700 Subject: [PATCH 414/688] bump scala to 2.11.6 --- uncore/build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index 6e53ae86..a889e7a6 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -4,7 +4,7 @@ version := "2.0" name := "uncore" -scalaVersion := "2.10.2" +scalaVersion := "2.11.6" // Provide a managed dependency on X if -DXVersion="" is supplied on the command line. libraryDependencies ++= (Seq("chisel","junctions").map { From 798ddeb5f56d1bc66f9f3c13ce35ee29001c2314 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 4 Aug 2015 13:15:17 -0700 Subject: [PATCH 415/688] Chisel3 compatibility: use >>Int instead of >>UInt The latter doesn't contract widths anymore. --- uncore/src/main/scala/cache.scala | 2 +- uncore/src/main/scala/ecc.scala | 2 +- uncore/src/main/scala/htif.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 12 ++++++------ 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6e5872c2..64966c20 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -87,7 +87,7 @@ class AMOALU extends CacheModule { val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: io.typ === MT_B || io.typ === MT_BU - val mask = ~UInt(0,64) ^ (io.addr(2) << UInt(31)) + val mask = ~UInt(0,64) ^ (io.addr(2) << 31) val adder_out = (io.lhs & mask).toUInt + (rhs & mask) val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala index eb11c7b8..3f16901c 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/ecc.scala @@ -79,7 +79,7 @@ class SECCode extends Code private def swizzle(z: UInt) = Vec((1 to n).filter(i => !isPow2(i)).map(i => z(i-1))).toBits def uncorrected = swizzle(y) - def corrected = swizzle(((y.toUInt << UInt(1)) ^ UIntToOH(s)) >> UInt(1)) + def corrected = swizzle(((y.toUInt << 1) ^ UIntToOH(s)) >> 1) def correctable = s.orR def uncorrectable = Bool(false) } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index cb49d8d3..8aeba790 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -96,7 +96,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - val rx_word_count = (rx_count >> UInt(log2Up(short_request_bits/w))) + val rx_word_count = (rx_count >> log2Up(short_request_bits/w)) val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR val packet_ram_depth = long_request_bits/short_request_bits-1 val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) @@ -173,7 +173,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { packet_ram(Cat(cnt, ui)) }.reverse.reduce(_##_) - val init_addr = addr.toUInt >> UInt(offsetBits-3) + val init_addr = addr.toUInt >> (offsetBits-3) io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq io.mem.acquire.bits := Mux(cmd === cmd_writemem, PutBlock( diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3c6df452..d281744a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1112,7 +1112,7 @@ trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) def managerSourcedClientXactId(in: ManagerSourcedWithId) = - in.client_xact_id >> UInt(log2Up(arbN)) + in.client_xact_id >> log2Up(arbN) def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } @@ -1378,7 +1378,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = io.nasti.r.bits.id(0), g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.nasti.r.bits.id >> UInt(1), + client_xact_id = io.nasti.r.bits.id >> 1, manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = io.nasti.r.bits.data) @@ -1389,7 +1389,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { dst = (if(dstIdBits > 0) io.nasti.b.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = Bool(true), g_type = Mux(io.nasti.b.bits.id(0), Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = io.nasti.b.bits.id >> UInt(1), + client_xact_id = io.nasti.b.bits.id >> 1, manager_xact_id = UInt(0)) } @@ -1450,7 +1450,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = Bool(true), g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> UInt(1), + client_xact_id = tag_out >> 1, manager_xact_id = UInt(0)) if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { @@ -1619,7 +1619,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = tag_in(0), g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> UInt(1), + client_xact_id = tag_in >> 1, manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = tl_buf_in(tl_cnt_in)) @@ -1651,7 +1651,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = io.mem.resp.bits.tag(0), g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> UInt(1), + client_xact_id = io.mem.resp.bits.tag >> 1, manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = io.mem.resp.bits.data) From eb6583d6078a830d8d99a9313209009066c102f7 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 5 Aug 2015 16:47:49 -0700 Subject: [PATCH 416/688] use cloneType in PhysicalNetworkIO --- uncore/src/main/scala/network.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 929f1e43..ea609842 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -13,7 +13,7 @@ class PhysicalHeader(n: Int) extends Bundle { class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { val header = new PhysicalHeader(n) - val payload = dType + val payload = dType.cloneType override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] } From a551a12d702fcbcfbc6484c5b851290e2c8a95ef Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 5 Aug 2015 17:05:31 -0700 Subject: [PATCH 417/688] add missing Wire wrap in BasicCrossbar --- uncore/src/main/scala/network.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index ea609842..ccb6c593 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -27,7 +27,7 @@ abstract class PhysicalNetwork extends Module class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) - val rdyVecs = List.fill(n){Vec.fill(n)(Bool())} + val rdyVecs = List.fill(n){Vec.fill(n)(Wire(Bool()))} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) From 01fc61ba9614cee0cc74bbfcd8166fbc29f0bf74 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 5 Aug 2015 18:43:40 -0700 Subject: [PATCH 418/688] Don't construct so many Vecs --- uncore/src/main/scala/coherence.scala | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index f24bae17..f731a6cc 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -44,9 +44,9 @@ trait HasClientSideCoherencePolicy { // Client coherence states and their permissions val nClientStates: Int def clientStateWidth = log2Ceil(nClientStates) - val clientStatesWithReadPermission: Vec[UInt] - val clientStatesWithWritePermission: Vec[UInt] - val clientStatesWithDirtyData: Vec[UInt] + def clientStatesWithReadPermission: Vec[UInt] + def clientStatesWithWritePermission: Vec[UInt] + def clientStatesWithDirtyData: Vec[UInt] // Transaction initiation logic def isValid(meta: ClientMetadata): Bool @@ -136,9 +136,9 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nClientStates = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - val clientStatesWithReadPermission = Vec(clientValid) - val clientStatesWithWritePermission = Vec(clientValid) - val clientStatesWithDirtyData = Vec(clientValid) + def clientStatesWithReadPermission = Vec(clientValid) + def clientStatesWithWritePermission = Vec(clientValid) + def clientStatesWithDirtyData = Vec(clientValid) def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid @@ -227,9 +227,9 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nClientStates = 3 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithDirtyData = Vec(clientExclusiveDirty) def isValid (meta: ClientMetadata) = meta.state != clientInvalid @@ -329,9 +329,9 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nClientStates = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) + def clientStatesWithWritePermission = Vec(clientExclusiveDirty) + def clientStatesWithDirtyData = Vec(clientExclusiveDirty) def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid @@ -447,9 +447,9 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val nClientStates = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithDirtyData = Vec(clientExclusiveDirty) def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid @@ -562,9 +562,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d val nClientStates = 7 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - val clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - val clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + def clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid From 005752e2a6cff4cb08f7db02fb26dbf7a5812c92 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 10 Aug 2015 14:35:08 -0700 Subject: [PATCH 419/688] use the parameters used to create the original object --- uncore/src/main/scala/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 9b53832b..fccf720f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -12,12 +12,12 @@ trait CoherenceAgentParameters extends UsesParameters { val nReleaseTransactors = 1 val nAcquireTransactors = params(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - def outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) val outerBeatAddrBits = log2Up(outerDataBeats) val outerByteAddrBits = log2Up(outerDataBits/8) - def innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) val innerBeatAddrBits = log2Up(innerDataBeats) From 05d311c517105a75d0ff113353d2971cb5716439 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 27 Aug 2015 09:47:02 -0700 Subject: [PATCH 420/688] Use Vec.apply, not Vec.fill, for type nodes --- uncore/src/main/scala/broadcast.scala | 8 ++++---- uncore/src/main/scala/cache.scala | 16 ++++++++-------- uncore/src/main/scala/htif.scala | 4 ++-- uncore/src/main/scala/network.scala | 6 +++--- uncore/src/main/scala/tilelink.scala | 12 ++++++------ uncore/src/main/scala/uncore.scala | 2 +- uncore/src/main/scala/util.scala | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 620c4bd2..9f6aba6e 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -40,7 +40,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent trackerList.map(_.io.incoherent := io.incoherent) // Queue to store impending Put data - val sdq = Reg(Vec.fill(sdqDepth){io.iacq().data}) + val sdq = Reg(Vec(io.iacq().data, sdqDepth)) val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR @@ -69,7 +69,7 @@ class L2BroadcastHub extends ManagerCoherenceAgent val voluntary = io.irel().isVoluntary() val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Reg(Vec.fill(innerDataBeats){io.irel().data}) //TODO Assumes nReleaseTransactors == 1 + val vwbdq = Reg(Vec(io.irel().data, innerDataBeats)) //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data @@ -131,7 +131,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTrac val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(Vec.fill(innerDataBeats){io.irel().data}) + val data_buffer = Reg(Vec(io.irel().data, innerDataBeats)) val coh = ManagerMetadata.onReset val collect_irel_data = Reg(init=Bool(false)) @@ -210,7 +210,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(Vec.fill(innerDataBeats){io.iacq().data}) + val data_buffer = Reg(Vec(io.iacq().data, innerDataBeats)) val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 64966c20..30168286 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -145,7 +145,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal)).flip - val resp = Vec.fill(nWays){rstVal.cloneType.asOutput} + val resp = Vec(rstVal.cloneType.asOutput, nWays) } val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) @@ -427,9 +427,9 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { class CacheBlockBuffer { // TODO val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) - def internal = Vec.fill(internalDataBeats){ Bits(width = rowBits) }.fromBits(buffer) - def inner = Vec.fill(innerDataBeats){ Bits(width = innerDataBits) }.fromBits(buffer) - def outer = Vec.fill(outerDataBeats){ Bits(width = outerDataBits) }.fromBits(buffer) + def internal = Vec(Bits(width = rowBits), internalDataBeats).fromBits(buffer) + def inner = Vec(Bits(width = innerDataBits), innerDataBeats).fromBits(buffer) + def outer = Vec(Bits(width = outerDataBits), outerDataBeats).fromBits(buffer) } def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { @@ -492,7 +492,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) + val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -587,8 +587,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding transaction metadata val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) - val wmask_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits/8)}) + val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) + val wmask_buffer = Reg(init=Vec(UInt(0, width = innerDataBits/8), innerDataBeats)) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -980,7 +980,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(new L2WritebackReq) - val data_buffer = Reg(init=Vec.fill(innerDataBeats){UInt(0, width = innerDataBits)}) + val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) val xact_addr_block = Cat(xact.tag, xact.idx) val pending_irels = diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8aeba790..d162be98 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -51,7 +51,7 @@ class HTIFIO extends HTIFBundle { } class SCRIO extends HTIFBundle { - val rdata = Vec.fill(nSCR){Bits(INPUT, 64)} + val rdata = Vec(Bits(INPUT, 64), nSCR) val wen = Bool(OUTPUT) val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, 64) @@ -59,7 +59,7 @@ class SCRIO extends HTIFBundle { class HTIFModuleIO extends HTIFBundle { val host = new HostIO - val cpu = Vec.fill(nCores){new HTIFIO}.flip + val cpu = Vec(new HTIFIO, nCores).flip val mem = new ClientUncachedTileLinkIO val scr = new SCRIO } diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index ccb6c593..16d2c1e7 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -18,8 +18,8 @@ class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { } class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { - val in = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))}.flip - val out = Vec.fill(n){Decoupled(new PhysicalNetworkIO(n,dType))} + val in = Vec(Decoupled(new PhysicalNetworkIO(n,dType)), n).flip + val out = Vec(Decoupled(new PhysicalNetworkIO(n,dType)), n) } abstract class PhysicalNetwork extends Module @@ -27,7 +27,7 @@ abstract class PhysicalNetwork extends Module class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) - val rdyVecs = List.fill(n){Vec.fill(n)(Wire(Bool()))} + val rdyVecs = Seq.fill(n){Seq.fill(n)(Wire(Bool()))} io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d281744a..6c4cf264 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1086,7 +1086,7 @@ trait TileLinkArbiterLike extends TileLinkParameters { /** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(arbN){new UncachedTileLinkIO}.flip + val in = Vec(new UncachedTileLinkIO, arbN).flip val out = new UncachedTileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -1097,7 +1097,7 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with Tile /** Abstract base case for any Arbiters that have cached TileLinkIOs */ abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec.fill(arbN){new TileLinkIO}.flip + val in = Vec(new TileLinkIO, arbN).flip val out = new TileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -1141,7 +1141,7 @@ class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) wi /** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { - val in = Vec.fill(arbN){new ClientUncachedTileLinkIO}.flip + val in = Vec(new ClientUncachedTileLinkIO, arbN).flip val out = new ClientUncachedTileLinkIO } hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) @@ -1151,7 +1151,7 @@ class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLin /** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { - val in = Vec.fill(arbN){new ClientTileLinkIO}.flip + val in = Vec(new ClientTileLinkIO, arbN).flip val out = new ClientTileLinkIO } hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) @@ -1461,7 +1461,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) val mif_done_out = Reg(init=Bool(false)) val tl_buf_out = Reg(Vec(io.tl.acquire.bits.data, tlDataBeats)) - val mif_buf_out = Vec.fill(mifDataBeats){ new MemData } + val mif_buf_out = Vec(new MemData, mifDataBeats) mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) val tl_prog_out = tl_cnt_out*UInt(tlDataBits) @@ -1611,7 +1611,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data val mif_done_in = Reg(init=Bool(false)) val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) - val tl_buf_in = Vec.fill(tlDataBeats){ io.tl.acquire.bits.data } + val tl_buf_in = Vec(io.tl.acquire.bits.data, tlDataBeats) tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) val mif_prog_in = mif_cnt_in*UInt(mifDataBits) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index fccf720f..a9cc66d6 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -50,7 +50,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends CoherenceAgentBundle { val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) - val incoherent = Vec.fill(inner.tlNCachingClients){Bool()}.asInput + val incoherent = Vec(Bool(), inner.tlNCachingClients).asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits def irel(dummy: Int = 0) = inner.release.bits diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 4c0b27c0..8fb21f2e 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -62,7 +62,7 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module val rbits = Reg{io.in.bits} val active = Reg(init=Bool(false)) - val shifter = Vec.fill(n){Bits(width = narrowWidth)} + val shifter = Vec(Bits(width = narrowWidth), n) (0 until n).foreach { i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } From 94287fed90f0dc6583c5dbccf2a529f2e3f64dac Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 27 Aug 2015 09:57:36 -0700 Subject: [PATCH 421/688] Avoid type-unsafe assignments --- uncore/src/main/scala/cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 30168286..11eb0682 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -364,7 +364,7 @@ class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags - (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent.toBits } + (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent } // Handle acquire transaction initiation val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) @@ -525,7 +525,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := SInt(-1) + io.data.write.bits.wmask := ~UInt(0, io.data.write.bits.wmask.getWidth) io.data.write.bits.data := data_buffer(curr_write_beat) // Send an acknowledgement @@ -663,7 +663,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) - wmask_buffer(beat) := SInt(-1) + wmask_buffer(beat) := ~UInt(0, wmask_buffer.head.getWidth) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { @@ -1057,7 +1057,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val coh = io.wb.req.bits.coh val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } - pending_reads := SInt(-1, width = innerDataBeats) + pending_reads := ~UInt(0, width = innerDataBeats) pending_resps := UInt(0) pending_orel_data := UInt(0) state := Mux(needs_inner_probes, s_inner_probe, s_busy) From 350d530766a833adf09878bdb3c58fe8d5095290 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 27 Aug 2015 10:00:43 -0700 Subject: [PATCH 422/688] Use Vec.fill, not Vec.apply, for Vec literals --- uncore/src/main/scala/cache.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 11eb0682..4d59482d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -492,7 +492,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -587,8 +587,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding transaction metadata val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) - val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) - val wmask_buffer = Reg(init=Vec(UInt(0, width = innerDataBits/8), innerDataBeats)) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -980,7 +980,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(new L2WritebackReq) - val data_buffer = Reg(init=Vec(UInt(0, width = innerDataBits), innerDataBeats)) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val xact_addr_block = Cat(xact.tag, xact.idx) val pending_irels = From 24389a52578a963db063fea192bcbd309e4a3bc7 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 11 Sep 2015 15:41:39 -0700 Subject: [PATCH 423/688] Chisel3 compatibility fixes --- uncore/src/main/scala/cache.scala | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 4d59482d..ff0d7739 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -145,22 +145,22 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal)).flip - val resp = Vec(rstVal.cloneType.asOutput, nWays) + val resp = Vec(rstVal.cloneType, nWays).asOutput } val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en.toSInt).toUInt + val wmask = Mux(rst, SInt(-1), io.write.bits.way_en.toSInt).toBools when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth - val tag_arr = SeqMem(UInt(width = metabits*nWays), nSets) + val tag_arr = SeqMem(Vec(UInt(width = metabits), nWays), nSets) when (rst || io.write.valid) { - tag_arr.write(waddr, Fill(nWays, wdata), FillInterleaved(metabits, wmask)) + tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) } - val tags = tag_arr.read(io.read.bits.idx, io.read.valid) + val tags = tag_arr.read(io.read.bits.idx, io.read.valid).toBits io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst @@ -318,16 +318,17 @@ class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataW class L2DataArray(delay: Int) extends L2HellaCacheModule { val io = new L2DataRWIO().flip - val array = SeqMem(Bits(width=rowBits), nWays*nSets*refillCycles) + val array = SeqMem(Vec(Bits(width=8), rowBits/8), nWays*nSets*refillCycles) val ren = !io.write.valid && io.read.valid val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) - val wmask = FillInterleaved(8, io.write.bits.wmask) - when (io.write.valid) { array.write(waddr, io.write.bits.data, wmask) } + val wdata = Vec.tabulate(rowBits/8)(i => io.write.bits.data(8*(i+1)-1,8*i)) + val wmask = io.write.bits.wmask.toBools + when (io.write.valid) { array.write(waddr, wdata, wmask) } val r_req = Pipe(io.read.fire(), io.read.bits) io.resp := Pipe(r_req.valid, r_req.bits, delay) - io.resp.bits.data := Pipe(r_req.valid, array.read(raddr, ren), delay).bits + io.resp.bits.data := Pipe(r_req.valid, array.read(raddr, ren).toBits, delay).bits io.read.ready := !io.write.valid io.write.ready := Bool(true) } @@ -339,11 +340,11 @@ class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParam val meta = Module(new L2MetadataArray) // TODO: add delay knob val data = Module(new L2DataArray(1)) val tshrfile = Module(new TSHRFile) - tshrfile.io.inner <> io.inner + io.inner <> tshrfile.io.inner io.outer <> tshrfile.io.outer - io.incoherent <> tshrfile.io.incoherent - tshrfile.io.meta <> meta.io - tshrfile.io.data <> data.io + tshrfile.io.incoherent <> io.incoherent + meta.io <> tshrfile.io.meta + data.io <> tshrfile.io.data } class TSHRFileIO extends HierarchicalTLIO { From 24f3fac90acf1a09ca9c7eeb566623130df94903 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 10 Aug 2015 19:06:02 -0700 Subject: [PATCH 424/688] fix broadcast hub and TL -> NASTI converter to support subblock operations --- uncore/src/main/scala/broadcast.scala | 74 ++++++++++++++++----------- uncore/src/main/scala/tilelink.scala | 71 +++++++++++++++++++------ uncore/src/main/scala/uncore.scala | 1 + 3 files changed, 102 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 9f6aba6e..a396de24 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -30,12 +30,18 @@ class L2BroadcastHub extends ManagerCoherenceAgent val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + val trackerTLParams = params.alterPartial({ + case TLDataBits => internalDataBits + case TLWriteMaskBits => innerWriteMaskBits + }) + // Create SHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id), {case TLDataBits => internalDataBits})) ++ - (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id), {case TLDataBits => internalDataBits})) - + val trackerList = + (0 until nReleaseTransactors).map(id => + Module(new BroadcastVoluntaryReleaseTracker(id))(trackerTLParams)) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new BroadcastAcquireTracker(id))(trackerTLParams)) + // Propagate incoherence flags trackerList.map(_.io.incoherent := io.incoherent) @@ -100,7 +106,8 @@ class L2BroadcastHub extends ManagerCoherenceAgent // Create an arbiter for the one memory port val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), { case TLId => params(OuterTLId) - case TLDataBits => internalDataBits }) + case TLDataBits => internalDataBits + case TLWriteMaskBits => innerWriteMaskBits }) outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) @@ -112,8 +119,6 @@ class L2BroadcastHub extends ManagerCoherenceAgent io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( inStoreQueue -> sdq(outer_data_ptr.idx), inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - io.outer.acquire.bits.union := Cat(Fill(io.outer.acquire.bits.tlWriteMaskBits, outer_arb.io.out.acquire.bits.union(1)), - outer_arb.io.out.acquire.bits.union(0)) // Update SDQ valid bits when (io.outer.acquire.valid || sdq_enq) { @@ -209,14 +214,17 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new AcquireFromSrc, { + case TLId => params(InnerTLId) + case TLDataBits => 0 + case TLWriteMaskBits => innerWriteMaskBits + })) val data_buffer = Reg(Vec(io.iacq().data, innerDataBeats)) val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && - Vec(Acquire.getType, Acquire.putType, Acquire.putAtomicType, - Acquire.prefetchType).contains(xact.a_type)), - "Broadcast Hub does not support PutAtomics, subblock Gets/Puts, or prefetches") // TODO + Vec(Acquire.putAtomicType, Acquire.prefetchType).contains(xact.a_type)), + "Broadcast Hub does not support PutAtomics or prefetches") // TODO val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) @@ -236,6 +244,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val pending_outer_write_ = io.iacq().hasData() val pending_outer_read = io.ignt().hasData() val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() + val subblock_type = xact.isSubBlockType() io.has_acquire_conflict := xact.conflicts(io.iacq()) && (state != s_idle) && @@ -246,22 +255,32 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { !io.irel().isVoluntary() && (state === s_probe) - val outer_write_acq = Bundle(PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)))(outerTLParams) - val outer_write_rel = Bundle(PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = io.irel().addr_beat, - data = io.irel().data))(outerTLParams) - val outer_read = Bundle(GetBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block))(outerTLParams) + val oacq_type = MuxLookup(state, Acquire.getBlockType, Seq( + (s_probe, Acquire.putBlockType), + (s_mem_write, Mux(subblock_type, Acquire.putType, Acquire.putBlockType)), + (s_mem_read, Mux(subblock_type, Acquire.getType, Acquire.getBlockType)))) + val oacq_beat = MuxLookup(state, UInt(0), Seq( + (s_probe, io.irel().addr_beat), + (s_mem_write, Mux(subblock_type, xact.addr_beat, oacq_data_cnt)), + (s_mem_read, Mux(subblock_type, xact.addr_beat, UInt(0))))) + val oacq_data = MuxLookup(state, Bits(0), Seq( + (s_probe, io.irel().data), + (s_mem_write, Mux(subblock_type, + data_buffer(0), data_buffer(oacq_data_cnt))))) + val oacq_union = MuxLookup(state, Bits(0), Seq( + (s_probe, Acquire.fullWriteMask), + (s_mem_write, xact.wmask()), + (s_mem_read, Cat(xact.addr_byte(), xact.op_size(), M_XRD)))) io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits := outer_read //default + io.outer.acquire.bits := Bundle(Acquire( + is_builtin_type = Bool(true), + a_type = oacq_type, + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_beat, + data = oacq_data, + union = Cat(oacq_union, Bool(true))))(outerTLParams) io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) @@ -331,7 +350,6 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { when(io.inner.release.valid) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) - io.outer.acquire.bits := outer_write_rel when(io.outer.acquire.ready) { when(oacq_data_done) { pending_ognt_ack := Bool(true) @@ -353,7 +371,6 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { } is(s_mem_write) { // Write data to outer memory io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) - io.outer.acquire.bits := outer_write_acq when(oacq_data_done) { pending_ognt_ack := Bool(true) state := Mux(pending_outer_read, s_mem_read, s_mem_resp) @@ -361,7 +378,6 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { } is(s_mem_read) { // Read data from outer memory (possibly what was just written) io.outer.acquire.valid := !pending_ognt_ack - io.outer.acquire.bits := outer_read when(io.outer.acquire.fire()) { state := s_mem_resp } } is(s_mem_resp) { // Wait to forward grants from outer memory diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6c4cf264..b78dcafd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -34,6 +34,8 @@ case object TLDataBits extends Field[Int] case object TLDataBeats extends Field[Int] /** Whether the underlying physical network preserved point-to-point ordering of messages */ case object TLNetworkIsOrderedP2P extends Field[Boolean] +/** Number of bits in write mask (usually one per byte in beat) */ +case object TLWriteMaskBits extends Field[Int] /** Utility trait for building Modules and Bundles that use TileLink parameters */ trait TileLinkParameters extends UsesParameters { @@ -53,7 +55,7 @@ trait TileLinkParameters extends UsesParameters { val tlDataBits = params(TLDataBits) val tlDataBytes = tlDataBits/8 val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = if(tlDataBits/8 < 1) 1 else tlDataBits/8 + val tlWriteMaskBits = params(TLWriteMaskBits) val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) val tlMemoryOpcodeBits = M_SZ @@ -1274,11 +1276,38 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val addr_out = Reg(UInt(width = nastiXAddrBits)) val has_data = Reg(init=Bool(false)) val data_from_rel = Reg(init=Bool(false)) + val is_subblock = io.tl.acquire.bits.isSubBlockType() val (tl_cnt_out, tl_wrap_out) = - Counter((io.tl.acquire.fire() && acq_has_data) || + Counter((io.tl.acquire.fire() && io.tl.acquire.bits.hasMultibeatData()) || (io.tl.release.fire() && rel_has_data), tlDataBeats) val tl_done_out = Reg(init=Bool(false)) + val roq_size = 4 + val roq_data = Reg(Vec(UInt(width = tlByteAddrBits), roq_size)) + val roq_tags = Reg(Vec(UInt(width = nastiRIdBits), roq_size)) + val roq_free = Reg(init = Fill(roq_size, Bits(1, 1))) + val roq_full = !roq_free.orR + + val roq_enq_addr = PriorityEncoder(roq_free) + val roq_enq_valid = io.tl.acquire.fire() && !acq_has_data && is_subblock + val roq_enq_data = io.tl.acquire.bits.addr_byte() + val roq_enq_tag = io.nasti.ar.bits.id + + val roq_deq_tag = io.nasti.r.bits.id + val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === roq_deq_tag)) + val roq_deq_data = roq_data(roq_deq_addr) + val roq_deq_valid = io.nasti.r.fire() && !io.nasti.r.bits.id(0) + + when (roq_enq_valid) { + roq_data(roq_enq_addr) := roq_enq_data + roq_tags(roq_enq_addr) := roq_enq_tag + roq_free(roq_enq_addr) := Bool(false) + } + + when (roq_deq_valid) { + roq_free(roq_deq_addr) := Bool(true) + } + io.nasti.ar.bits.id := tag_out io.nasti.ar.bits.addr := addr_out io.nasti.ar.bits.len := Mux(has_data, UInt(tlDataBeats-1), UInt(0)) @@ -1293,7 +1322,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.aw.bits := io.nasti.ar.bits io.nasti.w.bits.strb := Mux(data_from_rel, SInt(-1), io.tl.acquire.bits.wmask()) io.nasti.w.bits.data := Mux(data_from_rel, io.tl.release.bits.data, io.tl.acquire.bits.data) - io.nasti.w.bits.last := tl_wrap_out + io.nasti.w.bits.last := tl_wrap_out || (io.tl.acquire.fire() && is_subblock) when(!active_out){ io.tl.release.ready := io.nasti.w.ready @@ -1307,7 +1336,6 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.aw.valid := is_write io.nasti.ar.valid := !is_write cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) - tl_done_out := tl_wrap_out when(io.tl.release.valid) { data_from_rel := Bool(true) io.nasti.w.bits.data := io.tl.release.bits.data @@ -1319,34 +1347,35 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.aw.bits.id := tag io.nasti.aw.bits.addr := addr io.nasti.aw.bits.len := UInt(tlDataBeats-1) - io.nasti.aw.bits.size := MT_Q tag_out := tag addr_out := addr has_data := rel_has_data + tl_done_out := tl_wrap_out } .elsewhen(io.tl.acquire.valid) { data_from_rel := Bool(false) io.nasti.w.bits.data := io.tl.acquire.bits.data io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() + // The last bit indicates to the Grant logic what g_type to send back + // For read, true = getDataBlockType, false = getDataBeatType + // For write, it should always be false, so that putAckType is sent val tag = Cat(io.tl.acquire.bits.client_id, io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) + !is_write && !is_subblock) val addr = io.tl.acquire.bits.full_addr() when(is_write) { io.nasti.aw.bits.id := tag io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.putBlockType), - UInt(tlDataBeats-1), UInt(0)) - io.nasti.aw.bits.size := bytesToXSize(PopCount(io.tl.acquire.bits.wmask())) + io.nasti.aw.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) } .otherwise { io.nasti.ar.bits.id := tag io.nasti.ar.bits.addr := addr - io.nasti.ar.bits.len := Mux(io.tl.acquire.bits.isBuiltInType(Acquire.getBlockType), - UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() } tag_out := tag addr_out := addr has_data := acq_has_data + tl_done_out := tl_wrap_out || is_subblock } } } @@ -1364,24 +1393,36 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { } } when(tl_wrap_out) { tl_done_out := Bool(true) } - when(cmd_sent_out && (!has_data || tl_done_out)) { active_out := Bool(false) } + when(cmd_sent_out && !roq_full && (!has_data || tl_done_out)) { + active_out := Bool(false) + } } + assert (!io.nasti.r.valid || !io.nasti.r.bits.resp(1), + "NASTI read response error") + assert (!io.nasti.b.valid || !io.nasti.b.bits.resp(1), + "NASTI write response error") + // Aggregate incoming NASTI responses into TL Grants val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out + val r_aligned_data = Mux(io.nasti.r.bits.id(0), + io.nasti.r.bits.data, + io.nasti.r.bits.data << Cat(roq_deq_data, UInt(0, 3))) + gnt_arb.io.in(0).valid := io.nasti.r.valid io.nasti.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = io.nasti.r.bits.id(0), - g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol + is_builtin_type = Bool(true), + g_type = Mux(io.nasti.r.bits.id(0), + Grant.getDataBlockType, Grant.getDataBeatType), // TODO: Assumes MI or MEI protocol client_xact_id = io.nasti.r.bits.id >> 1, manager_xact_id = UInt(0), addr_beat = tl_cnt_in, - data = io.nasti.r.bits.data) + data = r_aligned_data) gnt_arb.io.in(1).valid := io.nasti.b.valid io.nasti.b.ready := gnt_arb.io.in(1).ready diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index a9cc66d6..969edcfd 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -20,6 +20,7 @@ trait CoherenceAgentParameters extends UsesParameters { val innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) + val innerWriteMaskBits = innerTLParams(TLWriteMaskBits) val innerBeatAddrBits = log2Up(innerDataBeats) val innerByteAddrBits = log2Up(innerDataBits/8) require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement From bdc6972a8d6be758522d4ab90b4a8e0cbfa2367b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 12 Aug 2015 21:23:17 -0700 Subject: [PATCH 425/688] separate RTC updates from HTIF --- uncore/src/main/scala/htif.scala | 54 ++++------------ uncore/src/main/scala/rtc.scala | 104 +++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 41 deletions(-) create mode 100644 uncore/src/main/scala/rtc.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index d162be98..360eaebb 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,7 +4,7 @@ package uncore import Chisel._ import Chisel.ImplicitConversions._ -import uncore._ +import junctions.{SMIIO, MMIOBase} case object HTIFWidth extends Field[Int] case object HTIFNSCR extends Field[Int] @@ -31,18 +31,10 @@ class HostIO extends HTIFBundle val debug_stats_pcr = Bool(OUTPUT) } -class PCRReq extends Bundle -{ - val rw = Bool() - val addr = Bits(width = 12) - val data = Bits(width = 64) -} - class HTIFIO extends HTIFBundle { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val pcr_req = Decoupled(new PCRReq).flip - val pcr_rep = Decoupled(Bits(width = 64)) + val pcr = new SMIIO(64, 12).flip val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) @@ -106,7 +98,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr_req.bits.addr.getWidth-1, 0) + val pcr_addr = addr(io.cpu(0).pcr.req.bits.addr.getWidth-1, 0) val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) val pcr_wdata = packet_ram(0) @@ -184,39 +176,19 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { GetBlock(addr_block = init_addr)) io.mem.grant.ready := Bool(true) - // real-time counter (which doesn't really belong here...) - val rtc = Reg(init=UInt(0,64)) - val rtc_tick = Counter(params(RTCPeriod)).inc() - when (rtc_tick) { rtc := rtc + UInt(1) } - - val pcrReadData = Reg(Bits(width = io.cpu(0).pcr_rep.bits.getWidth)) + val pcrReadData = Reg(Bits(width = io.cpu(0).pcr.resp.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) val my_ipi = Reg(init=Bool(false)) val cpu = io.cpu(i) val me = pcr_coreid === UInt(i) - cpu.pcr_req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) - cpu.pcr_req.bits.rw := cmd === cmd_writecr - cpu.pcr_req.bits.addr := pcr_addr - cpu.pcr_req.bits.data := pcr_wdata + cpu.pcr.req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) + cpu.pcr.req.bits.rw := cmd === cmd_writecr + cpu.pcr.req.bits.addr := pcr_addr + cpu.pcr.req.bits.data := pcr_wdata cpu.reset := my_reset - // use pcr port to update core's rtc value periodically - val rtc_sent = Reg(init=Bool(false)) - val rtc_outstanding = Reg(init=Bool(false)) - when (rtc_tick) { rtc_sent := Bool(false) } - when (cpu.pcr_rep.valid) { rtc_outstanding := Bool(false) } - when (rtc_outstanding) { cpu.pcr_req.valid := Bool(false) } - when (state != state_pcr_req && state != state_pcr_resp && !rtc_sent && !rtc_outstanding) { - cpu.pcr_req.valid := Bool(true) - cpu.pcr_req.bits.rw := Bool(true) - cpu.pcr_req.bits.addr := UInt(pcr_RESET) /* XXX this means write mtime */ - cpu.pcr_req.bits.data := rtc - rtc_sent := cpu.pcr_req.ready - rtc_outstanding := cpu.pcr_req.ready - } - when (cpu.ipi_rep.ready) { my_ipi := Bool(false) } @@ -228,7 +200,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - when (state === state_pcr_req && cpu.pcr_req.fire()) { + when (state === state_pcr_req && cpu.pcr.req.fire()) { state := state_pcr_resp } when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { @@ -239,9 +211,9 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { state := state_tx } - cpu.pcr_rep.ready := Bool(true) - when (state === state_pcr_resp && cpu.pcr_rep.valid) { - pcrReadData := cpu.pcr_rep.bits + cpu.pcr.resp.ready := Bool(true) + when (state === state_pcr_resp && cpu.pcr.resp.valid) { + pcrReadData := cpu.pcr.resp.bits state := state_tx } } @@ -251,7 +223,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt((BigInt(dataBits*dataBeats/8) << params(TLBlockAddrBits)) >> 20) + scr_rdata(1) := UInt(params(MMIOBase) >> 20) io.scr.wen := Bool(false) io.scr.wdata := pcr_wdata diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala new file mode 100644 index 00000000..9253d0ea --- /dev/null +++ b/uncore/src/main/scala/rtc.scala @@ -0,0 +1,104 @@ +package uncore + +import Chisel._ +import junctions.{NASTIMasterIO, NASTIAddrHashMap, SMIIO} + +class RTC(pcr_MTIME: Int) extends Module { + private val nCores = params(HTIFNCores) + + val io = new Bundle { + val smi = Vec.fill(nCores) { new SMIIO(64, 12) } + } + + val rtc = Reg(init=UInt(0,64)) + val rtc_tick = Counter(params(RTCPeriod)).inc() + + for ((smi, i) <- io.smi.zipWithIndex) { + val rtc_sending = Reg(init = Bool(false)) + val rtc_outstanding = Reg(init = Bool(false)) + + when (rtc_tick) { + rtc := rtc + UInt(1) + rtc_sending := Bool(true) + rtc_outstanding := Bool(true) + } + when (smi.req.fire()) { rtc_sending := Bool(false) } + when (smi.resp.fire()) { rtc_outstanding := Bool(false) } + + assert(!rtc_tick || !rtc_outstanding, "Last rtc tick not yet sent") + + smi.req.bits.addr := UInt(pcr_MTIME) + smi.req.bits.rw := Bool(true) + smi.req.bits.data := rtc + smi.req.valid := rtc_sending + smi.resp.ready := Bool(true) + } +} + +class RTCNASTI(pcr_MTIME: Int) extends Module { + val io = new NASTIMasterIO + + private val nCores = params(HTIFNCores) + private val addrMap = params(NASTIAddrHashMap) + + val addrTable = Vec.tabulate(nCores) { i => + UInt(addrMap(s"conf:csr$i").start + pcr_MTIME * 8) + } + + val rtc = Reg(init=UInt(0,64)) + val rtc_tick = Counter(params(RTCPeriod)).inc() + + val sending_addr = Reg(init = Bool(false)) + val sending_data = Reg(init = Bool(false)) + val send_acked = Reg(init = Vec(nCores, Bool(true))) + + when (rtc_tick) { + rtc := rtc + UInt(1) + send_acked := Vec(nCores, Bool(false)) + sending_addr := Bool(true) + sending_data := Bool(true) + } + + if (nCores > 1) { + val (core, addr_send_done) = Counter(io.aw.fire(), nCores) + val (_, data_send_done) = Counter(io.w.fire(), nCores) + + when (addr_send_done) { sending_addr := Bool(false) } + when (data_send_done) { sending_data := Bool(false) } + + io.aw.bits.id := core + io.aw.bits.addr := addrTable(core) + } else { + when (io.aw.fire()) { sending_addr := Bool(false) } + when (io.w.fire()) { sending_addr := Bool(false) } + + io.aw.bits.id := UInt(0) + io.aw.bits.addr := addrTable(0) + } + + when (io.b.fire()) { send_acked(io.b.bits.id) := Bool(true) } + + io.aw.valid := sending_addr + io.aw.bits.size := UInt(3) // 8 bytes + io.aw.bits.len := UInt(0) + io.aw.bits.burst := Bits("b01") + io.aw.bits.lock := Bool(false) + io.aw.bits.cache := UInt("b0000") + io.aw.bits.prot := UInt("b000") + io.aw.bits.qos := UInt("b0000") + io.aw.bits.region := UInt("b0000") + io.aw.bits.user := UInt(0) + + io.w.valid := sending_data + io.w.bits.data := rtc + io.w.bits.strb := Bits(0x00FF) + io.w.bits.user := UInt(0) + io.w.bits.last := Bool(true) + + io.b.ready := Bool(true) + io.ar.valid := Bool(false) + io.r.ready := Bool(false) + + assert(!rtc_tick || send_acked.toBits.andR, + s"Not all clocks were updated for rtc tick") +} From 21f96f382cd35d7aac6306b6f477ca08bb4168a8 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 1 Sep 2015 14:47:18 -0700 Subject: [PATCH 426/688] split off SCR functionality from HTIF --- uncore/src/main/scala/htif.scala | 37 ++++++++++------------------ uncore/src/main/scala/scr.scala | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 24 deletions(-) create mode 100644 uncore/src/main/scala/scr.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 360eaebb..c153d05b 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,7 +4,7 @@ package uncore import Chisel._ import Chisel.ImplicitConversions._ -import junctions.{SMIIO, MMIOBase} +import junctions.SMIIO case object HTIFWidth extends Field[Int] case object HTIFNSCR extends Field[Int] @@ -16,6 +16,7 @@ abstract trait HTIFParameters extends UsesParameters { val dataBeats = params(TLDataBeats) val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) + val scrAddrBits = log2Up(nSCR) val offsetBits = params(HTIFOffsetBits) val nCores = params(HTIFNCores) } @@ -42,18 +43,11 @@ class HTIFIO extends HTIFBundle { // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class SCRIO extends HTIFBundle { - val rdata = Vec(Bits(INPUT, 64), nSCR) - val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(nSCR)) - val wdata = Bits(OUTPUT, 64) -} - class HTIFModuleIO extends HTIFBundle { val host = new HostIO val cpu = Vec(new HTIFIO, nCores).flip val mem = new ClientUncachedTileLinkIO - val scr = new SCRIO + val scr = new SMIIO(64, scrAddrBits) } class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { @@ -200,9 +194,8 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - when (state === state_pcr_req && cpu.pcr.req.fire()) { - state := state_pcr_resp - } + when (cpu.pcr.req.fire()) { state := state_pcr_resp } + when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { when (cmd === cmd_writecr) { my_reset := pcr_wdata(0) @@ -218,19 +211,15 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - val scr_addr = addr(log2Up(nSCR)-1, 0) - val scr_rdata = Wire(Vec(Bits(width=64), io.scr.rdata.size)) - for (i <- 0 until scr_rdata.size) - scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt(params(MMIOBase) >> 20) + io.scr.req.valid := (state === state_pcr_req && pcr_coreid.andR) + io.scr.req.bits.addr := addr(scrAddrBits - 1, 0).toUInt + io.scr.req.bits.data := pcr_wdata + io.scr.req.bits.rw := (cmd === cmd_writecr) + io.scr.resp.ready := Bool(true) - io.scr.wen := Bool(false) - io.scr.wdata := pcr_wdata - io.scr.waddr := scr_addr.toUInt - when (state === state_pcr_req && pcr_coreid.andR) { - io.scr.wen := cmd === cmd_writecr - pcrReadData := scr_rdata(scr_addr) + when (io.scr.req.fire()) { state := state_pcr_resp } + when (state === state_pcr_resp && io.scr.resp.valid) { + pcrReadData := io.scr.resp.bits state := state_tx } diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala new file mode 100644 index 00000000..09158810 --- /dev/null +++ b/uncore/src/main/scala/scr.scala @@ -0,0 +1,41 @@ +package uncore + +import Chisel._ +import junctions.{SMIIO, MMIOBase} + +class SCRIO extends HTIFBundle { + val rdata = Vec(Bits(INPUT, 64), nSCR) + val wen = Bool(OUTPUT) + val waddr = UInt(OUTPUT, log2Up(nSCR)) + val wdata = Bits(OUTPUT, 64) +} + +class SCRFile extends Module with HTIFParameters { + val io = new Bundle { + val smi = new SMIIO(64, scrAddrBits).flip + val scr = new SCRIO + } + + val scr_rdata = Wire(Vec(Bits(width=64), io.scr.rdata.size)) + for (i <- 0 until scr_rdata.size) + scr_rdata(i) := io.scr.rdata(i) + scr_rdata(0) := UInt(nCores) + scr_rdata(1) := UInt(params(MMIOBase) >> 20) + + val read_addr = Reg(init = UInt(0, scrAddrBits)) + val resp_valid = Reg(init = Bool(false)) + + io.smi.req.ready := !resp_valid + io.smi.resp.valid := resp_valid + io.smi.resp.bits := scr_rdata(read_addr) + + io.scr.wen := io.smi.req.fire() && io.smi.req.bits.rw + io.scr.wdata := io.smi.req.bits.data + io.scr.waddr := io.smi.req.bits.addr + + when (io.smi.req.fire()) { + read_addr := io.smi.req.bits.addr + resp_valid := Bool(true) + } + when (io.smi.resp.fire()) { resp_valid := Bool(false) } +} From ae3d96013afae2e2ee270568cb7beddd7e40144b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 5 Sep 2015 21:28:18 -0700 Subject: [PATCH 427/688] make TL -> NASTI converter ingest ClientUncachedTileLinkIO and move functionality to Unwrapper --- uncore/src/main/scala/coherence.scala | 6 + uncore/src/main/scala/tilelink.scala | 265 +++++++++++++++++--------- 2 files changed, 176 insertions(+), 95 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index f731a6cc..0c35f420 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -98,6 +98,7 @@ trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getExclusiveGrantType(): UInt // Mutate ManagerMetadata based on messages or cmds def managerMetadataOnReset: ManagerMetadata @@ -193,6 +194,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + def getExclusiveGrantType(): UInt = grantExclusive def managerMetadataOnReset = ManagerMetadata() @@ -295,6 +297,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate) def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + def getExclusiveGrantType(): UInt = grantExclusive def managerMetadataOnReset = ManagerMetadata() @@ -413,6 +416,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive) + def getExclusiveGrantType(): UInt = grantExclusive def managerMetadataOnReset = ManagerMetadata() @@ -532,6 +536,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive) + def getExclusiveGrantType(): UInt = grantExclusive def managerMetadataOnReset = ManagerMetadata() @@ -674,6 +679,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? + def getExclusiveGrantType(): UInt = grantExclusive def managerMetadataOnReset = ManagerMetadata() diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b78dcafd..467581d0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1241,9 +1241,137 @@ trait HasDataBeatCounters { } } +class ReorderQueueWrite[T <: Data](dType: T, tagWidth: Int) extends Bundle { + val data = dType.cloneType + val tag = UInt(width = tagWidth) + + override def cloneType = + new ReorderQueueWrite(dType, tagWidth).asInstanceOf[this.type] +} + +class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) + extends Module { + val io = new Bundle { + val enq = Decoupled(new ReorderQueueWrite(dType, tagWidth)).flip + val deq = new Bundle { + val valid = Bool(INPUT) + val tag = UInt(INPUT, tagWidth) + val data = dType.cloneType.asOutput + } + val full = Bool(OUTPUT) + } + + val roq_data = Reg(Vec(dType.clone, size)) + val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) + val roq_free = Reg(init = Fill(size, Bits(1, 1))) + + val roq_enq_addr = PriorityEncoder(roq_free) + val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) + + io.enq.ready := roq_free.orR + io.deq.data := roq_data(roq_deq_addr) + + when (io.enq.valid && io.enq.ready) { + roq_data(roq_enq_addr) := io.enq.bits.data + roq_tags(roq_enq_addr) := io.enq.bits.tag + roq_free(roq_enq_addr) := Bool(false) + } + + when (io.deq.valid) { + roq_free(roq_deq_addr) := Bool(true) + } +} + +class ClientTileLinkIOUnwrapperInfo extends Bundle { + val voluntary = Bool() + val builtin = Bool() +} + +class ClientTileLinkIOUnwrapper extends TLModule { + val io = new Bundle { + val in = new ClientTileLinkIO().flip + val out = new ClientUncachedTileLinkIO + } + + def needsRoqEnq(channel: HasTileLinkData): Bool = + !channel.hasMultibeatData() || channel.addr_beat === UInt(0) + + def needsRoqDeq(channel: HasTileLinkData): Bool = + !channel.hasMultibeatData() || channel.addr_beat === UInt(tlDataBeats - 1) + + val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, + Some((acq: Acquire) => acq.hasMultibeatData()))) + val roqArb = Module(new RRArbiter(new ReorderQueueWrite( + new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits), 2)) + + val iacq = io.in.acquire.bits + val irel = io.in.release.bits + val ognt = io.out.grant.bits + + val iacq_wait = needsRoqEnq(iacq) && !roqArb.io.in(0).ready + val irel_wait = needsRoqEnq(irel) && !roqArb.io.in(1).ready + + roqArb.io.in(0).valid := io.in.acquire.valid && needsRoqEnq(iacq) + roqArb.io.in(0).bits.data.voluntary := Bool(false) + roqArb.io.in(0).bits.data.builtin := iacq.isBuiltInType() + roqArb.io.in(0).bits.tag := iacq.client_xact_id + + acqArb.io.in(0).valid := io.in.acquire.valid && !iacq_wait + acqArb.io.in(0).bits := Acquire( + is_builtin_type = Bool(true), + a_type = Mux(iacq.isBuiltInType(), + iacq.a_type, Acquire.getBlockType), + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = iacq.addr_beat, + data = iacq.data, + union = Mux(iacq.isBuiltInType(), + iacq.union, Cat(Acquire.fullWriteMask, Bool(false)))) + io.in.acquire.ready := acqArb.io.in(0).ready && !iacq_wait + + roqArb.io.in(1).valid := io.in.release.valid && needsRoqEnq(iacq) + roqArb.io.in(1).bits.data.voluntary := irel.isVoluntary() + roqArb.io.in(1).bits.data.builtin := Bool(true) + roqArb.io.in(1).bits.tag := irel.client_xact_id + + acqArb.io.in(1).valid := io.in.release.valid && !irel_wait + acqArb.io.in(1).bits := PutBlock( + client_xact_id = irel.client_xact_id, + addr_block = irel.addr_block, + addr_beat = irel.addr_beat, + data = irel.data, + wmask = Acquire.fullWriteMask) + io.in.release.ready := acqArb.io.in(1).ready && !irel_wait + + io.out.acquire <> acqArb.io.out + + val roq = Module(new ReorderQueue( + new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits, 4)) + roq.io.enq <> roqArb.io.out + roq.io.deq.valid := io.out.grant.valid && needsRoqDeq(ognt) + roq.io.deq.tag := ognt.client_xact_id + + val gnt_builtin = roq.io.deq.data.builtin + val gnt_voluntary = roq.io.deq.data.voluntary + + io.in.grant.valid := io.out.grant.valid + io.in.grant.bits := Grant( + is_builtin_type = gnt_builtin, + g_type = MuxCase(ognt.g_type, Seq( + (!gnt_builtin, tlCoh.getExclusiveGrantType), + (gnt_voluntary, Grant.voluntaryAckType))), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + io.out.grant.ready := io.in.grant.ready + + io.in.probe.valid := Bool(false) +} + class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val io = new Bundle { - val tl = new ManagerTileLinkIO + val tl = new ClientUncachedTileLinkIO().flip val nasti = new NASTIMasterIO } @@ -1254,9 +1382,6 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIMasterIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) io.tl.acquire.ready := Bool(false) - io.tl.probe.valid := Bool(false) - io.tl.release.ready := Bool(false) - io.tl.finish.ready := Bool(true) io.nasti.b.ready := Bool(false) io.nasti.r.ready := Bool(false) @@ -1266,8 +1391,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val dst_off = dstIdBits + tlClientXactIdBits val acq_has_data = io.tl.acquire.bits.hasData() - val rel_has_data = io.tl.release.bits.hasData() - val is_write = io.tl.release.valid || (io.tl.acquire.valid && acq_has_data) + val is_write = io.tl.acquire.valid && acq_has_data // Decompose outgoing TL Acquires into NASTI address and data channels val active_out = Reg(init=Bool(false)) @@ -1275,38 +1399,18 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val tag_out = Reg(UInt(width = nastiXIdBits)) val addr_out = Reg(UInt(width = nastiXAddrBits)) val has_data = Reg(init=Bool(false)) - val data_from_rel = Reg(init=Bool(false)) val is_subblock = io.tl.acquire.bits.isSubBlockType() - val (tl_cnt_out, tl_wrap_out) = - Counter((io.tl.acquire.fire() && io.tl.acquire.bits.hasMultibeatData()) || - (io.tl.release.fire() && rel_has_data), tlDataBeats) + val (tl_cnt_out, tl_wrap_out) = Counter( + io.tl.acquire.fire() && io.tl.acquire.bits.hasMultibeatData(), tlDataBeats) val tl_done_out = Reg(init=Bool(false)) - val roq_size = 4 - val roq_data = Reg(Vec(UInt(width = tlByteAddrBits), roq_size)) - val roq_tags = Reg(Vec(UInt(width = nastiRIdBits), roq_size)) - val roq_free = Reg(init = Fill(roq_size, Bits(1, 1))) - val roq_full = !roq_free.orR - - val roq_enq_addr = PriorityEncoder(roq_free) - val roq_enq_valid = io.tl.acquire.fire() && !acq_has_data && is_subblock - val roq_enq_data = io.tl.acquire.bits.addr_byte() - val roq_enq_tag = io.nasti.ar.bits.id - - val roq_deq_tag = io.nasti.r.bits.id - val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === roq_deq_tag)) - val roq_deq_data = roq_data(roq_deq_addr) - val roq_deq_valid = io.nasti.r.fire() && !io.nasti.r.bits.id(0) - - when (roq_enq_valid) { - roq_data(roq_enq_addr) := roq_enq_data - roq_tags(roq_enq_addr) := roq_enq_tag - roq_free(roq_enq_addr) := Bool(false) - } - - when (roq_deq_valid) { - roq_free(roq_deq_addr) := Bool(true) - } + val roq = Module(new ReorderQueue( + UInt(width = tlByteAddrBits), nastiRIdBits, 4)) + roq.io.enq.valid := io.tl.acquire.fire() && !acq_has_data && is_subblock + roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.data := io.tl.acquire.bits.addr_byte() + roq.io.deq.valid := io.nasti.r.fire() && !io.nasti.r.bits.id(0) + roq.io.deq.tag := io.nasti.r.bits.id io.nasti.ar.bits.id := tag_out io.nasti.ar.bits.addr := addr_out @@ -1320,63 +1424,40 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.ar.bits.region := UInt("b0000") io.nasti.ar.bits.user := UInt(0) io.nasti.aw.bits := io.nasti.ar.bits - io.nasti.w.bits.strb := Mux(data_from_rel, SInt(-1), io.tl.acquire.bits.wmask()) - io.nasti.w.bits.data := Mux(data_from_rel, io.tl.release.bits.data, io.tl.acquire.bits.data) + io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() + io.nasti.w.bits.data := io.tl.acquire.bits.data io.nasti.w.bits.last := tl_wrap_out || (io.tl.acquire.fire() && is_subblock) when(!active_out){ - io.tl.release.ready := io.nasti.w.ready - io.tl.acquire.ready := io.nasti.w.ready && !io.tl.release.valid - io.nasti.w.valid := (io.tl.release.valid && rel_has_data) || - (io.tl.acquire.valid && acq_has_data) - when(io.nasti.w.ready && (io.tl.release.valid || io.tl.acquire.valid)) { + io.tl.acquire.ready := io.nasti.w.ready + io.nasti.w.valid := is_write + when(io.tl.acquire.fire()) { active_out := (!is_write && !io.nasti.ar.ready) || (is_write && !(io.nasti.aw.ready && io.nasti.w.ready)) || (io.nasti.w.valid && Bool(tlDataBeats > 1)) io.nasti.aw.valid := is_write io.nasti.ar.valid := !is_write cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) - when(io.tl.release.valid) { - data_from_rel := Bool(true) - io.nasti.w.bits.data := io.tl.release.bits.data - io.nasti.w.bits.strb := SInt(-1) - val tag = Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - val addr = io.tl.release.bits.full_addr() + // The last bit indicates to the Grant logic what g_type to send back + // For read, true = getDataBlockType, false = getDataBeatType + // For write, it should always be false, so that putAckType is sent + val tag = Cat(io.tl.acquire.bits.client_xact_id, + !is_write && !is_subblock) + val addr = io.tl.acquire.bits.full_addr() + when(is_write) { io.nasti.aw.bits.id := tag io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := UInt(tlDataBeats-1) - tag_out := tag - addr_out := addr - has_data := rel_has_data - tl_done_out := tl_wrap_out - } .elsewhen(io.tl.acquire.valid) { - data_from_rel := Bool(false) - io.nasti.w.bits.data := io.tl.acquire.bits.data - io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() - // The last bit indicates to the Grant logic what g_type to send back - // For read, true = getDataBlockType, false = getDataBeatType - // For write, it should always be false, so that putAckType is sent - val tag = Cat(io.tl.acquire.bits.client_id, - io.tl.acquire.bits.client_xact_id, - !is_write && !is_subblock) - val addr = io.tl.acquire.bits.full_addr() - when(is_write) { - io.nasti.aw.bits.id := tag - io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) - } .otherwise { - io.nasti.ar.bits.id := tag - io.nasti.ar.bits.addr := addr - io.nasti.ar.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() - } - tag_out := tag - addr_out := addr - has_data := acq_has_data - tl_done_out := tl_wrap_out || is_subblock + io.nasti.aw.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) + } .otherwise { + io.nasti.ar.bits.id := tag + io.nasti.ar.bits.addr := addr + io.nasti.ar.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) + io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() } + tag_out := tag + addr_out := addr + has_data := acq_has_data + tl_done_out := tl_wrap_out || is_subblock } } when(active_out) { @@ -1384,16 +1465,11 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.aw.valid := !cmd_sent_out && has_data cmd_sent_out := cmd_sent_out || io.nasti.ar.fire() || io.nasti.aw.fire() when(has_data && !tl_done_out) { - when(data_from_rel) { - io.tl.release.ready := io.nasti.w.ready - io.nasti.w.valid := io.tl.release.valid - } .otherwise { - io.tl.acquire.ready := io.nasti.w.ready - io.nasti.w.valid := io.tl.acquire.valid - } + io.tl.acquire.ready := io.nasti.w.ready + io.nasti.w.valid := io.tl.acquire.valid } when(tl_wrap_out) { tl_done_out := Bool(true) } - when(cmd_sent_out && !roq_full && (!has_data || tl_done_out)) { + when(cmd_sent_out && roq.io.enq.ready && (!has_data || tl_done_out)) { active_out := Bool(false) } } @@ -1410,12 +1486,11 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val r_aligned_data = Mux(io.nasti.r.bits.id(0), io.nasti.r.bits.data, - io.nasti.r.bits.data << Cat(roq_deq_data, UInt(0, 3))) + io.nasti.r.bits.data << Cat(roq.io.deq.data, UInt(0, 3))) gnt_arb.io.in(0).valid := io.nasti.r.valid io.nasti.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) io.nasti.r.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = Bool(true), g_type = Mux(io.nasti.r.bits.id(0), Grant.getDataBlockType, Grant.getDataBeatType), // TODO: Assumes MI or MEI protocol @@ -1427,11 +1502,12 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { gnt_arb.io.in(1).valid := io.nasti.b.valid io.nasti.b.ready := gnt_arb.io.in(1).ready gnt_arb.io.in(1).bits := Grant( - dst = (if(dstIdBits > 0) io.nasti.b.bits.id(dst_off, tlClientXactIdBits + 1) else UInt(0)), is_builtin_type = Bool(true), - g_type = Mux(io.nasti.b.bits.id(0), Grant.voluntaryAckType, Grant.putAckType), + g_type = Grant.putAckType, client_xact_id = io.nasti.b.bits.id >> 1, - manager_xact_id = UInt(0)) + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = Bits(0)) } class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { @@ -1513,8 +1589,7 @@ class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters when(io.tl.release.valid) { active_out := Bool(true) cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, + tag_out := Cat(io.tl.release.bits.client_xact_id, io.tl.release.bits.isVoluntary()) addr_out := io.tl.release.bits.addr_block has_data := rel_has_data From 6ee6ea4f1ecfb7a24e14c15ca7b439693f17c5ee Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 10 Sep 2015 17:52:12 -0700 Subject: [PATCH 428/688] use Put/Get/PutBlock/GetBlock constructors in broadcast hub --- uncore/src/main/scala/broadcast.scala | 58 ++++++++++++++++----------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index a396de24..56d7c814 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -255,32 +255,42 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { !io.irel().isVoluntary() && (state === s_probe) - val oacq_type = MuxLookup(state, Acquire.getBlockType, Seq( - (s_probe, Acquire.putBlockType), - (s_mem_write, Mux(subblock_type, Acquire.putType, Acquire.putBlockType)), - (s_mem_read, Mux(subblock_type, Acquire.getType, Acquire.getBlockType)))) - val oacq_beat = MuxLookup(state, UInt(0), Seq( - (s_probe, io.irel().addr_beat), - (s_mem_write, Mux(subblock_type, xact.addr_beat, oacq_data_cnt)), - (s_mem_read, Mux(subblock_type, xact.addr_beat, UInt(0))))) - val oacq_data = MuxLookup(state, Bits(0), Seq( - (s_probe, io.irel().data), - (s_mem_write, Mux(subblock_type, - data_buffer(0), data_buffer(oacq_data_cnt))))) - val oacq_union = MuxLookup(state, Bits(0), Seq( - (s_probe, Acquire.fullWriteMask), - (s_mem_write, xact.wmask()), - (s_mem_read, Cat(xact.addr_byte(), xact.op_size(), M_XRD)))) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits := Bundle(Acquire( - is_builtin_type = Bool(true), - a_type = oacq_type, + val oacq_probe = PutBlock( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, - addr_beat = oacq_beat, - data = oacq_data, - union = Cat(oacq_union, Bool(true))))(outerTLParams) + addr_beat = io.irel().addr_beat, + data = io.irel().data) + + val oacq_write_beat = Put( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = xact.addr_beat, + data = data_buffer(0), + wmask = xact.wmask()) + + val oacq_write_block = PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = data_buffer(oacq_data_cnt)) + + val oacq_read_beat = Get( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = xact.addr_beat, + addr_byte = xact.addr_byte(), + operand_size = xact.op_size(), + alloc = Bool(false)) + + val oacq_read_block = GetBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block) + + io.outer.acquire.valid := Bool(false) + io.outer.acquire.bits := Mux(state === s_probe, oacq_probe, + Mux(state === s_mem_write, + Mux(subblock_type, oacq_write_beat, oacq_write_block), + Mux(subblock_type, oacq_read_beat, oacq_read_block))) io.outer.grant.ready := Bool(false) io.inner.probe.valid := Bool(false) From 64717706a91e77906f2db06db3cc20b1cdb7df75 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 10 Sep 2015 17:52:26 -0700 Subject: [PATCH 429/688] get rid of non-NASTI RTC module --- uncore/src/main/scala/rtc.scala | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 9253d0ea..1fb9b53f 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -4,38 +4,6 @@ import Chisel._ import junctions.{NASTIMasterIO, NASTIAddrHashMap, SMIIO} class RTC(pcr_MTIME: Int) extends Module { - private val nCores = params(HTIFNCores) - - val io = new Bundle { - val smi = Vec.fill(nCores) { new SMIIO(64, 12) } - } - - val rtc = Reg(init=UInt(0,64)) - val rtc_tick = Counter(params(RTCPeriod)).inc() - - for ((smi, i) <- io.smi.zipWithIndex) { - val rtc_sending = Reg(init = Bool(false)) - val rtc_outstanding = Reg(init = Bool(false)) - - when (rtc_tick) { - rtc := rtc + UInt(1) - rtc_sending := Bool(true) - rtc_outstanding := Bool(true) - } - when (smi.req.fire()) { rtc_sending := Bool(false) } - when (smi.resp.fire()) { rtc_outstanding := Bool(false) } - - assert(!rtc_tick || !rtc_outstanding, "Last rtc tick not yet sent") - - smi.req.bits.addr := UInt(pcr_MTIME) - smi.req.bits.rw := Bool(true) - smi.req.bits.data := rtc - smi.req.valid := rtc_sending - smi.resp.ready := Bool(true) - } -} - -class RTCNASTI(pcr_MTIME: Int) extends Module { val io = new NASTIMasterIO private val nCores = params(HTIFNCores) From f9965648f230cfcb1c5352df9f8f133f9e9cef11 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 10 Sep 2015 17:53:04 -0700 Subject: [PATCH 430/688] fix up some things in tilelink.scala --- uncore/src/main/scala/tilelink.scala | 40 +++++++++++++++------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 467581d0..3228b9a3 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -656,7 +656,7 @@ object Grant { manager_xact_id: UInt, addr_beat: UInt, data: UInt): Grant = { - val gnt = new Grant + val gnt = Wire(new Grant) gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type gnt.client_xact_id := client_xact_id @@ -1346,7 +1346,7 @@ class ClientTileLinkIOUnwrapper extends TLModule { io.out.acquire <> acqArb.io.out val roq = Module(new ReorderQueue( - new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits, 4)) + new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits, tlMaxClientsPerPort)) roq.io.enq <> roqArb.io.out roq.io.deq.valid := io.out.grant.valid && needsRoqDeq(ognt) roq.io.deq.tag := ognt.client_xact_id @@ -1375,6 +1375,15 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val nasti = new NASTIMasterIO } + private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( + MT_B -> UInt(0), + MT_BU -> UInt(0), + MT_H -> UInt(1), + MT_HU -> UInt(1), + MT_W -> UInt(2), + MT_D -> UInt(3), + MT_Q -> UInt(log2Up(tlDataBytes)))) + val dataBits = tlDataBits*tlDataBeats val dstIdBits = params(LNHeaderBits) require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction @@ -1405,28 +1414,23 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val tl_done_out = Reg(init=Bool(false)) val roq = Module(new ReorderQueue( - UInt(width = tlByteAddrBits), nastiRIdBits, 4)) + UInt(width = tlByteAddrBits), nastiRIdBits, tlMaxClientsPerPort)) roq.io.enq.valid := io.tl.acquire.fire() && !acq_has_data && is_subblock roq.io.enq.bits.tag := io.nasti.ar.bits.id roq.io.enq.bits.data := io.tl.acquire.bits.addr_byte() roq.io.deq.valid := io.nasti.r.fire() && !io.nasti.r.bits.id(0) roq.io.deq.tag := io.nasti.r.bits.id - io.nasti.ar.bits.id := tag_out - io.nasti.ar.bits.addr := addr_out - io.nasti.ar.bits.len := Mux(has_data, UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := UInt(log2Ceil(tlDataBits)) - io.nasti.ar.bits.burst := UInt("b01") - io.nasti.ar.bits.lock := Bool(false) - io.nasti.ar.bits.cache := UInt("b0000") - io.nasti.ar.bits.prot := UInt("b000") - io.nasti.ar.bits.qos := UInt("b0000") - io.nasti.ar.bits.region := UInt("b0000") - io.nasti.ar.bits.user := UInt(0) + io.nasti.ar.bits := NASTIReadAddressChannel( + id = tag_out, + addr = addr_out, + size = UInt(log2Ceil(tlDataBytes)), + len = Mux(has_data, UInt(tlDataBeats - 1), UInt(0))) io.nasti.aw.bits := io.nasti.ar.bits - io.nasti.w.bits.strb := io.tl.acquire.bits.wmask() - io.nasti.w.bits.data := io.tl.acquire.bits.data - io.nasti.w.bits.last := tl_wrap_out || (io.tl.acquire.fire() && is_subblock) + io.nasti.w.bits := NASTIWriteDataChannel( + strb = io.tl.acquire.bits.wmask(), + data = io.tl.acquire.bits.data, + last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) when(!active_out){ io.tl.acquire.ready := io.nasti.w.ready @@ -1452,7 +1456,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.ar.bits.id := tag io.nasti.ar.bits.addr := addr io.nasti.ar.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := io.tl.acquire.bits.op_size() + io.nasti.ar.bits.size := opSizeToXSize(io.tl.acquire.bits.op_size()) } tag_out := tag addr_out := addr From 9d89d2a5580f839c73e34c1a14e6e93689834644 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 10 Sep 2015 17:53:17 -0700 Subject: [PATCH 431/688] get rid of MemIO -> TileLink converters --- uncore/src/main/scala/tilelink.scala | 264 --------------------------- 1 file changed, 264 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3228b9a3..ee176280 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1513,267 +1513,3 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { addr_beat = UInt(0), data = Bits(0)) } - -class MemPipeIOTileLinkIOConverter(outstanding: Int) extends MIFModule { - val io = new Bundle { - val tl = new ManagerTileLinkIO - val mem = new MemPipeIO - } - - val a = Module(new MemIOTileLinkIOConverter(1)) - val b = Module(new MemPipeIOMemIOConverter(outstanding)) - io.tl <> a.io.tl - b.io.cpu.req_cmd <> Queue(a.io.mem.req_cmd, 2, pipe=true) - b.io.cpu.req_data <> Queue(a.io.mem.req_data, mifDataBeats, pipe=true) - a.io.mem.resp <> b.io.cpu.resp - io.mem <> b.io.mem -} - -//Adapter betweewn an UncachedTileLinkIO and a mem controller MemIO -class MemIOTileLinkIOConverter(qDepth: Int) extends TLModule with MIFParameters { - val io = new Bundle { - val tl = new ManagerTileLinkIO - val mem = new MemIO - } - val dataBits = tlDataBits*tlDataBeats - val dstIdBits = params(LNHeaderBits) - require(tlDataBits*tlDataBeats == mifDataBits*mifDataBeats, "Data sizes between LLC and MC don't agree") - require(dstIdBits + tlClientXactIdBits < mifTagBits, "MemIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + mifTagBits) - - io.tl.acquire.ready := Bool(false) - io.tl.probe.valid := Bool(false) - io.tl.release.ready := Bool(false) - io.tl.finish.ready := Bool(true) - io.mem.resp.ready := Bool(false) - - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.hasData() - val rel_has_data = io.tl.release.bits.hasData() - - // Decompose outgoing TL Acquires into MemIO cmd and data - val active_out = Reg(init=Bool(false)) - val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(UInt(width = mifTagBits)) - val addr_out = Reg(UInt(width = mifAddrBits)) - val has_data = Reg(init=Bool(false)) - val data_from_rel = Reg(init=Bool(false)) - val (tl_cnt_out, tl_wrap_out) = - Counter((io.tl.acquire.fire() && acq_has_data) || - (io.tl.release.fire() && rel_has_data), tlDataBeats) - val tl_done_out = Reg(init=Bool(false)) - val make_grant_ack = Reg(init=Bool(false)) - - gnt_arb.io.in(1).valid := Bool(false) - gnt_arb.io.in(1).bits := Grant( - dst = (if(dstIdBits > 0) tag_out(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = Bool(true), - g_type = Mux(data_from_rel, Grant.voluntaryAckType, Grant.putAckType), - client_xact_id = tag_out >> 1, - manager_xact_id = UInt(0)) - - if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { - val mem_cmd_q = Module(new Queue(new MemReqCmd, qDepth)) - val mem_data_q = Module(new Queue(new MemData, qDepth)) - mem_cmd_q.io.enq.valid := Bool(false) - mem_data_q.io.enq.valid := Bool(false) - val (mif_cnt_out, mif_wrap_out) = Counter(mem_data_q.io.enq.fire(), mifDataBeats) - val mif_done_out = Reg(init=Bool(false)) - val tl_buf_out = Reg(Vec(io.tl.acquire.bits.data, tlDataBeats)) - val mif_buf_out = Vec(new MemData, mifDataBeats) - mif_buf_out := mif_buf_out.fromBits(tl_buf_out.toBits) - val mif_prog_out = (mif_cnt_out+UInt(1, width = log2Up(mifDataBeats+1)))*UInt(mifDataBits) - val tl_prog_out = tl_cnt_out*UInt(tlDataBits) - - when(!active_out){ - io.tl.release.ready := Bool(true) - io.tl.acquire.ready := !io.tl.release.valid - when(io.tl.release.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - addr_out := io.tl.release.bits.addr_block - has_data := rel_has_data - data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.requiresAck() - tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.release.bits.data - } .elsewhen(io.tl.acquire.valid) { - active_out := Bool(true) - cmd_sent_out := Bool(false) - tag_out := Cat(io.tl.release.bits.client_id, - io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) - addr_out := io.tl.acquire.bits.addr_block - has_data := acq_has_data - data_from_rel := Bool(false) - make_grant_ack := acq_has_data - tl_done_out := tl_wrap_out - tl_buf_out(tl_cnt_out) := io.tl.acquire.bits.data - } - } - when(active_out) { - mem_cmd_q.io.enq.valid := !cmd_sent_out - cmd_sent_out := cmd_sent_out || mem_cmd_q.io.enq.fire() - when(has_data) { - when(!tl_done_out) { - io.tl.acquire.ready := Bool(true) - when(io.tl.acquire.valid) { - tl_buf_out(tl_cnt_out) := Mux(data_from_rel, - io.tl.release.bits.data, - io.tl.acquire.bits.data) - } - } - when(!mif_done_out) { - mem_data_q.io.enq.valid := tl_done_out || mif_prog_out <= tl_prog_out - } - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(mif_wrap_out) { mif_done_out := Bool(true) } - when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) - when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } - } - when(cmd_sent_out && (!has_data || mif_done_out) && !make_grant_ack) { - active_out := Bool(false) - } - } - - mem_cmd_q.io.enq.bits.rw := has_data - mem_cmd_q.io.enq.bits.tag := tag_out - mem_cmd_q.io.enq.bits.addr := addr_out - mem_data_q.io.enq.bits.data := mif_buf_out(mif_cnt_out).data - io.mem.req_cmd <> mem_cmd_q.io.deq - io.mem.req_data <> mem_data_q.io.deq - } else { // Don't make the data buffers and try to flow cmd and data - io.mem.req_cmd.valid := Bool(false) - io.mem.req_data.valid := Bool(false) - io.mem.req_cmd.bits.rw := has_data - io.mem.req_cmd.bits.tag := tag_out - io.mem.req_cmd.bits.addr := addr_out - io.mem.req_data.bits.data := Mux(data_from_rel, - io.tl.release.bits.data, - io.tl.acquire.bits.data) - when(!active_out){ - io.tl.release.ready := io.mem.req_data.ready - io.tl.acquire.ready := io.mem.req_data.ready && !io.tl.release.valid - io.mem.req_data.valid := (io.tl.release.valid && rel_has_data) || - (io.tl.acquire.valid && acq_has_data) - when(io.mem.req_data.ready && (io.tl.release.valid || io.tl.acquire.valid)) { - active_out := !io.mem.req_cmd.ready || io.mem.req_data.valid - io.mem.req_cmd.valid := Bool(true) - cmd_sent_out := io.mem.req_cmd.ready - tl_done_out := tl_wrap_out - when(io.tl.release.valid) { - data_from_rel := Bool(true) - make_grant_ack := io.tl.release.bits.requiresAck() - io.mem.req_data.bits.data := io.tl.release.bits.data - val tag = Cat(io.tl.release.bits.client_id, - io.tl.release.bits.client_xact_id, - io.tl.release.bits.isVoluntary()) - val addr = io.tl.release.bits.addr_block - io.mem.req_cmd.bits.tag := tag - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := rel_has_data - tag_out := tag - addr_out := addr - has_data := rel_has_data - } .elsewhen(io.tl.acquire.valid) { - data_from_rel := Bool(false) - make_grant_ack := acq_has_data // i.e. is it a Put - io.mem.req_data.bits.data := io.tl.acquire.bits.data - io.mem.req_cmd.bits.rw := acq_has_data - val tag = Cat(io.tl.acquire.bits.client_id, - io.tl.acquire.bits.client_xact_id, - io.tl.acquire.bits.isBuiltInType()) - val addr = io.tl.acquire.bits.addr_block - io.mem.req_cmd.bits.tag := tag - io.mem.req_cmd.bits.addr := addr - io.mem.req_cmd.bits.rw := acq_has_data - tag_out := tag - addr_out := addr - has_data := acq_has_data - } - } - } - when(active_out) { - io.mem.req_cmd.valid := !cmd_sent_out - cmd_sent_out := cmd_sent_out || io.mem.req_cmd.fire() - when(has_data && !tl_done_out) { - when(data_from_rel) { - io.tl.release.ready := io.mem.req_data.ready - io.mem.req_data.valid := io.tl.release.valid - } .otherwise { - io.tl.acquire.ready := io.mem.req_data.ready - io.mem.req_data.valid := io.tl.acquire.valid - } - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(tl_done_out && make_grant_ack) { - gnt_arb.io.in(1).valid := Bool(true) // TODO: grants for voluntary acks? - when(gnt_arb.io.in(1).ready) { make_grant_ack := Bool(false) } - } - when(cmd_sent_out && (!has_data || tl_done_out) && !make_grant_ack) { - active_out := Bool(false) - } - } - } - - // Aggregate incoming MemIO responses into TL Grants - val active_in = Reg(init=Bool(false)) - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val tag_in = Reg(UInt(width = mifTagBits)) - - if(tlDataBits != mifDataBits || tlDataBeats != mifDataBeats) { - val (mif_cnt_in, mif_wrap_in) = Counter(io.mem.resp.fire(), mifDataBeats) // TODO: Assumes all resps have data - val mif_done_in = Reg(init=Bool(false)) - val mif_buf_in = Reg(Vec(new MemData, mifDataBeats)) - val tl_buf_in = Vec(io.tl.acquire.bits.data, tlDataBeats) - tl_buf_in := tl_buf_in.fromBits(mif_buf_in.toBits) - val tl_prog_in = (tl_cnt_in+UInt(1, width = log2Up(tlDataBeats+1)))*UInt(tlDataBits) - val mif_prog_in = mif_cnt_in*UInt(mifDataBits) - gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) tag_in(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = tag_in(0), - g_type = Mux(tag_in(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = tag_in >> 1, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = tl_buf_in(tl_cnt_in)) - - when(!active_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - active_in := Bool(true) - mif_done_in := mif_wrap_in - tag_in := io.mem.resp.bits.tag - mif_buf_in(tl_cnt_in).data := io.mem.resp.bits.data - } - } - when(active_in) { - gnt_arb.io.in(0).valid := mif_done_in || tl_prog_in <= mif_prog_in - when(!mif_done_in) { - io.mem.resp.ready := Bool(true) - when(io.mem.resp.valid) { - mif_buf_in(mif_cnt_in).data := io.mem.resp.bits.data - } - } - when(mif_wrap_in) { mif_done_in := Bool(true) } - when(tl_wrap_in) { active_in := Bool(false) } - } - } else { // Don't generate all the uneeded data buffers and flow resp - gnt_arb.io.in(0).valid := io.mem.resp.valid - io.mem.resp.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - dst = (if(dstIdBits > 0) io.mem.resp.bits.tag(dst_off, tlClientXactIdBits + 1) else UInt(0)), - is_builtin_type = io.mem.resp.bits.tag(0), - g_type = Mux(io.mem.resp.bits.tag(0), Grant.getDataBlockType, UInt(0)), // TODO: Assumes MI or MEI protocol - client_xact_id = io.mem.resp.bits.tag >> 1, - manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, - data = io.mem.resp.bits.data) - } -} From bd536d8832dca4cb2836a6db3fbd8806308e4813 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 10 Sep 2015 17:53:42 -0700 Subject: [PATCH 432/688] make HTIFModuleIO an anonymous bundle --- uncore/src/main/scala/htif.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index c153d05b..9654edf1 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -43,15 +43,13 @@ class HTIFIO extends HTIFBundle { // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class HTIFModuleIO extends HTIFBundle { +class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { + val io = new Bundle { val host = new HostIO val cpu = Vec(new HTIFIO, nCores).flip val mem = new ClientUncachedTileLinkIO val scr = new SMIIO(64, scrAddrBits) -} - -class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { - val io = new HTIFModuleIO + } io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' From 8b2341b1b1890303706957ffb4dd01daaef9d4f4 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 18 Sep 2015 09:41:37 -0700 Subject: [PATCH 433/688] use reorder queue instead of extra tag bit to determine TL g_type in NASTI -> TL converter --- uncore/src/main/scala/tilelink.scala | 39 ++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ee176280..3b51ccaa 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1369,6 +1369,11 @@ class ClientTileLinkIOUnwrapper extends TLModule { io.in.probe.valid := Bool(false) } +class NASTIMasterIOTileLinkIOConverterInfo extends TLBundle { + val byteOff = UInt(width = tlByteAddrBits) + val subblock = Bool() +} + class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val io = new Bundle { val tl = new ClientUncachedTileLinkIO().flip @@ -1414,11 +1419,17 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val tl_done_out = Reg(init=Bool(false)) val roq = Module(new ReorderQueue( - UInt(width = tlByteAddrBits), nastiRIdBits, tlMaxClientsPerPort)) - roq.io.enq.valid := io.tl.acquire.fire() && !acq_has_data && is_subblock + new NASTIMasterIOTileLinkIOConverterInfo, + nastiRIdBits, tlMaxClientsPerPort)) + + val (nasti_cnt_out, nasti_wrap_out) = Counter( + io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) + + roq.io.enq.valid := io.tl.acquire.fire() && !acq_has_data roq.io.enq.bits.tag := io.nasti.ar.bits.id - roq.io.enq.bits.data := io.tl.acquire.bits.addr_byte() - roq.io.deq.valid := io.nasti.r.fire() && !io.nasti.r.bits.id(0) + roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() + roq.io.enq.bits.data.subblock := is_subblock + roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) roq.io.deq.tag := io.nasti.r.bits.id io.nasti.ar.bits := NASTIReadAddressChannel( @@ -1442,11 +1453,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { io.nasti.aw.valid := is_write io.nasti.ar.valid := !is_write cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) - // The last bit indicates to the Grant logic what g_type to send back - // For read, true = getDataBlockType, false = getDataBeatType - // For write, it should always be false, so that putAckType is sent - val tag = Cat(io.tl.acquire.bits.client_xact_id, - !is_write && !is_subblock) + val tag = io.tl.acquire.bits.client_xact_id val addr = io.tl.acquire.bits.full_addr() when(is_write) { io.nasti.aw.bits.id := tag @@ -1488,17 +1495,17 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out - val r_aligned_data = Mux(io.nasti.r.bits.id(0), - io.nasti.r.bits.data, - io.nasti.r.bits.data << Cat(roq.io.deq.data, UInt(0, 3))) + val r_aligned_data = Mux(roq.io.deq.data.subblock, + io.nasti.r.bits.data << Cat(roq.io.deq.data.byteOff, UInt(0, 3)), + io.nasti.r.bits.data) gnt_arb.io.in(0).valid := io.nasti.r.valid io.nasti.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( is_builtin_type = Bool(true), - g_type = Mux(io.nasti.r.bits.id(0), - Grant.getDataBlockType, Grant.getDataBeatType), // TODO: Assumes MI or MEI protocol - client_xact_id = io.nasti.r.bits.id >> 1, + g_type = Mux(roq.io.deq.data.subblock, + Grant.getDataBeatType, Grant.getDataBlockType), + client_xact_id = io.nasti.r.bits.id, manager_xact_id = UInt(0), addr_beat = tl_cnt_in, data = r_aligned_data) @@ -1508,7 +1515,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { gnt_arb.io.in(1).bits := Grant( is_builtin_type = Bool(true), g_type = Grant.putAckType, - client_xact_id = io.nasti.b.bits.id >> 1, + client_xact_id = io.nasti.b.bits.id, manager_xact_id = UInt(0), addr_beat = UInt(0), data = Bits(0)) From b4d21148ecbc1759c0898f0f52786342029a7e1a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 22 Sep 2015 09:43:42 -0700 Subject: [PATCH 434/688] get rid of NASTI error assertion --- uncore/src/main/scala/tilelink.scala | 5 ----- 1 file changed, 5 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 3b51ccaa..ff7163ea 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1485,11 +1485,6 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { } } - assert (!io.nasti.r.valid || !io.nasti.r.bits.resp(1), - "NASTI read response error") - assert (!io.nasti.b.valid || !io.nasti.b.bits.resp(1), - "NASTI write response error") - // Aggregate incoming NASTI responses into TL Grants val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) From ee6754dacaf66bf2fddb24e8f14a7d955c97dfe9 Mon Sep 17 00:00:00 2001 From: ducky Date: Thu, 24 Sep 2015 16:16:08 -0700 Subject: [PATCH 435/688] Fix clone -> cloneType --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ff7163ea..7b4bbf84 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1261,7 +1261,7 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val full = Bool(OUTPUT) } - val roq_data = Reg(Vec(dType.clone, size)) + val roq_data = Reg(Vec(dType.cloneType, size)) val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) val roq_free = Reg(init = Fill(size, Bits(1, 1))) From 3b86790c3f6c847a98e007d7ba7505d858a5b56b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 24 Sep 2015 16:58:20 -0700 Subject: [PATCH 436/688] replace NASTIMasterIO and NASTISlaveIO with NASTIIO --- uncore/src/main/scala/rtc.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 1fb9b53f..29ffb870 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -1,10 +1,10 @@ package uncore import Chisel._ -import junctions.{NASTIMasterIO, NASTIAddrHashMap, SMIIO} +import junctions.{NASTIIO, NASTIAddrHashMap, SMIIO} class RTC(pcr_MTIME: Int) extends Module { - val io = new NASTIMasterIO + val io = new NASTIIO private val nCores = params(HTIFNCores) private val addrMap = params(NASTIAddrHashMap) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ff7163ea..00e7a6fe 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1369,15 +1369,15 @@ class ClientTileLinkIOUnwrapper extends TLModule { io.in.probe.valid := Bool(false) } -class NASTIMasterIOTileLinkIOConverterInfo extends TLBundle { +class NASTIIOTileLinkIOConverterInfo extends TLBundle { val byteOff = UInt(width = tlByteAddrBits) val subblock = Bool() } -class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { +class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { val io = new Bundle { val tl = new ClientUncachedTileLinkIO().flip - val nasti = new NASTIMasterIO + val nasti = new NASTIIO } private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( @@ -1393,7 +1393,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val dstIdBits = params(LNHeaderBits) require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIMasterIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) + require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) io.tl.acquire.ready := Bool(false) @@ -1419,7 +1419,7 @@ class NASTIMasterIOTileLinkIOConverter extends TLModule with NASTIParameters { val tl_done_out = Reg(init=Bool(false)) val roq = Module(new ReorderQueue( - new NASTIMasterIOTileLinkIOConverterInfo, + new NASTIIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxClientsPerPort)) val (nasti_cnt_out, nasti_wrap_out) = Counter( From 3ff830e1182da0b0e86de9178365f59316baeff9 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 24 Sep 2015 17:43:53 -0700 Subject: [PATCH 437/688] ReorderQueue uses Vec of Bools instead of Bits for roq_free --- uncore/src/main/scala/tilelink.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index e8b5477d..859a115f 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1261,14 +1261,14 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val full = Bool(OUTPUT) } - val roq_data = Reg(Vec(dType.cloneType, size)) + val roq_data = Reg(Vec(dType.cloneType, size)) val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) - val roq_free = Reg(init = Fill(size, Bits(1, 1))) + val roq_free = Reg(init = Vec(size, Bool(true))) val roq_enq_addr = PriorityEncoder(roq_free) val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) - io.enq.ready := roq_free.orR + io.enq.ready := roq_free.reduce(_ || _) io.deq.data := roq_data(roq_deq_addr) when (io.enq.valid && io.enq.ready) { From d1f2d40a908c45c31209c5eb3585cdd5bf8f7baa Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 24 Sep 2015 17:50:09 -0700 Subject: [PATCH 438/688] replace remaining uses of Vec.fill --- uncore/src/main/scala/cache.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ff0d7739..6884165b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -157,7 +157,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val metabits = rstVal.getWidth val tag_arr = SeqMem(Vec(UInt(width = metabits), nWays), nSets) when (rst || io.write.valid) { - tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) + tag_arr.write(waddr, Vec(nWays, wdata), wmask) } val tags = tag_arr.read(io.read.bits.idx, io.read.valid).toBits @@ -493,7 +493,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -588,8 +588,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding transaction metadata val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) + val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) + val wmask_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits/8))) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -981,7 +981,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(new L2WritebackReq) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) val xact_addr_block = Cat(xact.tag, xact.idx) val pending_irels = From 8c4ac0f4f341a245221582bb9d3fd98cb84c4da6 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 25 Sep 2015 12:07:03 -0700 Subject: [PATCH 439/688] make sure CSR/SCR data width matches xLen --- uncore/src/main/scala/htif.scala | 7 +++++-- uncore/src/main/scala/rtc.scala | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 9654edf1..20bd317f 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -10,6 +10,7 @@ case object HTIFWidth extends Field[Int] case object HTIFNSCR extends Field[Int] case object HTIFOffsetBits extends Field[Int] case object HTIFNCores extends Field[Int] +case object HTIFSCRDataBits extends Field[Int] abstract trait HTIFParameters extends UsesParameters { val dataBits = params(TLDataBits) @@ -17,6 +18,8 @@ abstract trait HTIFParameters extends UsesParameters { val w = params(HTIFWidth) val nSCR = params(HTIFNSCR) val scrAddrBits = log2Up(nSCR) + val scrDataBits = params(HTIFSCRDataBits) + val scrDataBytes = scrDataBits / 8 val offsetBits = params(HTIFOffsetBits) val nCores = params(HTIFNCores) } @@ -35,7 +38,7 @@ class HostIO extends HTIFBundle class HTIFIO extends HTIFBundle { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val pcr = new SMIIO(64, 12).flip + val pcr = new SMIIO(scrDataBits, 12).flip val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip val debug_stats_pcr = Bool(OUTPUT) @@ -48,7 +51,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val host = new HostIO val cpu = Vec(new HTIFIO, nCores).flip val mem = new ClientUncachedTileLinkIO - val scr = new SMIIO(64, scrAddrBits) + val scr = new SMIIO(scrDataBits, scrAddrBits) } io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 29ffb870..6c2e72c6 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -1,16 +1,15 @@ package uncore import Chisel._ -import junctions.{NASTIIO, NASTIAddrHashMap, SMIIO} +import junctions._ -class RTC(pcr_MTIME: Int) extends Module { +class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { val io = new NASTIIO - private val nCores = params(HTIFNCores) private val addrMap = params(NASTIAddrHashMap) val addrTable = Vec.tabulate(nCores) { i => - UInt(addrMap(s"conf:csr$i").start + pcr_MTIME * 8) + UInt(addrMap(s"conf:csr$i").start + pcr_MTIME * scrDataBytes) } val rtc = Reg(init=UInt(0,64)) From 308022210ac98929e2e0743fa435a67e1b4ee983 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 25 Sep 2015 12:07:27 -0700 Subject: [PATCH 440/688] use updated NASTI channel constructors --- uncore/src/main/scala/rtc.scala | 29 ++++++++++------------------ uncore/src/main/scala/tilelink.scala | 2 +- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 6c2e72c6..ade83749 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -18,6 +18,7 @@ class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { val sending_addr = Reg(init = Bool(false)) val sending_data = Reg(init = Bool(false)) val send_acked = Reg(init = Vec(nCores, Bool(true))) + val coreId = Wire(UInt(width = log2Up(nCores))) when (rtc_tick) { rtc := rtc + UInt(1) @@ -27,45 +28,35 @@ class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { } if (nCores > 1) { - val (core, addr_send_done) = Counter(io.aw.fire(), nCores) + val (addr_send_cnt, addr_send_done) = Counter(io.aw.fire(), nCores) val (_, data_send_done) = Counter(io.w.fire(), nCores) when (addr_send_done) { sending_addr := Bool(false) } when (data_send_done) { sending_data := Bool(false) } - io.aw.bits.id := core - io.aw.bits.addr := addrTable(core) + coreId := addr_send_cnt } else { when (io.aw.fire()) { sending_addr := Bool(false) } when (io.w.fire()) { sending_addr := Bool(false) } - io.aw.bits.id := UInt(0) - io.aw.bits.addr := addrTable(0) + coreId := UInt(0) } when (io.b.fire()) { send_acked(io.b.bits.id) := Bool(true) } io.aw.valid := sending_addr - io.aw.bits.size := UInt(3) // 8 bytes - io.aw.bits.len := UInt(0) - io.aw.bits.burst := Bits("b01") - io.aw.bits.lock := Bool(false) - io.aw.bits.cache := UInt("b0000") - io.aw.bits.prot := UInt("b000") - io.aw.bits.qos := UInt("b0000") - io.aw.bits.region := UInt("b0000") - io.aw.bits.user := UInt(0) + io.aw.bits := NASTIWriteAddressChannel( + id = coreId, + addr = addrTable(coreId), + size = UInt(log2Up(scrDataBytes))) io.w.valid := sending_data - io.w.bits.data := rtc - io.w.bits.strb := Bits(0x00FF) - io.w.bits.user := UInt(0) - io.w.bits.last := Bool(true) + io.w.bits := NASTIWriteDataChannel(data = rtc) io.b.ready := Bool(true) io.ar.valid := Bool(false) io.r.ready := Bool(false) - assert(!rtc_tick || send_acked.toBits.andR, + assert(!rtc_tick || send_acked.reduce(_ && _), s"Not all clocks were updated for rtc tick") } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 859a115f..47e5f5fd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1439,8 +1439,8 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { len = Mux(has_data, UInt(tlDataBeats - 1), UInt(0))) io.nasti.aw.bits := io.nasti.ar.bits io.nasti.w.bits := NASTIWriteDataChannel( - strb = io.tl.acquire.bits.wmask(), data = io.tl.acquire.bits.data, + strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) when(!active_out){ From 2179cb64ae4aa632284a5cd4b71e656b84ff1f0d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 25 Sep 2015 15:27:20 -0700 Subject: [PATCH 441/688] Let isRead be true for store-conditional This works around a deadlock bug in the L1 D$, and is arguably true. --- uncore/src/main/scala/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index 56d02f68..ed6db122 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -43,7 +43,7 @@ trait MemoryOpConstants { def isAMO(cmd: UInt) = cmd(3) || cmd === M_XA_SWAP def isPrefetch(cmd: UInt) = cmd === M_PFR || cmd === M_PFW - def isRead(cmd: UInt) = cmd === M_XRD || cmd === M_XLR || isAMO(cmd) + def isRead(cmd: UInt) = cmd === M_XRD || cmd === M_XLR || cmd === M_XSC || isAMO(cmd) def isWrite(cmd: UInt) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: UInt) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } From 20b7a82ab68ee20a6212a2a6b02c5a2388e1fa22 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 25 Sep 2015 17:06:24 -0700 Subject: [PATCH 442/688] Use Vec.fill, not Vec.apply, when making Vec literals --- uncore/src/main/scala/rtc.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index ade83749..2a32fe88 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -17,12 +17,12 @@ class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { val sending_addr = Reg(init = Bool(false)) val sending_data = Reg(init = Bool(false)) - val send_acked = Reg(init = Vec(nCores, Bool(true))) + val send_acked = Reg(init = Vec.fill(nCores)(Bool(true))) val coreId = Wire(UInt(width = log2Up(nCores))) when (rtc_tick) { rtc := rtc + UInt(1) - send_acked := Vec(nCores, Bool(false)) + send_acked := Vec.fill(nCores)(Bool(false)) sending_addr := Bool(true) sending_data := Bool(true) } diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 47e5f5fd..f6dbc54d 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1263,7 +1263,7 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val roq_data = Reg(Vec(dType.cloneType, size)) val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) - val roq_free = Reg(init = Vec(size, Bool(true))) + val roq_free = Reg(init = Vec.fill(size)(Bool(true))) val roq_enq_addr = PriorityEncoder(roq_free) val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) From 3b1da4c57e3bc89099016caa695c4f08f3104fcb Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 25 Sep 2015 17:06:06 -0700 Subject: [PATCH 443/688] Revert "replace remaining uses of Vec.fill" This reverts commit b6bb4e42127d1ed42b55ec8b859a4e074b347d47. --- uncore/src/main/scala/cache.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6884165b..ff0d7739 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -157,7 +157,7 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { val metabits = rstVal.getWidth val tag_arr = SeqMem(Vec(UInt(width = metabits), nWays), nSets) when (rst || io.write.valid) { - tag_arr.write(waddr, Vec(nWays, wdata), wmask) + tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) } val tags = tag_arr.read(io.read.bits.idx, io.read.valid).toBits @@ -493,7 +493,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -588,8 +588,8 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { // State holding transaction metadata val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) - val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) - val wmask_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits/8))) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -981,7 +981,7 @@ class L2WritebackUnit(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) val xact = Reg(new L2WritebackReq) - val data_buffer = Reg(init=Vec(innerDataBeats, UInt(0, innerDataBits))) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val xact_addr_block = Cat(xact.tag, xact.idx) val pending_irels = From 1e7f65652742798f64429615d9adc1befce2957e Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 28 Sep 2015 15:02:51 -0700 Subject: [PATCH 444/688] get release block address from inner release --- uncore/src/main/scala/broadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 56d7c814..e6587ce3 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -257,7 +257,7 @@ class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { val oacq_probe = PutBlock( client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, + addr_block = io.irel().addr_block, addr_beat = io.irel().addr_beat, data = io.irel().data) From 0fe16ac1c01f7499fd7cf67bb236394df76bbab8 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 30 Sep 2015 14:36:49 -0700 Subject: [PATCH 445/688] Chisel3 compatibility fixes --- uncore/src/main/scala/htif.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 20bd317f..de6e0c33 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -86,7 +86,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val rx_word_count = (rx_count >> log2Up(short_request_bits/w)) val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR val packet_ram_depth = long_request_bits/short_request_bits-1 - val packet_ram = Mem(Bits(width = short_request_bits), packet_ram_depth) + val packet_ram = Mem(packet_ram_depth, Bits(width = short_request_bits)) when (rx_word_done && io.host.in.ready) { packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in } From 993ed86198391efec3393a35e7756c673e4999be Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 13 Oct 2015 09:49:22 -0700 Subject: [PATCH 446/688] move ReorderQueue to utils.scala --- uncore/src/main/scala/tilelink.scala | 41 ---------------------------- uncore/src/main/scala/util.scala | 41 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index f6dbc54d..36520946 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1241,47 +1241,6 @@ trait HasDataBeatCounters { } } -class ReorderQueueWrite[T <: Data](dType: T, tagWidth: Int) extends Bundle { - val data = dType.cloneType - val tag = UInt(width = tagWidth) - - override def cloneType = - new ReorderQueueWrite(dType, tagWidth).asInstanceOf[this.type] -} - -class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) - extends Module { - val io = new Bundle { - val enq = Decoupled(new ReorderQueueWrite(dType, tagWidth)).flip - val deq = new Bundle { - val valid = Bool(INPUT) - val tag = UInt(INPUT, tagWidth) - val data = dType.cloneType.asOutput - } - val full = Bool(OUTPUT) - } - - val roq_data = Reg(Vec(dType.cloneType, size)) - val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) - val roq_free = Reg(init = Vec.fill(size)(Bool(true))) - - val roq_enq_addr = PriorityEncoder(roq_free) - val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) - - io.enq.ready := roq_free.reduce(_ || _) - io.deq.data := roq_data(roq_deq_addr) - - when (io.enq.valid && io.enq.ready) { - roq_data(roq_enq_addr) := io.enq.bits.data - roq_tags(roq_enq_addr) := io.enq.bits.tag - roq_free(roq_enq_addr) := Bool(false) - } - - when (io.deq.valid) { - roq_free(roq_deq_addr) := Bool(true) - } -} - class ClientTileLinkIOUnwrapperInfo extends Bundle { val voluntary = Bool() val builtin = Bool() diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 8fb21f2e..ab4f9947 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -104,3 +104,44 @@ object FlowThroughSerializer { fs.io.out } } + +class ReorderQueueWrite[T <: Data](dType: T, tagWidth: Int) extends Bundle { + val data = dType.cloneType + val tag = UInt(width = tagWidth) + + override def cloneType = + new ReorderQueueWrite(dType, tagWidth).asInstanceOf[this.type] +} + +class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) + extends Module { + val io = new Bundle { + val enq = Decoupled(new ReorderQueueWrite(dType, tagWidth)).flip + val deq = new Bundle { + val valid = Bool(INPUT) + val tag = UInt(INPUT, tagWidth) + val data = dType.cloneType.asOutput + } + val full = Bool(OUTPUT) + } + + val roq_data = Reg(Vec(dType.cloneType, size)) + val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) + val roq_free = Reg(init = Vec.fill(size)(Bool(true))) + + val roq_enq_addr = PriorityEncoder(roq_free) + val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) + + io.enq.ready := roq_free.reduce(_ || _) + io.deq.data := roq_data(roq_deq_addr) + + when (io.enq.valid && io.enq.ready) { + roq_data(roq_enq_addr) := io.enq.bits.data + roq_tags(roq_enq_addr) := io.enq.bits.tag + roq_free(roq_enq_addr) := Bool(false) + } + + when (io.deq.valid) { + roq_free(roq_deq_addr) := Bool(true) + } +} From 83df05cb6a65185cef6e256856a41f659dde8c9d Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 13 Oct 2015 12:45:39 -0700 Subject: [PATCH 447/688] add TileLink data narrower --- uncore/src/main/scala/tilelink.scala | 99 ++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 36520946..093e913a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1474,3 +1474,102 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { addr_beat = UInt(0), data = Bits(0)) } + +class TileLinkIONarrower(factor: Int) extends TLModule { + val outerParams = params.alterPartial({ + case TLDataBeats => tlDataBeats * factor + }) + val outerDataBeats = outerParams(TLDataBeats) + val outerDataBits = outerParams(TLDataBits) + val outerWriteMaskBits = outerParams(TLWriteMaskBits) + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = Bundle(new ClientUncachedTileLinkIO)(outerParams) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType + + val acq_data_buffer = Reg(UInt(width = tlDataBits)) + val acq_wmask_buffer = Reg(UInt(width = tlWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate()) + + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0)) + + val sending_put = Reg(init = Bool(false)) + + io.out.acquire.bits := MuxBundle(iacq, Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire))) + io.out.acquire.valid := sending_put || (io.in.acquire.valid && !stretch) + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) + } + + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(tlDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + io.in.grant.bits := Mux(sending_get, get_block_grant, ognt) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) + } + } + + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } +} From 47da284e56a45c843f249685f42c120694aafd0b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 13 Oct 2015 13:28:47 -0700 Subject: [PATCH 448/688] TileLinkNarrower should do nothing if interfaces are the same width --- uncore/src/main/scala/tilelink.scala | 152 ++++++++++++++------------- 1 file changed, 77 insertions(+), 75 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 093e913a..dac27317 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1488,88 +1488,90 @@ class TileLinkIONarrower(factor: Int) extends TLModule { val out = Bundle(new ClientUncachedTileLinkIO)(outerParams) } - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits + if (factor > 1) { + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits - val stretch = iacq.a_type === Acquire.putBlockType - val shrink = iacq.a_type === Acquire.getBlockType + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType - val acq_data_buffer = Reg(UInt(width = tlDataBits)) - val acq_wmask_buffer = Reg(UInt(width = tlWriteMaskBits)) - val acq_client_id = Reg(iacq.client_xact_id) - val acq_addr_block = Reg(iacq.addr_block) - val acq_addr_beat = Reg(iacq.addr_beat) - val oacq_ctr = Counter(factor) + val acq_data_buffer = Reg(UInt(width = tlDataBits)) + val acq_wmask_buffer = Reg(UInt(width = tlWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - alloc = iacq.allocate()) + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate()) - val put_block_acquire = PutBlock( - client_xact_id = acq_client_id, - addr_block = acq_addr_block, - addr_beat = if (factor > 1) - Cat(acq_addr_beat, oacq_ctr.value) - else acq_addr_beat, - data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0)) + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0)) - val sending_put = Reg(init = Bool(false)) + val sending_put = Reg(init = Bool(false)) - io.out.acquire.bits := MuxBundle(iacq, Seq( - (sending_put, put_block_acquire), - (shrink, get_block_acquire))) - io.out.acquire.valid := sending_put || (io.in.acquire.valid && !stretch) - io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + io.out.acquire.bits := MuxBundle(iacq, Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire))) + io.out.acquire.valid := sending_put || (io.in.acquire.valid && !stretch) + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) - when (io.in.acquire.fire() && stretch) { - acq_data_buffer := iacq.data - acq_wmask_buffer := iacq.wmask() - acq_client_id := iacq.client_xact_id - acq_addr_block := iacq.addr_block - acq_addr_beat := iacq.addr_beat - sending_put := Bool(true) - } - - when (sending_put && io.out.acquire.ready) { - acq_data_buffer := acq_data_buffer >> outerDataBits - acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits - when (oacq_ctr.inc()) { sending_put := Bool(false) } - } - - val ognt_block = ognt.hasMultibeatData() - val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) - val gnt_client_id = Reg(ognt.client_xact_id) - val gnt_manager_id = Reg(ognt.manager_xact_id) - - val ignt_ctr = Counter(tlDataBeats) - val ognt_ctr = Counter(factor) - val sending_get = Reg(init = Bool(false)) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits) - - io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) - io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - io.in.grant.bits := Mux(sending_get, get_block_grant, ognt) - - when (io.out.grant.valid && ognt_block && !sending_get) { - gnt_data_buffer(ognt_ctr.value) := ognt.data - when (ognt_ctr.inc()) { - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - sending_get := Bool(true) + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) } - } - when (io.in.grant.ready && sending_get) { - ignt_ctr.inc() - sending_get := Bool(false) - } + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(tlDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + io.in.grant.bits := Mux(sending_get, get_block_grant, ognt) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) + } + } + + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } + } else { io.out <> io.in } } From 908922c1a43f1af4f5da6fea6cff750cf0cdc5a5 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 2 Oct 2015 14:20:31 -0700 Subject: [PATCH 449/688] refactor NASTI to not use param --- uncore/src/main/scala/rtc.scala | 10 +++++----- uncore/src/main/scala/tilelink.scala | 16 ++++++++-------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 2a32fe88..8cfaec7e 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -3,10 +3,10 @@ package uncore import Chisel._ import junctions._ -class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { - val io = new NASTIIO +class RTC(pcr_MTIME: Int)(implicit val p: Parameters) extends Module with HTIFParameters { + val io = new NastiIO - private val addrMap = params(NASTIAddrHashMap) + private val addrMap = new AddrHashMap(params(NastiAddrMap)) val addrTable = Vec.tabulate(nCores) { i => UInt(addrMap(s"conf:csr$i").start + pcr_MTIME * scrDataBytes) @@ -45,13 +45,13 @@ class RTC(pcr_MTIME: Int) extends Module with HTIFParameters { when (io.b.fire()) { send_acked(io.b.bits.id) := Bool(true) } io.aw.valid := sending_addr - io.aw.bits := NASTIWriteAddressChannel( + io.aw.bits := NastiWriteAddressChannel( id = coreId, addr = addrTable(coreId), size = UInt(log2Up(scrDataBytes))) io.w.valid := sending_data - io.w.bits := NASTIWriteDataChannel(data = rtc) + io.w.bits := NastiWriteDataChannel(data = rtc) io.b.ready := Bool(true) io.ar.valid := Bool(false) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index dac27317..ac6f5c23 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1328,15 +1328,15 @@ class ClientTileLinkIOUnwrapper extends TLModule { io.in.probe.valid := Bool(false) } -class NASTIIOTileLinkIOConverterInfo extends TLBundle { +class NastiIOTileLinkIOConverterInfo extends TLBundle { val byteOff = UInt(width = tlByteAddrBits) val subblock = Bool() } -class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { +class NastiIOTileLinkIOConverter(implicit val p: Parameters) extends TLModule with HasNastiParameters { val io = new Bundle { val tl = new ClientUncachedTileLinkIO().flip - val nasti = new NASTIIO + val nasti = new NastiIO } private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( @@ -1352,7 +1352,7 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { val dstIdBits = params(LNHeaderBits) require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NASTIIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) + require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NastiIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) io.tl.acquire.ready := Bool(false) @@ -1366,7 +1366,7 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { val acq_has_data = io.tl.acquire.bits.hasData() val is_write = io.tl.acquire.valid && acq_has_data - // Decompose outgoing TL Acquires into NASTI address and data channels + // Decompose outgoing TL Acquires into Nasti address and data channels val active_out = Reg(init=Bool(false)) val cmd_sent_out = Reg(init=Bool(false)) val tag_out = Reg(UInt(width = nastiXIdBits)) @@ -1378,7 +1378,7 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { val tl_done_out = Reg(init=Bool(false)) val roq = Module(new ReorderQueue( - new NASTIIOTileLinkIOConverterInfo, + new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxClientsPerPort)) val (nasti_cnt_out, nasti_wrap_out) = Counter( @@ -1391,13 +1391,13 @@ class NASTIIOTileLinkIOConverter extends TLModule with NASTIParameters { roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) roq.io.deq.tag := io.nasti.r.bits.id - io.nasti.ar.bits := NASTIReadAddressChannel( + io.nasti.ar.bits := NastiReadAddressChannel( id = tag_out, addr = addr_out, size = UInt(log2Ceil(tlDataBytes)), len = Mux(has_data, UInt(tlDataBeats - 1), UInt(0))) io.nasti.aw.bits := io.nasti.ar.bits - io.nasti.w.bits := NASTIWriteDataChannel( + io.nasti.w.bits := NastiWriteDataChannel( data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) From 31be6407ecfbdab1f39062ecb7c6df6774a10476 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 5 Oct 2015 21:41:46 -0700 Subject: [PATCH 450/688] Removed all traces of params --- uncore/src/main/scala/broadcast.scala | 33 +-- uncore/src/main/scala/cache.scala | 161 ++++++------ uncore/src/main/scala/coherence.scala | 82 +++---- uncore/src/main/scala/htif.scala | 101 ++++---- uncore/src/main/scala/metadata.scala | 58 ++--- uncore/src/main/scala/network.scala | 26 +- uncore/src/main/scala/rtc.scala | 12 +- uncore/src/main/scala/scr.scala | 14 +- uncore/src/main/scala/tilelink.scala | 340 ++++++++++++++------------ uncore/src/main/scala/uncore.scala | 50 ++-- uncore/src/main/scala/util.scala | 4 +- 11 files changed, 456 insertions(+), 425 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index e6587ce3..c48606c3 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -5,19 +5,20 @@ import Chisel._ case object L2StoreDataQueueDepth extends Field[Int] -trait BroadcastHubParameters extends CoherenceAgentParameters { - val sdqDepth = params(L2StoreDataQueueDepth)*innerDataBeats +trait HasBroadcastHubParameters extends HasCoherenceAgentParameters { + val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases } -class DataQueueLocation extends Bundle with BroadcastHubParameters { +class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p) + with HasBroadcastHubParameters { val idx = UInt(width = dqIdxBits) val loc = UInt(width = log2Ceil(nDataQueueLocations)) } object DataQueueLocation { - def apply(idx: UInt, loc: UInt) = { + def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = { val d = Wire(new DataQueueLocation) d.idx := idx d.loc := loc @@ -25,12 +26,12 @@ object DataQueueLocation { } } -class L2BroadcastHub extends ManagerCoherenceAgent - with BroadcastHubParameters { +class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) + with HasBroadcastHubParameters { val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - val trackerTLParams = params.alterPartial({ + val trackerTLParams = p.alterPartial({ case TLDataBits => internalDataBits case TLWriteMaskBits => innerWriteMaskBits }) @@ -104,10 +105,10 @@ class L2BroadcastHub extends ManagerCoherenceAgent doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size), - { case TLId => params(OuterTLId) + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size)(p.alterPartial( + { case TLId => p(OuterTLId) case TLDataBits => internalDataBits - case TLWriteMaskBits => innerWriteMaskBits }) + case TLWriteMaskBits => innerWriteMaskBits }))) outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) @@ -127,15 +128,16 @@ class L2BroadcastHub extends ManagerCoherenceAgent } } -class BroadcastXactTracker extends XactTracker { +class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { val io = new ManagerXactTrackerIO } -class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTracker { +class BroadcastVoluntaryReleaseTracker(trackerId: Int) + (implicit p: Parameters) extends BroadcastXactTracker()(p) { val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => p(InnerTLId); case TLDataBits => 0 })) val data_buffer = Reg(Vec(io.irel().data, innerDataBeats)) val coh = ManagerMetadata.onReset @@ -210,12 +212,13 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) extends BroadcastXactTrac } } -class BroadcastAcquireTracker(trackerId: Int) extends BroadcastXactTracker { +class BroadcastAcquireTracker(trackerId: Int) + (implicit p: Parameters) extends BroadcastXactTracker()(p) { val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) val xact = Reg(Bundle(new AcquireFromSrc, { - case TLId => params(InnerTLId) + case TLId => p(InnerTLId) case TLDataBits => 0 case TLWriteMaskBits => innerWriteMaskBits })) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ff0d7739..b77ecc10 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -18,23 +18,26 @@ case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] -abstract trait CacheParameters extends UsesParameters { - val nSets = params(NSets) - val blockOffBits = params(CacheBlockOffsetBits) +trait HasCacheParameters { + implicit val p: Parameters + val nSets = p(NSets) + val blockOffBits = p(CacheBlockOffsetBits) val idxBits = log2Up(nSets) val untagBits = blockOffBits + idxBits - val tagBits = params(PAddrBits) - untagBits - val nWays = params(NWays) + val tagBits = p(PAddrBits) - untagBits + val nWays = p(NWays) val wayBits = log2Up(nWays) val isDM = nWays == 1 - val rowBits = params(RowBits) + val rowBits = p(RowBits) val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) - val code = params(ECCCode).getOrElse(new IdentityCode) + val code = p(ECCCode).getOrElse(new IdentityCode) } -abstract class CacheBundle extends Bundle with CacheParameters -abstract class CacheModule extends Module with CacheParameters +abstract class CacheModule(implicit val p: Parameters) extends Module + with HasCacheParameters +abstract class CacheBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) + with HasCacheParameters class StoreGen(typ: UInt, addr: UInt, dat: UInt) { val byte = typ === MT_B || typ === MT_BU @@ -66,8 +69,8 @@ class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool) { val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) } -class AMOALU extends CacheModule { - val operandBits = params(AmoAluOperandBits) +class AMOALU(implicit p: Parameters) extends CacheModule()(p) { + val operandBits = p(AmoAluOperandBits) require(operandBits == 64) val io = new Bundle { val addr = Bits(INPUT, blockOffBits) @@ -125,23 +128,23 @@ class RandomReplacement(ways: Int) extends ReplacementPolicy { def hit = {} } -abstract class Metadata extends CacheBundle { +abstract class Metadata(implicit p: Parameters) extends CacheBundle()(p) { val tag = Bits(width = tagBits) val coh: CoherenceMetadata } -class MetaReadReq extends CacheBundle { +class MetaReadReq(implicit p: Parameters) extends CacheBundle()(p) { val idx = Bits(width = idxBits) } -class MetaWriteReq[T <: Metadata](gen: T) extends MetaReadReq { +class MetaWriteReq[T <: Metadata](gen: T)(implicit p: Parameters) extends MetaReadReq()(p) { val way_en = Bits(width = nWays) val data = gen.cloneType - override def cloneType = new MetaWriteReq(gen).asInstanceOf[this.type] + override def cloneType = new MetaWriteReq(gen)(p).asInstanceOf[this.type] } -class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { - val rstVal = makeRstVal() +class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) extends CacheModule()(p) { + val rstVal = onReset() val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal)).flip @@ -166,24 +169,24 @@ class MetadataArray[T <: Metadata](makeRstVal: () => T) extends CacheModule { io.write.ready := !rst } -abstract trait L2HellaCacheParameters extends CacheParameters with CoherenceAgentParameters { +trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val blockAddrBits = params(TLBlockAddrBits) + val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats - val internalDataBeats = params(CacheBlockBytes)*8/rowBits + val internalDataBeats = p(CacheBlockBytes)*8/rowBits require(refillCyclesPerBeat == 1) - val amoAluOperandBits = params(AmoAluOperandBits) + val amoAluOperandBits = p(AmoAluOperandBits) require(amoAluOperandBits <= innerDataBits) require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states - val nSecondaryMisses = params(NSecondaryMisses) + val nSecondaryMisses = p(NSecondaryMisses) val isLastLevelCache = true - val ignoresWriteMask = !params(ECCCode).isEmpty + val ignoresWriteMask = !p(ECCCode).isEmpty } -abstract class L2HellaCacheBundle extends Bundle with L2HellaCacheParameters -abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { +abstract class L2HellaCacheModule(implicit val p: Parameters) extends Module + with HasL2HellaCacheParameters { def doInternalOutputArbitration[T <: Data : ClassTag]( out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { @@ -192,39 +195,42 @@ abstract class L2HellaCacheModule extends Module with L2HellaCacheParameters { arb.io.in <> ins } - def doInternalInputRouting[T <: HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { + def doInternalInputRouting[T <: Bundle with HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { outs.map(_.bits := in.bits) outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && in.bits.id === UInt(i) } } } -trait HasL2Id extends Bundle with CoherenceAgentParameters { +abstract class L2HellaCacheBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) + with HasL2HellaCacheParameters + +trait HasL2Id extends HasCoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) } -trait HasL2InternalRequestState extends L2HellaCacheBundle { +trait HasL2InternalRequestState extends HasL2HellaCacheParameters { val tag_match = Bool() val meta = new L2Metadata val way_en = Bits(width = nWays) } -trait HasL2BeatAddr extends L2HellaCacheBundle { +trait HasL2BeatAddr extends HasL2HellaCacheParameters { val addr_beat = UInt(width = log2Up(refillCycles)) } -trait HasL2Data extends L2HellaCacheBundle +trait HasL2Data extends HasL2HellaCacheParameters with HasL2BeatAddr { val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) } -class L2Metadata extends Metadata with L2HellaCacheParameters { +class L2Metadata(implicit p: Parameters) extends Metadata()(p) with HasL2HellaCacheParameters { val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: HierarchicalMetadata) = { + def apply(tag: Bits, coh: HierarchicalMetadata)(implicit p: Parameters) = { val meta = Wire(new L2Metadata) meta.tag := tag meta.coh := coh @@ -232,31 +238,33 @@ object L2Metadata { } } -class L2MetaReadReq extends MetaReadReq with HasL2Id { +class L2MetaReadReq(implicit p: Parameters) extends MetaReadReq()(p) with HasL2Id { val tag = Bits(width = tagBits) } -class L2MetaWriteReq extends MetaWriteReq[L2Metadata](new L2Metadata) +class L2MetaWriteReq(implicit p: Parameters) extends MetaWriteReq[L2Metadata](new L2Metadata)(p) with HasL2Id { override def cloneType = new L2MetaWriteReq().asInstanceOf[this.type] } -class L2MetaResp extends L2HellaCacheBundle +class L2MetaResp(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2Id with HasL2InternalRequestState -trait HasL2MetaReadIO extends L2HellaCacheBundle { +trait HasL2MetaReadIO extends HasL2HellaCacheParameters { val read = Decoupled(new L2MetaReadReq) val resp = Valid(new L2MetaResp).flip } -trait HasL2MetaWriteIO extends L2HellaCacheBundle { +trait HasL2MetaWriteIO extends HasL2HellaCacheParameters { val write = Decoupled(new L2MetaWriteReq) } -class L2MetaRWIO extends L2HellaCacheBundle with HasL2MetaReadIO with HasL2MetaWriteIO +class L2MetaRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) + with HasL2MetaReadIO + with HasL2MetaWriteIO -class L2MetadataArray extends L2HellaCacheModule { +class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val io = new L2MetaRWIO().flip def onReset = L2Metadata(UInt(0), HierarchicalMetadata.onReset) @@ -274,7 +282,7 @@ class L2MetadataArray extends L2HellaCacheModule { val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - val replacer = params(Replacer)() + val replacer = p(Replacer)() val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => @@ -290,32 +298,36 @@ class L2MetadataArray extends L2HellaCacheModule { io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } -class L2DataReadReq extends L2HellaCacheBundle +class L2DataReadReq(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2BeatAddr with HasL2Id { val addr_idx = UInt(width = idxBits) val way_en = Bits(width = nWays) } -class L2DataWriteReq extends L2DataReadReq +class L2DataWriteReq(implicit p: Parameters) extends L2DataReadReq()(p) with HasL2Data { val wmask = Bits(width = rowBits/8) } -class L2DataResp extends L2HellaCacheBundle with HasL2Id with HasL2Data +class L2DataResp(implicit p: Parameters) extends L2HellaCacheBundle()(p) + with HasL2Id + with HasL2Data -trait HasL2DataReadIO extends L2HellaCacheBundle { +trait HasL2DataReadIO extends HasL2HellaCacheParameters { val read = Decoupled(new L2DataReadReq) val resp = Valid(new L2DataResp).flip } -trait HasL2DataWriteIO extends L2HellaCacheBundle { +trait HasL2DataWriteIO extends HasL2HellaCacheParameters { val write = Decoupled(new L2DataWriteReq) } -class L2DataRWIO extends L2HellaCacheBundle with HasL2DataReadIO with HasL2DataWriteIO +class L2DataRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) + with HasL2DataReadIO + with HasL2DataWriteIO -class L2DataArray(delay: Int) extends L2HellaCacheModule { +class L2DataArray(delay: Int)(implicit p: Parameters) extends L2HellaCacheModule()(p) { val io = new L2DataRWIO().flip val array = SeqMem(Vec(Bits(width=8), rowBits/8), nWays*nSets*refillCycles) @@ -333,7 +345,8 @@ class L2DataArray(delay: Int) extends L2HellaCacheModule { io.write.ready := Bool(true) } -class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParameters { +class L2HellaCacheBank(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) + with HasL2HellaCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) @@ -347,12 +360,13 @@ class L2HellaCacheBank extends HierarchicalCoherenceAgent with L2HellaCacheParam data.io <> tshrfile.io.data } -class TSHRFileIO extends HierarchicalTLIO { +class TSHRFileIO(implicit p: Parameters) extends HierarchicalTLIO()(p) { val meta = new L2MetaRWIO val data = new L2DataRWIO } -class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { +class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) + with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions @@ -418,15 +432,16 @@ class TSHRFile extends L2HellaCacheModule with HasCoherenceAgentWiringHelpers { } -class L2XactTrackerIO extends HierarchicalXactTrackerIO { +class L2XactTrackerIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) { val data = new L2DataRWIO val meta = new L2MetaRWIO val wb = new L2WritebackIO } -abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { +abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) + with HasL2HellaCacheParameters { class CacheBlockBuffer { // TODO - val buffer = Reg(Bits(width = params(CacheBlockBytes)*8)) + val buffer = Reg(Bits(width = p(CacheBlockBytes)*8)) def internal = Vec(Bits(width = rowBits), internalDataBeats).fromBits(buffer) def inner = Vec(Bits(width = innerDataBits), innerDataBeats).fromBits(buffer) @@ -440,29 +455,29 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { } else { (UInt(0), inc) } } - def connectInternalDataBeatCounter[T <: HasL2BeatAddr]( + def connectInternalDataBeatCounter[T <: L2HellaCacheBundle with HasL2BeatAddr]( in: DecoupledIO[T], beat: UInt = UInt(0), full_block: Bool = Bool(true)): (UInt, Bool) = { connectDataBeatCounter(in.fire(), in.bits, beat, full_block) } - def connectInternalDataBeatCounter[T <: HasL2Data]( + def connectInternalDataBeatCounter[T <: L2HellaCacheBundle with HasL2Data]( in: ValidIO[T], full_block: Bool): Bool = { connectDataBeatCounter(in.valid, in.bits, UInt(0), full_block)._2 } - def addPendingBitInternal[T <: HasL2BeatAddr](in: DecoupledIO[T]) = + def addPendingBitInternal[T <: L2HellaCacheBundle with HasL2BeatAddr](in: DecoupledIO[T]) = Fill(in.bits.refillCycles, in.fire()) & UIntToOH(in.bits.addr_beat) - def addPendingBitInternal[T <: HasL2BeatAddr](in: ValidIO[T]) = + def addPendingBitInternal[T <: L2HellaCacheBundle with HasL2BeatAddr](in: ValidIO[T]) = Fill(in.bits.refillCycles, in.valid) & UIntToOH(in.bits.addr_beat) - def dropPendingBit[T <: HasL2BeatAddr] (in: DecoupledIO[T]) = + def dropPendingBit[T <: L2HellaCacheBundle with HasL2BeatAddr] (in: DecoupledIO[T]) = ~Fill(in.bits.refillCycles, in.fire()) | ~UIntToOH(in.bits.addr_beat) - def dropPendingBitInternal[T <: HasL2BeatAddr] (in: ValidIO[T]) = + def dropPendingBitInternal[T <: L2HellaCacheBundle with HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { @@ -474,10 +489,10 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { def pinAllReadyValidLow[T <: Data](b: Bundle) { b.elements.foreach { _._2 match { - case d: DecoupledIO[T] => + case d: DecoupledIO[_] => if(d.ready.dir == OUTPUT) d.ready := Bool(false) else if(d.valid.dir == OUTPUT) d.valid := Bool(false) - case v: ValidIO[T] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) case b: Bundle => pinAllReadyValidLow(b) case _ => } @@ -485,14 +500,14 @@ abstract class L2XactTracker extends XactTracker with L2HellaCacheParameters { } } -class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { +class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { val io = new L2XactTrackerIO pinAllReadyValidLow(io) val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => params(InnerTLId); case TLDataBits => 0 })) + val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => p(InnerTLId); case TLDataBits => 0 })) val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } @@ -579,7 +594,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int) extends L2XactTracker { } -class L2AcquireTracker(trackerId: Int) extends L2XactTracker { +class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { val io = new L2XactTrackerIO pinAllReadyValidLow(io) @@ -587,7 +602,7 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { val state = Reg(init=s_idle) // State holding transaction metadata - val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => params(InnerTLId) })) + val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => p(InnerTLId) })) val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) val xact_tag_match = Reg{ Bool() } @@ -667,15 +682,15 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { wmask_buffer(beat) := ~UInt(0, wmask_buffer.head.getWidth) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } - def mergeDataInternal[T <: HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + def mergeDataInternal[T <: L2HellaCacheBundle with HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } - def mergeDataInner[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + def mergeDataInner[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { when(in.fire() && in.bits.hasData()) { mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) } } - def mergeDataOuter[T <: HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { + def mergeDataOuter[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { when(in.fire() && in.bits.hasData()) { mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) } @@ -956,24 +971,24 @@ class L2AcquireTracker(trackerId: Int) extends L2XactTracker { "AcquireTracker accepted data beat from different network source than initial request.") } -class L2WritebackReq extends L2Metadata with HasL2Id { +class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { val idx = Bits(width = idxBits) val way_en = Bits(width = nWays) } -class L2WritebackResp extends L2HellaCacheBundle with HasL2Id +class L2WritebackResp(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2Id -class L2WritebackIO extends L2HellaCacheBundle { +class L2WritebackIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) { val req = Decoupled(new L2WritebackReq) val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnitIO extends HierarchicalXactTrackerIO { +class L2WritebackUnitIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) { val wb = new L2WritebackIO().flip val data = new L2DataRWIO } -class L2WritebackUnit(trackerId: Int) extends L2XactTracker { +class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { val io = new L2WritebackUnitIO pinAllReadyValidLow(io) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 0c35f420..2ec15046 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -8,10 +8,10 @@ import Chisel._ * HasClientSideCoherencePolicy, for client coherence agents * HasManagerSideCoherencePolicy, for manager coherence agents */ -abstract class CoherencePolicy(val dir: DirectoryRepresentation) extends - HasCustomTileLinkMessageTypes with - HasClientSideCoherencePolicy with - HasManagerSideCoherencePolicy +abstract class CoherencePolicy(val dir: DirectoryRepresentation) + extends HasCustomTileLinkMessageTypes + with HasClientSideCoherencePolicy + with HasManagerSideCoherencePolicy /** This API defines the custom, coherence-policy-defined message types, * as opposed to the built-in ones found in tilelink.scala. @@ -73,7 +73,6 @@ trait HasClientSideCoherencePolicy { def getReleaseType(p: Probe, meta: ClientMetadata): UInt // Mutate ClientMetadata based on messages or cmds - def clientMetadataOnReset: ClientMetadata def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata @@ -101,12 +100,11 @@ trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { def getExclusiveGrantType(): UInt // Mutate ManagerMetadata based on messages or cmds - def managerMetadataOnReset: ManagerMetadata def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached meta.sharers, - dir.push(meta.sharers, dst))) + dir.push(meta.sharers, dst)))(meta.p) //state = meta.state) TODO: Fix 0-width wires in Chisel } @@ -158,19 +156,17 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def clientMetadataOnReset = ClientMetadata(clientInvalid) - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state)) + ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state))(meta.p) def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid)) + ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid))(meta.p) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata(Mux(incoming.p_type === probeInvalidate, - clientInvalid, meta.state)) + clientInvalid, meta.state))(meta.p) // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol @@ -196,10 +192,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnReset = ManagerMetadata() - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { - val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) @@ -252,28 +246,26 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def clientMetadataOnReset = ClientMetadata(clientInvalid) - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( M_FLUSH -> clientInvalid, - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state))))(meta.p) def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(incoming.isBuiltInType(), clientInvalid, - Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean))) + Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean)))(meta.p) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state))) + probeCopy -> meta.state)))(meta.p) // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol @@ -299,10 +291,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnReset = ManagerMetadata() - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { - val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) @@ -355,17 +345,15 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def clientMetadataOnReset = ClientMetadata(clientInvalid) - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( MuxLookup(cmd, meta.state, Array( M_FLUSH -> clientInvalid, M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), - clientShared, meta.state)))) + clientShared, meta.state))))(meta.p) def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( @@ -373,14 +361,14 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> clientExclusiveDirty, - grantExclusiveAck -> clientExclusiveDirty)))) + grantExclusiveAck -> clientExclusiveDirty))))(meta.p) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, - probeCopy -> meta.state))) + probeCopy -> meta.state)))(meta.p) // Manager states and functions: val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing @@ -418,10 +406,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnReset = ManagerMetadata() - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { - val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) @@ -474,10 +460,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { probeDowngrade -> getReleaseType(M_PRODUCE, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) - def clientMetadataOnReset = ClientMetadata(clientInvalid) - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = - ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state)) + ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( @@ -485,7 +469,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> clientInvalid, M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state), - M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state)))) + M_CLEAN -> Mux(meta.state === clientExclusiveDirty, + clientExclusiveClean, meta.state))))(meta.p) def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( @@ -493,14 +478,14 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.g_type, clientInvalid, Array( grantShared -> clientShared, grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), - grantExclusiveAck -> clientExclusiveDirty)))) + grantExclusiveAck -> clientExclusiveDirty))))(meta.p) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, - probeCopy -> meta.state))) + probeCopy -> meta.state)))(meta.p) // Manager states and functions: val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing @@ -538,10 +523,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnReset = ManagerMetadata() - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { - val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) @@ -605,14 +588,12 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Mux(dirty, with_data, without_data) } - def clientMetadataOnReset = ClientMetadata(clientInvalid) - def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(isWrite(cmd), MuxLookup(meta.state, clientExclusiveDirty, Array( clientExclusiveClean -> clientExclusiveDirty, clientMigratoryClean -> clientMigratoryDirty)), - meta.state)) + meta.state))(meta.p) def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata( @@ -622,7 +603,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientShared, meta.state), M_CLEAN -> MuxLookup(meta.state, meta.state, Array( clientExclusiveDirty -> clientExclusiveClean, - clientMigratoryDirty -> clientMigratoryClean))))) + clientMigratoryDirty -> clientMigratoryClean)))))(meta.p) def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = ClientMetadata( @@ -631,7 +612,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d grantShared -> clientShared, grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty, - grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean))))) + grantReadMigratory -> Mux(isWrite(cmd), + clientMigratoryDirty, clientMigratoryClean)))))(meta.p) def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = ClientMetadata( @@ -644,7 +626,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientExclusiveDirty -> clientSharedByTwo, clientSharedByTwo -> clientShared, clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid))))) + clientMigratoryDirty -> clientInvalid)))))(meta.p) // Manager states and functions: val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types @@ -681,10 +663,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnReset = ManagerMetadata() - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { - val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src)) + val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped, diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index de6e0c33..5924bf2c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -4,57 +4,58 @@ package uncore import Chisel._ import Chisel.ImplicitConversions._ -import junctions.SMIIO +import junctions._ -case object HTIFWidth extends Field[Int] -case object HTIFNSCR extends Field[Int] -case object HTIFOffsetBits extends Field[Int] -case object HTIFNCores extends Field[Int] -case object HTIFSCRDataBits extends Field[Int] +case object HtifKey extends Field[HtifParameters] -abstract trait HTIFParameters extends UsesParameters { - val dataBits = params(TLDataBits) - val dataBeats = params(TLDataBeats) - val w = params(HTIFWidth) - val nSCR = params(HTIFNSCR) - val scrAddrBits = log2Up(nSCR) - val scrDataBits = params(HTIFSCRDataBits) - val scrDataBytes = scrDataBits / 8 - val offsetBits = params(HTIFOffsetBits) - val nCores = params(HTIFNCores) +case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, nSCR: Int = 64) + +trait HasHtifParameters { + implicit val p: Parameters + lazy val external = p(HtifKey) + lazy val dataBits = p(TLDataBits) + lazy val dataBeats = p(TLDataBeats) + lazy val w = external.width + lazy val nSCR = external.nSCR + lazy val scrAddrBits = log2Up(nSCR) + lazy val scrDataBits = 64 + lazy val scrDataBytes = scrDataBits / 8 + lazy val offsetBits = external.offsetBits + lazy val nCores = external.nCores } -abstract class HTIFBundle extends Bundle with HTIFParameters +abstract class HtifModule(implicit val p: Parameters) extends Module with HasHtifParameters +abstract class HtifBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) + with HasHtifParameters -class HostIO extends HTIFBundle -{ +class HostIO(implicit p: Parameters) extends HtifBundle()(p) { val clk = Bool(OUTPUT) val clk_edge = Bool(OUTPUT) val in = Decoupled(Bits(width = w)).flip val out = Decoupled(Bits(width = w)) - val debug_stats_pcr = Bool(OUTPUT) + val debug_stats_csr = Bool(OUTPUT) } -class HTIFIO extends HTIFBundle { +class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val pcr = new SMIIO(scrDataBits, 12).flip + val csr = new SMIIO(scrDataBits, 12).flip val ipi_req = Decoupled(Bits(width = log2Up(nCores))) val ipi_rep = Decoupled(Bool()).flip - val debug_stats_pcr = Bool(OUTPUT) + val debug_stats_csr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } -class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { +class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { val io = new Bundle { val host = new HostIO - val cpu = Vec(new HTIFIO, nCores).flip + val cpu = Vec(new HtifIO, nCores).flip val mem = new ClientUncachedTileLinkIO val scr = new SMIIO(scrDataBits, scrAddrBits) } - io.host.debug_stats_pcr := io.cpu.map(_.debug_stats_pcr).reduce(_||_) + io.host.debug_stats_csr := io.cpu.map(_.debug_stats_csr).reduce(_||_) // system is 'interesting' if any tile is 'interesting' val short_request_bits = 64 @@ -93,9 +94,9 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - val pcr_addr = addr(io.cpu(0).pcr.req.bits.addr.getWidth-1, 0) - val pcr_coreid = addr(log2Up(nCores)-1+20+1,20) - val pcr_wdata = packet_ram(0) + val csr_addr = addr(io.cpu(0).csr.req.bits.addr.getWidth-1, 0) + val csr_coreid = addr(log2Up(nCores)-1+20+1,20) + val csr_wdata = packet_ram(0) val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, @@ -114,7 +115,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) - val state_rx :: state_pcr_req :: state_pcr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) + val state_rx :: state_csr_req :: state_csr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) val state = Reg(init=state_rx) val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || @@ -123,7 +124,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { when (state === state_rx && rx_done) { state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, Mux(rx_cmd === cmd_writemem, state_mem_wreq, - Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_pcr_req, + Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_csr_req, state_tx))) } when (state === state_mem_wreq) { @@ -171,17 +172,17 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { GetBlock(addr_block = init_addr)) io.mem.grant.ready := Bool(true) - val pcrReadData = Reg(Bits(width = io.cpu(0).pcr.resp.bits.getWidth)) + val csrReadData = Reg(Bits(width = io.cpu(0).csr.resp.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) val my_ipi = Reg(init=Bool(false)) val cpu = io.cpu(i) - val me = pcr_coreid === UInt(i) - cpu.pcr.req.valid := state === state_pcr_req && me && pcr_addr != UInt(pcr_RESET) - cpu.pcr.req.bits.rw := cmd === cmd_writecr - cpu.pcr.req.bits.addr := pcr_addr - cpu.pcr.req.bits.data := pcr_wdata + val me = csr_coreid === UInt(i) + cpu.csr.req.valid := state === state_csr_req && me && csr_addr != UInt(csr_RESET) + cpu.csr.req.bits.rw := cmd === cmd_writecr + cpu.csr.req.bits.addr := csr_addr + cpu.csr.req.bits.data := csr_wdata cpu.reset := my_reset when (cpu.ipi_rep.ready) { @@ -195,32 +196,32 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { } } - when (cpu.pcr.req.fire()) { state := state_pcr_resp } + when (cpu.csr.req.fire()) { state := state_csr_resp } - when (state === state_pcr_req && me && pcr_addr === UInt(pcr_RESET)) { + when (state === state_csr_req && me && csr_addr === UInt(csr_RESET)) { when (cmd === cmd_writecr) { - my_reset := pcr_wdata(0) + my_reset := csr_wdata(0) } - pcrReadData := my_reset.toBits + csrReadData := my_reset.toBits state := state_tx } - cpu.pcr.resp.ready := Bool(true) - when (state === state_pcr_resp && cpu.pcr.resp.valid) { - pcrReadData := cpu.pcr.resp.bits + cpu.csr.resp.ready := Bool(true) + when (state === state_csr_resp && cpu.csr.resp.valid) { + csrReadData := cpu.csr.resp.bits state := state_tx } } - io.scr.req.valid := (state === state_pcr_req && pcr_coreid.andR) + io.scr.req.valid := (state === state_csr_req && csr_coreid.andR) io.scr.req.bits.addr := addr(scrAddrBits - 1, 0).toUInt - io.scr.req.bits.data := pcr_wdata + io.scr.req.bits.data := csr_wdata io.scr.req.bits.rw := (cmd === cmd_writecr) io.scr.resp.ready := Bool(true) - when (io.scr.req.fire()) { state := state_pcr_resp } - when (state === state_pcr_resp && io.scr.resp.valid) { - pcrReadData := io.scr.resp.bits + when (io.scr.req.fire()) { state := state_csr_resp } + when (state === state_csr_resp && io.scr.resp.valid) { + csrReadData := io.scr.resp.bits state := state_tx } @@ -228,7 +229,7 @@ class HTIF(pcr_RESET: Int) extends Module with HTIFParameters { val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) val tx_data = Mux(tx_word_count === UInt(0), tx_header, - Mux(cmd === cmd_readcr || cmd === cmd_writecr, pcrReadData, + Mux(cmd === cmd_readcr || cmd === cmd_writecr, csrReadData, packet_ram(packet_ram_raddr))) io.host.in.ready := state === state_rx diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 68b2981c..f03dfad7 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -4,9 +4,8 @@ package uncore import Chisel._ /** Base class to represent coherence information in clients and managers */ -abstract class CoherenceMetadata extends Bundle { - val co = params(TLCoherencePolicy) - val id = params(TLId) +abstract class CoherenceMetadata(implicit p: Parameters) extends TLBundle()(p) { + val co = tlCoh } /** Stores the client-side coherence information, @@ -14,7 +13,7 @@ abstract class CoherenceMetadata extends Bundle { * Its API can be used to make TileLink messages in response to * memory operations or [[uncore.Probe]] messages. */ -class ClientMetadata extends CoherenceMetadata { +class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { /** Actual state information stored in this bundle */ val state = UInt(width = co.clientStateWidth) @@ -53,8 +52,7 @@ class ClientMetadata extends CoherenceMetadata { a_type = co.getAcquireType(op_code, this), client_xact_id = client_xact_id, addr_block = addr_block, - union = Cat(op_code, Bool(true))), - { case TLId => id }) + union = Cat(op_code, Bool(true)))(p)) } /** Constructs a Release message based on this metadata on cache control op @@ -76,7 +74,7 @@ class ClientMetadata extends CoherenceMetadata { client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - data = data), { case TLId => id }) + data = data)(p)) } /** Constructs a Release message based on this metadata on an eviction @@ -114,7 +112,7 @@ class ClientMetadata extends CoherenceMetadata { client_xact_id = UInt(0), addr_block = prb.addr_block, addr_beat = addr_beat, - data = data), { case TLId => id }) + data = data)(p)) } /** New metadata after receiving a [[uncore.Grant]] @@ -123,38 +121,38 @@ class ClientMetadata extends CoherenceMetadata { * @param pending the mem op that triggered this transaction */ def onGrant(incoming: Grant, pending: UInt): ClientMetadata = - Bundle(co.clientMetadataOnGrant(incoming, pending, this), { case TLId => id }) + co.clientMetadataOnGrant(incoming, pending, this) /** New metadata after receiving a [[uncore.Probe]] * * @param incoming the incoming [[uncore.Probe]] */ def onProbe(incoming: Probe): ClientMetadata = - Bundle(co.clientMetadataOnProbe(incoming, this), { case TLId => id }) + co.clientMetadataOnProbe(incoming, this) /** New metadata after a op_code hits this block * * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] */ def onHit(op_code: UInt): ClientMetadata = - Bundle(co.clientMetadataOnHit(op_code, this), { case TLId => id }) + co.clientMetadataOnHit(op_code, this) /** New metadata after op_code releases permissions on this block * * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] */ def onCacheControl(op_code: UInt): ClientMetadata = - Bundle(co.clientMetadataOnCacheControl(op_code, this), { case TLId => id }) + co.clientMetadataOnCacheControl(op_code, this) } /** Factories for ClientMetadata, including on reset */ object ClientMetadata { - def apply(state: UInt) = { + def apply(state: UInt)(implicit p: Parameters) = { val meta = Wire(new ClientMetadata) meta.state := state meta } - def onReset = new ClientMetadata().co.clientMetadataOnReset + def onReset(implicit p: Parameters) = ClientMetadata(UInt(0))(p) // TODO: assumes clientInvalid === 0 } /** Stores manager-side information about the status @@ -162,7 +160,7 @@ object ClientMetadata { * * Its API can be used to create [[uncore.Probe]] and [[uncore.Grant]] messages. */ -class ManagerMetadata extends CoherenceMetadata { +class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { // Currently no coherence policies assume manager-side state information // val state = UInt(width = co.masterStateWidth) TODO: Fix 0-width wires in Chisel @@ -191,7 +189,7 @@ class ManagerMetadata extends CoherenceMetadata { * @param acq Acquire message triggering this Probe */ def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block), { case TLId => id }) + Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p)) /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op * @@ -200,7 +198,7 @@ class ManagerMetadata extends CoherenceMetadata { * @param addr_block address of the cache block being probed */ def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block), { case TLId => id }) + Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block)(p)) /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction * @@ -221,7 +219,7 @@ class ManagerMetadata extends CoherenceMetadata { is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, - manager_xact_id = manager_xact_id), { case TLId => id }) + manager_xact_id = manager_xact_id)(p)) } /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] @@ -247,7 +245,7 @@ class ManagerMetadata extends CoherenceMetadata { client_xact_id = acq.client_xact_id, manager_xact_id = manager_xact_id, addr_beat = addr_beat, - data = data), { case TLId => id }) + data = data)(p)) } /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides @@ -275,31 +273,31 @@ class ManagerMetadata extends CoherenceMetadata { * @param incoming the incoming [[uncore.ReleaseFromSrc]] */ def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = - Bundle(co.managerMetadataOnRelease(incoming, incoming.client_id, this), { case TLId => id }) + co.managerMetadataOnRelease(incoming, incoming.client_id, this) /** New metadata after sending a [[uncore.GrantToDst]] * * @param outgoing the outgoing [[uncore.GrantToDst]] */ def onGrant(outgoing: GrantToDst): ManagerMetadata = - Bundle(co.managerMetadataOnGrant(outgoing, outgoing.client_id, this), { case TLId => id }) + co.managerMetadataOnGrant(outgoing, outgoing.client_id, this) } /** Factories for ManagerMetadata, including on reset */ object ManagerMetadata { - def apply(sharers: UInt, state: UInt = UInt(width = 0)) = { + def apply(sharers: UInt, state: UInt = UInt(width = 0))(implicit p: Parameters) = { val meta = Wire(new ManagerMetadata) //meta.state := state TODO: Fix 0-width wires in Chisel meta.sharers := sharers meta } - def apply() = { + def apply(implicit p: Parameters) = { val meta = Wire(new ManagerMetadata) //meta.state := UInt(width = 0) TODO: Fix 0-width wires in Chisel meta.sharers := meta.co.dir.flush meta } - def onReset = new ManagerMetadata().co.managerMetadataOnReset + def onReset(implicit p: Parameters) = ManagerMetadata(p) } /** HierarchicalMetadata is used in a cache in a multi-level memory hierarchy @@ -310,9 +308,9 @@ object ManagerMetadata { * applied by contextually mapping [[uncore.TLId]] to one of * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. */ -class HierarchicalMetadata extends CoherenceMetadata { - val inner: ManagerMetadata = Bundle(new ManagerMetadata, {case TLId => params(InnerTLId)}) - val outer: ClientMetadata = Bundle(new ClientMetadata, {case TLId => params(OuterTLId)}) +class HierarchicalMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { + val inner: ManagerMetadata = Bundle(new ManagerMetadata()(p.alterPartial({case TLId => p(InnerTLId)}))) + val outer: ClientMetadata = Bundle(new ClientMetadata()(p.alterPartial({case TLId => p(OuterTLId)}))) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) @@ -320,13 +318,15 @@ class HierarchicalMetadata extends CoherenceMetadata { /** Factories for HierarchicalMetadata, including on reset */ object HierarchicalMetadata { - def apply(inner: ManagerMetadata, outer: ClientMetadata): HierarchicalMetadata = { + def apply(inner: ManagerMetadata, outer: ClientMetadata) + (implicit p: Parameters): HierarchicalMetadata = { val m = Wire(new HierarchicalMetadata) m.inner := inner m.outer := outer m } - def onReset: HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) + def onReset(implicit p: Parameters): HierarchicalMetadata = + apply(ManagerMetadata.onReset, ClientMetadata.onReset) } /** Identifies the TLId of the inner network in a hierarchical cache controller */ diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 16d2c1e7..928b3ee4 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -45,22 +45,23 @@ class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Opti abstract class LogicalNetwork extends Module -class LogicalHeader extends Bundle { - val src = UInt(width = params(LNHeaderBits)) - val dst = UInt(width = params(LNHeaderBits)) +class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { + val src = UInt(width = p(LNHeaderBits)) + val dst = UInt(width = p(LNHeaderBits)) } -class LogicalNetworkIO[T <: Data](dType: T) extends Bundle { +class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { val header = new LogicalHeader val payload = dType.cloneType - override def cloneType = new LogicalNetworkIO(dType).asInstanceOf[this.type] + override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] } object DecoupledLogicalNetworkIOWrapper { def apply[T <: Data]( - in: DecoupledIO[T], - src: UInt = UInt(0), - dst: UInt = UInt(0)): DecoupledIO[LogicalNetworkIO[T]] = { + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) out.valid := in.valid out.bits.payload := in.bits @@ -72,7 +73,8 @@ object DecoupledLogicalNetworkIOWrapper { } object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[T] = { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[T] = { val out = Wire(Decoupled(in.bits.payload)) out.valid := in.valid out.bits := in.bits.payload @@ -82,7 +84,8 @@ object DecoupledLogicalNetworkIOUnwrapper { } object DefaultFromPhysicalShim { - def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]): DecoupledIO[LogicalNetworkIO[T]] = { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) out.bits.header := in.bits.header out.bits.payload := in.bits.payload @@ -93,7 +96,8 @@ object DefaultFromPhysicalShim { } object DefaultToPhysicalShim { - def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]): DecoupledIO[PhysicalNetworkIO[T]] = { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) out.bits.header := in.bits.header out.bits.payload := in.bits.payload diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 8cfaec7e..ed0932c0 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -3,17 +3,19 @@ package uncore import Chisel._ import junctions._ -class RTC(pcr_MTIME: Int)(implicit val p: Parameters) extends Module with HTIFParameters { +case object RTCPeriod extends Field[Int] + +class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule { val io = new NastiIO - private val addrMap = new AddrHashMap(params(NastiAddrMap)) + private val addrMap = new AddrHashMap(p(NastiAddrMap)) val addrTable = Vec.tabulate(nCores) { i => - UInt(addrMap(s"conf:csr$i").start + pcr_MTIME * scrDataBytes) + UInt(addrMap(s"conf:csr$i").start + csr_MTIME * scrDataBytes) } - val rtc = Reg(init=UInt(0,64)) - val rtc_tick = Counter(params(RTCPeriod)).inc() + val rtc = Reg(init=UInt(0, scrDataBits)) + val rtc_tick = Counter(p(RTCPeriod)).inc() val sending_addr = Reg(init = Bool(false)) val sending_data = Reg(init = Bool(false)) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 09158810..365dedb5 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -3,24 +3,24 @@ package uncore import Chisel._ import junctions.{SMIIO, MMIOBase} -class SCRIO extends HTIFBundle { - val rdata = Vec(Bits(INPUT, 64), nSCR) +class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { + val rdata = Vec(Bits(INPUT, scrDataBits), nSCR) val wen = Bool(OUTPUT) val waddr = UInt(OUTPUT, log2Up(nSCR)) - val wdata = Bits(OUTPUT, 64) + val wdata = Bits(OUTPUT, scrDataBits) } -class SCRFile extends Module with HTIFParameters { +class SCRFile(implicit p: Parameters) extends HtifModule()(p) { val io = new Bundle { - val smi = new SMIIO(64, scrAddrBits).flip + val smi = new SMIIO(scrDataBits, scrAddrBits).flip val scr = new SCRIO } - val scr_rdata = Wire(Vec(Bits(width=64), io.scr.rdata.size)) + val scr_rdata = Wire(Vec(Bits(width=scrDataBits), io.scr.rdata.size)) for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt(params(MMIOBase) >> 20) + scr_rdata(1) := UInt(p(MMIOBase) >> 20) val read_addr = Reg(init = UInt(0, scrAddrBits)) val resp_valid = Reg(init = Bool(false)) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index ac6f5c23..23991083 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -38,24 +38,25 @@ case object TLNetworkIsOrderedP2P extends Field[Boolean] case object TLWriteMaskBits extends Field[Int] /** Utility trait for building Modules and Bundles that use TileLink parameters */ -trait TileLinkParameters extends UsesParameters { - val tlCoh = params(TLCoherencePolicy) - val tlNManagers = params(TLNManagers) - val tlNClients = params(TLNClients) - val tlNCachingClients = params(TLNCachingClients) - val tlNCachelessClients = params(TLNCachelessClients) +trait HasTileLinkParameters { + implicit val p: Parameters + val tlCoh = p(TLCoherencePolicy) + val tlNManagers = p(TLNManagers) + val tlNClients = p(TLNClients) + val tlNCachingClients = p(TLNCachingClients) + val tlNCachelessClients = p(TLNCachelessClients) val tlClientIdBits = log2Up(tlNClients) val tlManagerIdBits = log2Up(tlNManagers) - val tlMaxClientXacts = params(TLMaxClientXacts) - val tlMaxClientsPerPort = params(TLMaxClientsPerPort) - val tlMaxManagerXacts = params(TLMaxManagerXacts) + val tlMaxClientXacts = p(TLMaxClientXacts) + val tlMaxClientsPerPort = p(TLMaxClientsPerPort) + val tlMaxManagerXacts = p(TLMaxManagerXacts) val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) - val tlBlockAddrBits = params(TLBlockAddrBits) - val tlDataBits = params(TLDataBits) + val tlBlockAddrBits = p(TLBlockAddrBits) + val tlDataBits = p(TLDataBits) val tlDataBytes = tlDataBits/8 - val tlDataBeats = params(TLDataBeats) - val tlWriteMaskBits = params(TLWriteMaskBits) + val tlDataBeats = p(TLDataBeats) + val tlWriteMaskBits = p(TLWriteMaskBits) val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) val tlMemoryOpcodeBits = M_SZ @@ -68,32 +69,34 @@ trait TileLinkParameters extends UsesParameters { tlMemoryOpcodeBits)) + 1 val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 - val tlNetworkPreservesPointToPointOrdering = params(TLNetworkIsOrderedP2P) + val tlNetworkPreservesPointToPointOrdering = p(TLNetworkIsOrderedP2P) val tlNetworkDoesNotInterleaveBeats = true - val amoAluOperandBits = params(AmoAluOperandBits) + val amoAluOperandBits = p(AmoAluOperandBits) } -abstract class TLBundle extends Bundle with TileLinkParameters -abstract class TLModule extends Module with TileLinkParameters +abstract class TLModule(implicit val p: Parameters) extends Module + with HasTileLinkParameters +abstract class TLBundle(implicit val p: Parameters) extends junctions.ParameterizedBundle()(p) + with HasTileLinkParameters /** Base trait for all TileLink channels */ -trait TileLinkChannel extends TLBundle { +abstract class TileLinkChannel(implicit p: Parameters) extends TLBundle()(p) { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool } /** Directionality of message channel. Used to hook up logical network ports to physical network ports */ -trait ClientToManagerChannel extends TileLinkChannel +abstract class ClientToManagerChannel(implicit p: Parameters) extends TileLinkChannel()(p) /** Directionality of message channel. Used to hook up logical network ports to physical network ports */ -trait ManagerToClientChannel extends TileLinkChannel +abstract class ManagerToClientChannel(implicit p: Parameters) extends TileLinkChannel()(p) /** Directionality of message channel. Used to hook up logical network ports to physical network ports */ -trait ClientToClientChannel extends TileLinkChannel // Unused for now +abstract class ClientToClientChannel(implicit p: Parameters) extends TileLinkChannel()(p) // Unused for now /** Common signals that are used in multiple channels. * These traits are useful for type parameterizing bundle wiring functions. */ /** Address of a cache block. */ -trait HasCacheBlockAddress extends TLBundle { +trait HasCacheBlockAddress extends HasTileLinkParameters { val addr_block = UInt(width = tlBlockAddrBits) def conflicts(that: HasCacheBlockAddress) = this.addr_block === that.addr_block @@ -101,17 +104,17 @@ trait HasCacheBlockAddress extends TLBundle { } /** Sub-block address or beat id of multi-beat data */ -trait HasTileLinkBeatId extends TLBundle { +trait HasTileLinkBeatId extends HasTileLinkParameters { val addr_beat = UInt(width = tlBeatAddrBits) } /* Client-side transaction id. Usually Miss Status Handling Register File index */ -trait HasClientTransactionId extends TLBundle { +trait HasClientTransactionId extends HasTileLinkParameters { val client_xact_id = Bits(width = tlClientXactIdBits) } /** Manager-side transaction id. Usually Transaction Status Handling Register File index. */ -trait HasManagerTransactionId extends TLBundle { +trait HasManagerTransactionId extends HasTileLinkParameters { val manager_xact_id = Bits(width = tlManagerXactIdBits) } @@ -124,7 +127,7 @@ trait HasTileLinkData extends HasTileLinkBeatId { } /** The id of a client source or destination. Used in managers. */ -trait HasClientId extends TLBundle { +trait HasClientId extends HasTileLinkParameters { val client_id = UInt(width = tlClientIdBits) } @@ -138,7 +141,7 @@ trait HasClientId extends TLBundle { * PutAtomic built-in types. After sending an Acquire, clients must * wait for a manager to send them a [[uncore.Grant]] message in response. */ -class Acquire extends ClientToManagerChannel +class Acquire(implicit p: Parameters) extends ClientToManagerChannel()(p) with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { @@ -219,7 +222,7 @@ class Acquire extends ClientToManagerChannel } /** [[uncore.Acquire]] with an extra field stating its source id */ -class AcquireFromSrc extends Acquire with HasClientId +class AcquireFromSrc(implicit p: Parameters) extends Acquire()(p) with HasClientId /** Contains definitions of the the built-in Acquire types and a factory * for [[uncore.Acquire]] @@ -249,17 +252,18 @@ object Acquire { def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) - def fullWriteMask = SInt(-1, width = new Acquire().tlWriteMaskBits).toUInt + def fullWriteMask(implicit p: Parameters) = SInt(-1, width = p(TLWriteMaskBits)).toUInt // Most generic constructor def apply( - is_builtin_type: Bool, - a_type: Bits, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0), - union: UInt = UInt(0)): Acquire = { + is_builtin_type: Bool, + a_type: Bits, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + union: UInt = UInt(0)) + (implicit p: Parameters): Acquire = { val acq = Wire(new Acquire) acq.is_builtin_type := is_builtin_type acq.a_type := a_type @@ -272,7 +276,7 @@ object Acquire { } // Copy constructor def apply(a: Acquire): Acquire = { - val acq = Wire(new Acquire) + val acq = Wire(new Acquire()(a.p)) acq := a acq } @@ -292,10 +296,11 @@ object Acquire { */ object Get { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - alloc: Bool = Bool(true)): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + alloc: Bool = Bool(true)) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.getType, @@ -305,12 +310,13 @@ object Get { union = Cat(MT_Q, M_XRD, alloc)) } def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - addr_byte: UInt, - operand_size: UInt, - alloc: Bool): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + operand_size: UInt, + alloc: Bool) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.getType, @@ -332,9 +338,10 @@ object Get { */ object GetBlock { def apply( - client_xact_id: UInt = UInt(0), - addr_block: UInt, - alloc: Bool = Bool(true)): Acquire = { + client_xact_id: UInt = UInt(0), + addr_block: UInt, + alloc: Bool = Bool(true)) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.getBlockType, @@ -352,8 +359,9 @@ object GetBlock { */ object GetPrefetch { def apply( - client_xact_id: UInt, - addr_block: UInt): Acquire = { + client_xact_id: UInt, + addr_block: UInt) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.prefetchType, @@ -376,11 +384,12 @@ object GetPrefetch { */ object Put { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - data: UInt, - wmask: UInt = Acquire.fullWriteMask): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + wmask: Option[UInt]= None) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putType, @@ -388,7 +397,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(wmask, Bool(true))) + union = Cat(wmask.getOrElse(Acquire.fullWriteMask), Bool(true))) } } @@ -407,11 +416,12 @@ object Put { */ object PutBlock { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - data: UInt, - wmask: UInt): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + wmask: UInt) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -422,11 +432,12 @@ object PutBlock { union = Cat(wmask, (wmask != Acquire.fullWriteMask))) } def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - data: UInt, - alloc: Bool = Bool(true)): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + data: UInt, + alloc: Bool = Bool(true)) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putBlockType, @@ -446,8 +457,9 @@ object PutBlock { */ object PutPrefetch { def apply( - client_xact_id: UInt, - addr_block: UInt): Acquire = { + client_xact_id: UInt, + addr_block: UInt) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.prefetchType, @@ -470,13 +482,14 @@ object PutPrefetch { */ object PutAtomic { def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - addr_byte: UInt, - atomic_opcode: UInt, - operand_size: UInt, - data: UInt): Acquire = { + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt, + addr_byte: UInt, + atomic_opcode: UInt, + operand_size: UInt, + data: UInt) + (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), a_type = Acquire.putAtomicType, @@ -493,7 +506,7 @@ object PutAtomic { * The available types of Probes are customized by a particular * [[uncore.CoherencePolicy]]. */ -class Probe extends ManagerToClientChannel +class Probe(implicit p: Parameters) extends ManagerToClientChannel()(p) with HasCacheBlockAddress { val p_type = UInt(width = tlCoh.probeTypeWidth) @@ -503,7 +516,7 @@ class Probe extends ManagerToClientChannel } /** [[uncore.Probe]] with an extra field stating its destination id */ -class ProbeToDst extends Probe with HasClientId +class ProbeToDst(implicit p: Parameters) extends Probe()(p) with HasClientId /** Contains factories for [[uncore.Probe]] and [[uncore.ProbeToDst]] * @@ -515,13 +528,13 @@ class ProbeToDst extends Probe with HasClientId * @param addr_block address of the cache block */ object Probe { - def apply(p_type: UInt, addr_block: UInt): Probe = { + def apply(p_type: UInt, addr_block: UInt)(implicit p: Parameters): Probe = { val prb = Wire(new Probe) prb.p_type := p_type prb.addr_block := addr_block prb } - def apply(dst: UInt, p_type: UInt, addr_block: UInt): ProbeToDst = { + def apply(dst: UInt, p_type: UInt, addr_block: UInt)(implicit p: Parameters): ProbeToDst = { val prb = Wire(new ProbeToDst) prb.client_id := dst prb.p_type := p_type @@ -537,7 +550,7 @@ object Probe { * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grant Grants]]. */ -class Release extends ClientToManagerChannel +class Release(implicit p: Parameters) extends ClientToManagerChannel()(p) with HasCacheBlockAddress with HasClientTransactionId with HasTileLinkData { @@ -555,7 +568,7 @@ class Release extends ClientToManagerChannel } /** [[uncore.Release]] with an extra field stating its source id */ -class ReleaseFromSrc extends Release with HasClientId +class ReleaseFromSrc(implicit p: Parameters) extends Release()(p) with HasClientId /** Contains a [[uncore.Release]] factory * @@ -571,12 +584,13 @@ class ReleaseFromSrc extends Release with HasClientId */ object Release { def apply( - voluntary: Bool, - r_type: UInt, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)) + (implicit p: Parameters): Release = { val rel = Wire(new Release) rel.r_type := r_type rel.client_xact_id := client_xact_id @@ -595,7 +609,7 @@ object Release { * coherence policies may also define custom Grant types. Grants may contain data * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. */ -class Grant extends ManagerToClientChannel +class Grant(implicit p: Parameters) extends ManagerToClientChannel()(p) with HasTileLinkData with HasClientTransactionId with HasManagerTransactionId { @@ -616,14 +630,14 @@ class Grant extends ManagerToClientChannel def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Wire(Bundle(new Finish, { case TLMaxManagerXacts => tlMaxManagerXacts })) + val f = Wire(Bundle(new Finish)) f.manager_xact_id := this.manager_xact_id f } } /** [[uncore.Grant]] with an extra field stating its destination */ -class GrantToDst extends Grant with HasClientId +class GrantToDst(implicit p: Parameters) extends Grant()(p) with HasClientId /** Contains definitions of the the built-in grant types and factories * for [[uncore.Grant]] and [[uncore.GrantToDst]] @@ -650,12 +664,13 @@ object Grant { def typesWithMultibeatData= Vec(getDataBlockType) def apply( - is_builtin_type: Bool, - g_type: UInt, - client_xact_id: UInt, - manager_xact_id: UInt, - addr_beat: UInt, - data: UInt): Grant = { + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt, + data: UInt) + (implicit p: Parameters): Grant = { val gnt = Wire(new Grant) gnt.is_builtin_type := is_builtin_type gnt.g_type := g_type @@ -667,13 +682,14 @@ object Grant { } def apply( - dst: UInt, - is_builtin_type: Bool, - g_type: UInt, - client_xact_id: UInt, - manager_xact_id: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): GrantToDst = { + dst: UInt, + is_builtin_type: Bool, + g_type: UInt, + client_xact_id: UInt, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)) + (implicit p: Parameters): GrantToDst = { val gnt = Wire(new GrantToDst) gnt.client_id := dst gnt.is_builtin_type := is_builtin_type @@ -692,20 +708,21 @@ object Grant { * When a Finish message is received, a manager knows it is safe to begin * processing other transactions that touch the same cache block. */ -class Finish extends ClientToManagerChannel with HasManagerTransactionId { +class Finish(implicit p: Parameters) extends ClientToManagerChannel()(p) + with HasManagerTransactionId { def hasData(dummy: Int = 0) = Bool(false) def hasMultibeatData(dummy: Int = 0) = Bool(false) } /** Complete IO definition for incoherent TileLink, including networking headers */ -class UncachedTileLinkIO extends TLBundle { +class UncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) val grant = new DecoupledIO(new LogicalNetworkIO(new Grant)).flip val finish = new DecoupledIO(new LogicalNetworkIO(new Finish)) } /** Complete IO definition for coherent TileLink, including networking headers */ -class TileLinkIO extends UncachedTileLinkIO { +class TileLinkIO(implicit p: Parameters) extends UncachedTileLinkIO()(p) { val probe = new DecoupledIO(new LogicalNetworkIO(new Probe)).flip val release = new DecoupledIO(new LogicalNetworkIO(new Release)) } @@ -722,7 +739,7 @@ class TileLinkIO extends UncachedTileLinkIO { * assumption that a [[uncore.FinishUnit]] has been coupled to the TileLinkIO port * to deal with acking received [[uncore.Grant Grants]]. */ -class ClientUncachedTileLinkIO extends TLBundle { +class ClientUncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val acquire = new DecoupledIO(new Acquire) val grant = new DecoupledIO(new Grant).flip } @@ -730,7 +747,7 @@ class ClientUncachedTileLinkIO extends TLBundle { /** This version of TileLinkIO does not contain network headers. * It is intended for use within client agents. */ -class ClientTileLinkIO extends ClientUncachedTileLinkIO { +class ClientTileLinkIO(implicit p: Parameters) extends ClientUncachedTileLinkIO()(p) { val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) } @@ -749,7 +766,7 @@ class ClientTileLinkIO extends ClientUncachedTileLinkIO { * see Finished so they know when to allow new transactions on a cache * block to proceed. */ -class ManagerTileLinkIO extends TLBundle { +class ManagerTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val acquire = new DecoupledIO(new AcquireFromSrc).flip val grant = new DecoupledIO(new GrantToDst) val finish = new DecoupledIO(new Finish).flip @@ -759,31 +776,21 @@ class ManagerTileLinkIO extends TLBundle { /** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ object TileLinkIOWrapper { - def apply(utl: ClientUncachedTileLinkIO, p: Parameters): ClientTileLinkIO = { - val conv = Module(new ClientTileLinkIOWrapper)(p) - conv.io.in <> utl + def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) + conv.io.in <> tl conv.io.out } - def apply(utl: ClientUncachedTileLinkIO): ClientTileLinkIO = { - val conv = Module(new ClientTileLinkIOWrapper) - conv.io.in <> utl + def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> tl conv.io.out } def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl - def apply(utl: UncachedTileLinkIO, p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper)(p) - conv.io.in <> utl - conv.io.out - } - def apply(utl: UncachedTileLinkIO): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> utl - conv.io.out - } def apply(tl: TileLinkIO): TileLinkIO = tl } -class TileLinkIOWrapper extends TLModule { +class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val in = new UncachedTileLinkIO().flip val out = new TileLinkIO @@ -795,7 +802,7 @@ class TileLinkIOWrapper extends TLModule { io.out.release.valid := Bool(false) } -class ClientTileLinkIOWrapper extends TLModule { +class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val in = new ClientUncachedTileLinkIO().flip val out = new ClientTileLinkIO @@ -809,14 +816,16 @@ class ClientTileLinkIOWrapper extends TLModule { /** Used to track metadata for transactions where multiple secondary misses have been merged * and handled by a single transaction tracker. */ -class SecondaryMissInfo extends TLBundle // TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId +class SecondaryMissInfo(implicit p: Parameters) extends TLBundle()(p) with HasTileLinkBeatId with HasClientTransactionId +// TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId /** A helper module that automatically issues [[uncore.Finish]] messages in repsonse * to [[uncore.Grant]] that it receives from a manager and forwards to a client */ -class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with HasDataBeatCounters { +class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) + with HasDataBeatCounters { val io = new Bundle { val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip val refill = Decoupled(new Grant) @@ -862,12 +871,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2) extends TLModule with Has } } -class FinishQueueEntry extends TLBundle { +class FinishQueueEntry(implicit p: Parameters) extends TLBundle()(p) { val fin = new Finish - val dst = UInt(width = log2Up(params(LNEndpoints))) + val dst = UInt(width = log2Up(p(LNEndpoints))) } -class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) +class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishQueueEntry()(p), entries) /** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] * @@ -879,7 +888,8 @@ class FinishQueue(entries: Int) extends Queue(new FinishQueueEntry, entries) * @param clientId network port id of this agent * @param addrConvert how a physical address maps to a destination manager port id */ -class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extends TLModule { +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val client = new ClientTileLinkIO().flip val network = new TileLinkIO @@ -904,9 +914,10 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) extend object ClientTileLinkHeaderCreator { def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) out.bits.payload := in.bits out.bits.header.src := UInt(clientId) @@ -927,7 +938,8 @@ object ClientTileLinkHeaderCreator { * @param managerId the network port id of this agent * @param idConvert how a sharer id maps to a destination client port id */ -class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extends TLModule { +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val manager = new ManagerTileLinkIO().flip val network = new TileLinkIO().flip @@ -943,9 +955,10 @@ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) extend object ManagerTileLinkHeaderCreator { def apply[T <: ManagerToClientChannel with HasClientId]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt): DecoupledIO[LogicalNetworkIO[T]] = { + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) out.bits.payload := in.bits out.bits.header.src := UInt(managerId) @@ -960,7 +973,7 @@ object ManagerTileLinkHeaderCreator { case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) /** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ -class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { +class TileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { val io = new Bundle { val client = new TileLinkIO().flip val manager = new TileLinkIO @@ -973,18 +986,18 @@ class TileLinkEnqueuer(depths: TileLinkDepths) extends Module { } object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: TileLinkDepths)(p: Parameters): TileLinkIO = { - val t = Module(new TileLinkEnqueuer(depths))(p) + def apply(in: TileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths)) t.io.client <> in t.io.manager } - def apply(in: TileLinkIO, depth: Int)(p: Parameters): TileLinkIO = { - apply(in, TileLinkDepths(depth, depth, depth, depth, depth))(p) + def apply(in: TileLinkIO, depth: Int)(implicit p: Parameters): TileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) } } /** Utility functions for constructing TileLinkIO arbiters */ -trait TileLinkArbiterLike extends TileLinkParameters { +trait TileLinkArbiterLike extends HasTileLinkParameters { // Some shorthand type variables type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId @@ -1086,7 +1099,8 @@ trait TileLinkArbiterLike extends TileLinkParameters { } /** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ -abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { +abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { val io = new Bundle { val in = Vec(new UncachedTileLinkIO, arbN).flip val out = new UncachedTileLinkIO @@ -1097,7 +1111,8 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int) extends Module with Tile } /** Abstract base case for any Arbiters that have cached TileLinkIOs */ -abstract class TileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike { +abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { val io = new Bundle { val in = Vec(new TileLinkIO, arbN).flip val out = new TileLinkIO @@ -1133,15 +1148,15 @@ trait UsesNewId extends TileLinkArbiterLike { } // Now we can mix-in thevarious id-generation traits to make concrete arbiter classes -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends UncachedTileLinkIOArbiter(n) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int) extends UncachedTileLinkIOArbiter(n) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int) extends UncachedTileLinkIOArbiter(n) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int) extends TileLinkIOArbiter(n) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int) extends TileLinkIOArbiter(n) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int) extends TileLinkIOArbiter(n) with UsesNewId +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with UsesNewId /** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { +class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { val in = Vec(new ClientUncachedTileLinkIO, arbN).flip val out = new ClientUncachedTileLinkIO @@ -1151,7 +1166,7 @@ class ClientUncachedTileLinkIOArbiter(val arbN: Int) extends Module with TileLin } /** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientTileLinkIOArbiter(val arbN: Int) extends Module with TileLinkArbiterLike with AppendsArbiterId { +class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { val in = Vec(new ClientTileLinkIO, arbN).flip val out = new ClientTileLinkIO @@ -1246,7 +1261,7 @@ class ClientTileLinkIOUnwrapperInfo extends Bundle { val builtin = Bool() } -class ClientTileLinkIOUnwrapper extends TLModule { +class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val in = new ClientTileLinkIO().flip val out = new ClientUncachedTileLinkIO @@ -1328,12 +1343,13 @@ class ClientTileLinkIOUnwrapper extends TLModule { io.in.probe.valid := Bool(false) } -class NastiIOTileLinkIOConverterInfo extends TLBundle { +class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { val byteOff = UInt(width = tlByteAddrBits) val subblock = Bool() } -class NastiIOTileLinkIOConverter(implicit val p: Parameters) extends TLModule with HasNastiParameters { +class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { val io = new Bundle { val tl = new ClientUncachedTileLinkIO().flip val nasti = new NastiIO @@ -1349,7 +1365,7 @@ class NastiIOTileLinkIOConverter(implicit val p: Parameters) extends TLModule wi MT_Q -> UInt(log2Up(tlDataBytes)))) val dataBits = tlDataBits*tlDataBeats - val dstIdBits = params(LNHeaderBits) + val dstIdBits = p(LNHeaderBits) require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NastiIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 969edcfd..cee7e392 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -6,18 +6,18 @@ import Chisel._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -case object RTCPeriod extends Field[Int] -trait CoherenceAgentParameters extends UsesParameters { +trait HasCoherenceAgentParameters { + implicit val p: Parameters val nReleaseTransactors = 1 - val nAcquireTransactors = params(NAcquireTransactors) + val nAcquireTransactors = p(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val outerTLParams = params.alterPartial({ case TLId => params(OuterTLId)}) + val outerTLParams = p.alterPartial({ case TLId => p(OuterTLId)}) val outerDataBeats = outerTLParams(TLDataBeats) val outerDataBits = outerTLParams(TLDataBits) val outerBeatAddrBits = log2Up(outerDataBeats) val outerByteAddrBits = log2Up(outerDataBits/8) - val innerTLParams = params.alterPartial({case TLId => params(InnerTLId)}) + val innerTLParams = p.alterPartial({case TLId => p(InnerTLId)}) val innerDataBeats = innerTLParams(TLDataBeats) val innerDataBits = innerTLParams(TLDataBits) val innerWriteMaskBits = innerTLParams(TLWriteMaskBits) @@ -26,8 +26,10 @@ trait CoherenceAgentParameters extends UsesParameters { require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement } -abstract class CoherenceAgentBundle extends Bundle with CoherenceAgentParameters -abstract class CoherenceAgentModule extends Module with CoherenceAgentParameters +abstract class CoherenceAgentModule(implicit val p: Parameters) extends Module + with HasCoherenceAgentParameters +abstract class CoherenceAgentBundle(implicit val p: Parameters) extends junctions.ParameterizedBundle()(p) + with HasCoherenceAgentParameters trait HasCoherenceAgentWiringHelpers { def doOutputArbitration[T <: TileLinkChannel]( @@ -39,7 +41,7 @@ trait HasCoherenceAgentWiringHelpers { arb.io.in <> ins } - def doInputRouting[T <: HasManagerTransactionId]( + def doInputRouting[T <: Bundle with HasManagerTransactionId]( in: DecoupledIO[T], outs: Seq[DecoupledIO[T]]) { val idx = in.bits.manager_xact_id @@ -49,7 +51,7 @@ trait HasCoherenceAgentWiringHelpers { } } -trait HasInnerTLIO extends CoherenceAgentBundle { +trait HasInnerTLIO extends HasCoherenceAgentParameters { val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) val incoherent = Vec(Bool(), inner.tlNCachingClients).asInput def iacq(dummy: Int = 0) = inner.acquire.bits @@ -59,13 +61,13 @@ trait HasInnerTLIO extends CoherenceAgentBundle { def ifin(dummy: Int = 0) = inner.finish.bits } -trait HasUncachedOuterTLIO extends CoherenceAgentBundle { +trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters { val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } -trait HasCachedOuterTLIO extends CoherenceAgentBundle { +trait HasCachedOuterTLIO extends HasCoherenceAgentParameters { val outer = Bundle(new ClientTileLinkIO)(outerTLParams) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits @@ -73,25 +75,29 @@ trait HasCachedOuterTLIO extends CoherenceAgentBundle { def ognt(dummy: Int = 0) = outer.grant.bits } -class ManagerTLIO extends HasInnerTLIO with HasUncachedOuterTLIO +class ManagerTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p) + with HasInnerTLIO + with HasUncachedOuterTLIO -abstract class CoherenceAgent extends CoherenceAgentModule { +abstract class CoherenceAgent(implicit p: Parameters) extends CoherenceAgentModule()(p) { def innerTL: ManagerTileLinkIO def outerTL: ClientTileLinkIO def incoherent: Vec[Bool] } -abstract class ManagerCoherenceAgent extends CoherenceAgent +abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) with HasCoherenceAgentWiringHelpers { val io = new ManagerTLIO def innerTL = io.inner - def outerTL = TileLinkIOWrapper(io.outer, outerTLParams) + def outerTL = TileLinkIOWrapper(io.outer)(outerTLParams) def incoherent = io.incoherent } -class HierarchicalTLIO extends HasInnerTLIO with HasCachedOuterTLIO +class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p) + with HasInnerTLIO + with HasCachedOuterTLIO -abstract class HierarchicalCoherenceAgent extends CoherenceAgent { +abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) { val io = new HierarchicalTLIO def innerTL = io.inner def outerTL = io.outer @@ -104,10 +110,14 @@ trait HasTrackerConflictIO extends Bundle { val has_release_match = Bool(OUTPUT) } -class ManagerXactTrackerIO extends ManagerTLIO with HasTrackerConflictIO -class HierarchicalXactTrackerIO extends HierarchicalTLIO with HasTrackerConflictIO +class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) + with HasTrackerConflictIO -abstract class XactTracker extends CoherenceAgentModule with HasDataBeatCounters { +class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p) + with HasTrackerConflictIO + +abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p) + with HasDataBeatCounters { def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index ab4f9947..bf05d9d3 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -42,7 +42,7 @@ object ZCounter { } } -class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module { +class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen).flip val out = Decoupled(gen) @@ -96,7 +96,7 @@ class FlowThroughSerializer[T <: HasTileLinkData](gen: T, n: Int) extends Module } object FlowThroughSerializer { - def apply[T <: HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { + def apply[T <: Bundle with HasTileLinkData](in: DecoupledIO[T], n: Int): DecoupledIO[T] = { val fs = Module(new FlowThroughSerializer(in.bits, n)) fs.io.in.valid := in.valid fs.io.in.bits := in.bits From 66ea39638eb7eae8bfb07aa06d273b62aeba3370 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 6 Oct 2015 18:19:45 -0700 Subject: [PATCH 451/688] GlobalAddrMap --- uncore/src/main/scala/cache.scala | 3 ++- uncore/src/main/scala/rtc.scala | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b77ecc10..20a02366 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -11,7 +11,6 @@ case object NWays extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object AmoAluOperandBits extends Field[Int] -case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] @@ -169,6 +168,8 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext io.write.ready := !rst } +case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] + trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index ed0932c0..e292cb3d 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -8,8 +8,7 @@ case object RTCPeriod extends Field[Int] class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule { val io = new NastiIO - private val addrMap = new AddrHashMap(p(NastiAddrMap)) - + val addrMap = new AddrHashMap(p(GlobalAddrMap)) val addrTable = Vec.tabulate(nCores) { i => UInt(addrMap(s"conf:csr$i").start + csr_MTIME * scrDataBytes) } From 7fa3eb95e382ebf11163aedb24d490fadde322b6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 13 Oct 2015 23:42:28 -0700 Subject: [PATCH 452/688] refactor tilelink params --- uncore/src/main/scala/broadcast.scala | 50 ++++---- uncore/src/main/scala/cache.scala | 44 ++++--- uncore/src/main/scala/coherence.scala | 50 ++++---- uncore/src/main/scala/htif.scala | 24 ++-- uncore/src/main/scala/metadata.scala | 19 ++- uncore/src/main/scala/tilelink.scala | 170 ++++++++++++++++---------- uncore/src/main/scala/uncore.scala | 32 +++-- 7 files changed, 213 insertions(+), 176 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c48606c3..ef3cbc79 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -31,17 +31,17 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) val internalDataBits = new DataQueueLocation().getWidth val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - val trackerTLParams = p.alterPartial({ - case TLDataBits => internalDataBits - case TLWriteMaskBits => innerWriteMaskBits + val usingStoreDataQueue = p.alterPartial({ + case TLKey(`innerTLId`) => innerTLParams.copy()(internalDataBits, innerWriteMaskBits) + case TLKey(`outerTLId`) => outerTLParams.copy()(internalDataBits, outerWriteMaskBits) }) // Create SHRs for outstanding transactions val trackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id))(trackerTLParams)) ++ + Module(new BroadcastVoluntaryReleaseTracker(id)(usingStoreDataQueue))) ++ (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id))(trackerTLParams)) + Module(new BroadcastAcquireTracker(id)(usingStoreDataQueue))) // Propagate incoherence flags trackerList.map(_.io.incoherent := io.incoherent) @@ -105,10 +105,8 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port - val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size)(p.alterPartial( - { case TLId => p(OuterTLId) - case TLDataBits => internalDataBits - case TLWriteMaskBits => innerWriteMaskBits }))) + val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size) + (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) }))) outer_arb.io.in <> trackerList.map(_.io.outer) // Get the pending data out of the store data queue val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) @@ -137,8 +135,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => p(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(Vec(io.irel().data, innerDataBeats)) + val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val coh = ManagerMetadata.onReset val collect_irel_data = Reg(init=Bool(false)) @@ -161,17 +158,17 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) //TODO: Use io.outer.release instead? - io.outer.acquire.bits := Bundle( - PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)))(outerTLParams) + io.outer.acquire.bits := PutBlock( + client_xact_id = UInt(trackerId), + addr_block = xact.addr_block, + addr_beat = oacq_data_cnt, + data = xact.data_buffer(oacq_data_cnt)) + (p.alterPartial({ case TLId => outerTLId })) when(collect_irel_data) { io.inner.release.ready := Bool(true) when(io.inner.release.valid) { - data_buffer(io.irel().addr_beat) := io.irel().data + xact.data_buffer(io.irel().addr_beat) := io.irel().data irel_data_valid := irel_data_valid.bitSet(io.irel().addr_beat, Bool(true)) } when(irel_data_done) { collect_irel_data := Bool(false) } @@ -182,7 +179,7 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) io.inner.release.ready := Bool(true) when( io.inner.release.valid ) { xact := io.irel() - data_buffer(UInt(0)) := io.irel().data + xact.data_buffer(UInt(0)) := io.irel().data collect_irel_data := io.irel().hasMultibeatData() irel_data_valid := io.irel().hasData() << io.irel().addr_beat state := Mux(io.irel().hasData(), s_outer, @@ -217,12 +214,7 @@ class BroadcastAcquireTracker(trackerId: Int) val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new AcquireFromSrc, { - case TLId => p(InnerTLId) - case TLDataBits => 0 - case TLWriteMaskBits => innerWriteMaskBits - })) - val data_buffer = Reg(Vec(io.iacq().data, innerDataBeats)) + val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && @@ -268,14 +260,14 @@ class BroadcastAcquireTracker(trackerId: Int) client_xact_id = UInt(trackerId), addr_block = xact.addr_block, addr_beat = xact.addr_beat, - data = data_buffer(0), + data = xact.data_buffer(0), wmask = xact.wmask()) val oacq_write_block = PutBlock( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, addr_beat = oacq_data_cnt, - data = data_buffer(oacq_data_cnt)) + data = xact.data_buffer(oacq_data_cnt)) val oacq_read_beat = Get( client_xact_id = UInt(trackerId), @@ -321,7 +313,7 @@ class BroadcastAcquireTracker(trackerId: Int) when(collect_iacq_data) { io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { - data_buffer(io.iacq().addr_beat) := io.iacq().data + xact.data_buffer(io.iacq().addr_beat) := io.iacq().data iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true)) } when(iacq_data_done) { collect_iacq_data := Bool(false) } @@ -338,7 +330,7 @@ class BroadcastAcquireTracker(trackerId: Int) io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact := io.iacq() - data_buffer(UInt(0)) := io.iacq().data + xact.data_buffer(UInt(0)) := io.iacq().data collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat val needs_probes = mask_incoherent.orR diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 20a02366..143f68e1 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -173,7 +173,7 @@ case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { val idxMSB = idxBits-1 val idxLSB = 0 - val blockAddrBits = p(TLBlockAddrBits) + //val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats val internalDataBeats = p(CacheBlockBytes)*8/rowBits @@ -419,7 +419,8 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) // Create an arbiter for the one memory port val outerList = trackerList.map(_.io.outer) :+ wb.io.outer - val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size))(outerTLParams) + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size) + (p.alterPartial({ case TLId => p(OuterTLId)}))) outer_arb.io.in <> outerList io.outer <> outer_arb.io.out @@ -508,8 +509,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new ReleaseFromSrc, { case TLId => p(InnerTLId); case TLDataBits => 0 })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val xact = Reg(Bundle(new BufferedReleaseFromSrc()(p.alterPartial({case TLId => p(InnerTLId)})))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -525,7 +525,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends // Accept a voluntary Release (and any further beats of data) pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) io.inner.release.ready := state === s_idle || pending_irels.orR - when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } + when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } // Begin a transaction by getting the current block metadata io.meta.read.valid := state === s_meta_read @@ -543,7 +543,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := ~UInt(0, io.data.write.bits.wmask.getWidth) - io.data.write.bits.data := data_buffer(curr_write_beat) + io.data.write.bits.data := xact.data_buffer(curr_write_beat) // Send an acknowledgement io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels @@ -603,8 +603,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val state = Reg(init=s_idle) // State holding transaction metadata - val xact = Reg(Bundle(new AcquireFromSrc, { case TLId => p(InnerTLId) })) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + val xact = Reg(Bundle(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => p(InnerTLId) })))) val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } @@ -612,7 +611,8 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val pending_coh = Reg{ xact_old_meta.coh } // Secondary miss queue - val ignt_q = Module(new Queue(new SecondaryMissInfo, nSecondaryMisses))(innerTLParams) + val ignt_q = Module(new Queue(new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })), + nSecondaryMisses)) // State holding progress made on processing this transaction val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) @@ -657,8 +657,8 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData - amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData - val amo_result = xact.data // Reuse xact buffer space to store AMO result + amoalu.io.rhs := xact.data_buffer.head // default, overwritten by calls to mergeData + val amo_result = Reg(init = UInt(0, xact.tlDataBits)) // Utility functions for updating the data and metadata that will be kept in // the cache or granted to the original requestor after this transaction: @@ -672,11 +672,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is already in the buffer + val new_data = xact.data_buffer(beat) // Newly Put data is already in the buffer amoalu.io.lhs := old_data >> xact.amo_shift_bits() amoalu.io.rhs := new_data >> xact.amo_shift_bits() val wmask = FillInterleaved(8, wmask_buffer(beat)) - data_buffer(beat) := ~wmask & old_data | + xact.data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << xact.amo_shift_bits(), new_data) @@ -780,14 +780,12 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // built-in Acquire from the inner TL to the outer TL io.outer.acquire.valid := state === s_outer_acquire && (xact.allocate() || !pending_puts(oacq_data_idx)) - io.outer.acquire.bits := Mux( - xact.allocate(), - xact_old_meta.coh.outer.makeAcquire( + io.outer.acquire.bits := Mux(xact.allocate(), xact_old_meta.coh.outer, ClientMetadata.onReset) + .makeAcquire( client_xact_id = UInt(0), addr_block = xact.addr_block, - op_code = xact.op_code()), - Bundle(Acquire(xact))(outerTLParams)) - io.oacq().data := data_buffer(oacq_data_idx) + op_code = xact.op_code()) + io.oacq().data := xact.data_buffer(oacq_data_idx) // Handle the response from outer memory io.outer.grant.ready := state === s_busy @@ -814,7 +812,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra manager_xact_id = UInt(trackerId), data = Mux(xact.is(Acquire.putAtomicType), amo_result, - data_buffer(ignt_data_idx))) + xact.data_buffer(ignt_data_idx))) io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( @@ -864,7 +862,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) io.data.write.bits.addr_beat := curr_write_beat io.data.write.bits.wmask := wmask_buffer(curr_write_beat) - io.data.write.bits.data := data_buffer(curr_write_beat) + io.data.write.bits.data := xact.data_buffer(curr_write_beat) // End a transaction by updating the block metadata io.meta.write.valid := state === s_meta_write @@ -879,7 +877,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val beat = io.iacq().addr_beat val wmask = io.iacq().wmask() val full = FillInterleaved(8, wmask) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + xact.data_buffer(beat) := (~full & xact.data_buffer(beat)) | (full & io.iacq().data) wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) } @@ -892,7 +890,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { xact := io.iacq() - xact.data := UInt(0) + amo_result := UInt(0) pending_puts := Mux( // Make sure to collect all data from a PutBlock io.iacq().isBuiltInType(Acquire.putBlockType), dropPendingBitWhenBeatHasData(io.inner.acquire), diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2ec15046..fb6ec049 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -90,18 +90,18 @@ trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { def masterStateWidth = log2Ceil(nManagerStates) // Transaction probing logic - def requiresProbes(acq: Acquire, meta: ManagerMetadata): Bool + def requiresProbes(acq: AcquireMetadata, meta: ManagerMetadata): Bool def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool // Determine which custom message type to use in response def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - def getProbeType(acq: Acquire, meta: ManagerMetadata): UInt - def getGrantType(acq: Acquire, meta: ManagerMetadata): UInt + def getProbeType(acq: AcquireMetadata, meta: ManagerMetadata): UInt + def getGrantType(acq: AcquireMetadata, meta: ManagerMetadata): UInt def getExclusiveGrantType(): UInt // Mutate ManagerMetadata based on messages or cmds - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnGrant(outgoing: Grant, dst: UInt, meta: ManagerMetadata) = + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: GrantMetadata, dst: UInt, meta: ManagerMetadata) = ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached meta.sharers, dir.push(meta.sharers, dst)))(meta.p) @@ -171,7 +171,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = !dir.none(meta.sharers) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -179,7 +179,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -189,10 +189,10 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = grantExclusive def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -270,7 +270,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol - def requiresProbes(a: Acquire, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = !dir.none(meta.sharers) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = @@ -278,7 +278,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -288,10 +288,10 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = grantExclusive + def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = grantExclusive def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -376,7 +376,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // notification msg to track clean drops) // Also could avoid probes on outer WBs. - def requiresProbes(a: Acquire, meta: ManagerMetadata) = + def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -388,7 +388,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -400,13 +400,13 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { acquireShared -> probeDowngrade, acquireExclusive -> probeInvalidate))) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -493,7 +493,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // notification msg to track clean drops) // Also could avoid probes on outer WBs. - def requiresProbes(a: Acquire, meta: ManagerMetadata) = + def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -505,7 +505,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -517,13 +517,13 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { acquireShared -> probeDowngrade, acquireExclusive -> probeInvalidate))) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.a_type === acquireShared, Mux(!dir.none(meta.sharers), grantShared, grantExclusive), grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -631,7 +631,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d // Manager states and functions: val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types - def requiresProbes(a: Acquire, meta: ManagerMetadata) = + def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -643,7 +643,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: Acquire, meta: ManagerMetadata): UInt = + def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -656,14 +656,14 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireExclusive -> probeInvalidate, acquireInvalidateOthers -> probeInvalidateOthers))) - def getGrantType(a: Acquire, meta: ManagerMetadata): UInt = + def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = MuxLookup(a.a_type, grantShared, Array( acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: Release, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5924bf2c..fedadf71 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -12,23 +12,23 @@ case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, nSCR: Int = trait HasHtifParameters { implicit val p: Parameters - lazy val external = p(HtifKey) - lazy val dataBits = p(TLDataBits) - lazy val dataBeats = p(TLDataBeats) - lazy val w = external.width - lazy val nSCR = external.nSCR - lazy val scrAddrBits = log2Up(nSCR) - lazy val scrDataBits = 64 - lazy val scrDataBytes = scrDataBits / 8 - lazy val offsetBits = external.offsetBits - lazy val nCores = external.nCores + val external = p(HtifKey) + val dataBits = p(TLKey(p(TLId))).dataBitsPerBeat + val dataBeats = p(TLKey(p(TLId))).dataBeats + val w = external.width + val nSCR = external.nSCR + val scrAddrBits = log2Up(nSCR) + val scrDataBits = 64 + val scrDataBytes = scrDataBits / 8 + val offsetBits = external.offsetBits + val nCores = external.nCores } abstract class HtifModule(implicit val p: Parameters) extends Module with HasHtifParameters abstract class HtifBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasHtifParameters -class HostIO(implicit p: Parameters) extends HtifBundle()(p) { +class HostIO(w: Int) extends Bundle { val clk = Bool(OUTPUT) val clk_edge = Bool(OUTPUT) val in = Decoupled(Bits(width = w)).flip @@ -49,7 +49,7 @@ class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { val io = new Bundle { - val host = new HostIO + val host = new HostIO(w) val cpu = Vec(new HtifIO, nCores).flip val mem = new ClientUncachedTileLinkIO val scr = new SMIIO(scrDataBits, scrAddrBits) diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index f03dfad7..ca8dda50 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -176,7 +176,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) /** Does this [[uncore.Acquire]] require [[uncore.Probe Probes]] to be sent */ - def requiresProbes(acq: Acquire): Bool = co.requiresProbes(acq, this) + def requiresProbes(acq: AcquireMetadata): Bool = co.requiresProbes(acq, this) /** Does this memory op require [[uncore.Probe Probes]] to be sent */ def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) /** Does an eviction require [[uncore.Probe Probes]] to be sent */ @@ -188,7 +188,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param dst Destination client id for this Probe * @param acq Acquire message triggering this Probe */ - def makeProbe(dst: UInt, acq: Acquire): ProbeToDst = + def makeProbe(dst: UInt, acq: AcquireMetadata): ProbeToDst = Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p)) /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op @@ -213,7 +213,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param rel Release message being acknowledged by this Grant * @param manager_xact_id manager's transaction id */ - def makeGrant(rel: ReleaseFromSrc, manager_xact_id: UInt): GrantToDst = { + def makeGrant(rel: ReleaseMetadata with HasClientId, manager_xact_id: UInt): GrantToDst = { Bundle(Grant( dst = rel.client_id, is_builtin_type = Bool(true), @@ -232,7 +232,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being refilled to the original requestor */ def makeGrant( - acq: AcquireFromSrc, + acq: AcquireMetadata with HasClientId, manager_xact_id: UInt, addr_beat: UInt = UInt(0), data: UInt = UInt(0)): GrantToDst = { @@ -259,7 +259,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being refilled to the original requestor */ def makeGrant( - pri: AcquireFromSrc, + pri: AcquireMetadata with HasClientId, sec: SecondaryMissInfo, manager_xact_id: UInt, data: UInt): GrantToDst = { @@ -272,14 +272,14 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * * @param incoming the incoming [[uncore.ReleaseFromSrc]] */ - def onRelease(incoming: ReleaseFromSrc): ManagerMetadata = + def onRelease(incoming: ReleaseMetadata with HasClientId): ManagerMetadata = co.managerMetadataOnRelease(incoming, incoming.client_id, this) /** New metadata after sending a [[uncore.GrantToDst]] * * @param outgoing the outgoing [[uncore.GrantToDst]] */ - def onGrant(outgoing: GrantToDst): ManagerMetadata = + def onGrant(outgoing: GrantMetadata with HasClientId): ManagerMetadata = co.managerMetadataOnGrant(outgoing, outgoing.client_id, this) } @@ -328,8 +328,3 @@ object HierarchicalMetadata { def onReset(implicit p: Parameters): HierarchicalMetadata = apply(ManagerMetadata.onReset, ClientMetadata.onReset) } - -/** Identifies the TLId of the inner network in a hierarchical cache controller */ -case object InnerTLId extends Field[String] -/** Identifies the TLId of the outer network in a hierarchical cache controller */ -case object OuterTLId extends Field[String] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 23991083..5c02b8b7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -5,58 +5,62 @@ import Chisel._ import junctions._ import scala.math.max +case object TLId extends Field[String] +case class TLKey(id: String) extends Field[TileLinkParameters] + /** Parameters exposed to the top-level design, set based on * external requirements or design space exploration - */ -/** Unique name per TileLink network*/ -case object TLId extends Field[String] -/** Coherency policy used to define custom mesage types */ -case object TLCoherencePolicy extends Field[CoherencePolicy] -/** Number of manager agents */ -case object TLNManagers extends Field[Int] -/** Number of client agents */ -case object TLNClients extends Field[Int] -/** Number of client agents that cache data and use custom [[uncore.Acquire]] types */ -case object TLNCachingClients extends Field[Int] -/** Number of client agents that do not cache data and use built-in [[uncore.Acquire]] types */ -case object TLNCachelessClients extends Field[Int] -/** Maximum number of unique outstanding transactions per client */ -case object TLMaxClientXacts extends Field[Int] -/** Maximum number of clients multiplexed onto a single port */ -case object TLMaxClientsPerPort extends Field[Int] -/** Maximum number of unique outstanding transactions per manager */ -case object TLMaxManagerXacts extends Field[Int] -/** Width of cache block addresses */ -case object TLBlockAddrBits extends Field[Int] -/** Width of data beats */ -case object TLDataBits extends Field[Int] -/** Number of data beats per cache block */ -case object TLDataBeats extends Field[Int] -/** Whether the underlying physical network preserved point-to-point ordering of messages */ -case object TLNetworkIsOrderedP2P extends Field[Boolean] -/** Number of bits in write mask (usually one per byte in beat) */ -case object TLWriteMaskBits extends Field[Int] + * + * Coherency policy used to define custom mesage types + * Number of manager agents + * Number of client agents that cache data and use custom [[uncore.Acquire]] types + * Number of client agents that do not cache data and use built-in [[uncore.Acquire]] types + * Maximum number of unique outstanding transactions per client + * Maximum number of clients multiplexed onto a single port + * Maximum number of unique outstanding transactions per manager + * Width of cache block addresses + * Total amount of data per cache block + * Number of data beats per cache block + **/ +case class TileLinkParameters( + coherencePolicy: CoherencePolicy, + nManagers: Int, + nCachingClients: Int, + nCachelessClients: Int, + maxClientXacts: Int, + maxClientsPerPort: Int, + maxManagerXacts: Int, + addrBits: Int, + dataBits: Int, + dataBeats: Int = 4) + (val dataBitsPerBeat: Int = dataBits / dataBeats, + val writeMaskBits: Int = ((dataBits / dataBeats) - 1) / 8 + 1) { + val nClients = nCachingClients + nCachelessClients +} + + /** Utility trait for building Modules and Bundles that use TileLink parameters */ trait HasTileLinkParameters { implicit val p: Parameters - val tlCoh = p(TLCoherencePolicy) - val tlNManagers = p(TLNManagers) - val tlNClients = p(TLNClients) - val tlNCachingClients = p(TLNCachingClients) - val tlNCachelessClients = p(TLNCachelessClients) + val tlExternal = p(TLKey(p(TLId))) + val tlCoh = tlExternal.coherencePolicy + val tlNManagers = tlExternal.nManagers + val tlNCachingClients = tlExternal.nCachingClients + val tlNCachelessClients = tlExternal.nCachelessClients + val tlNClients = tlExternal.nClients val tlClientIdBits = log2Up(tlNClients) val tlManagerIdBits = log2Up(tlNManagers) - val tlMaxClientXacts = p(TLMaxClientXacts) - val tlMaxClientsPerPort = p(TLMaxClientsPerPort) - val tlMaxManagerXacts = p(TLMaxManagerXacts) + val tlMaxClientXacts = tlExternal.maxClientXacts + val tlMaxClientsPerPort = tlExternal.maxClientsPerPort + val tlMaxManagerXacts = tlExternal.maxManagerXacts val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) - val tlBlockAddrBits = p(TLBlockAddrBits) - val tlDataBits = p(TLDataBits) + val tlBlockAddrBits = tlExternal.addrBits + val tlDataBeats = tlExternal.dataBeats + val tlDataBits = tlExternal.dataBitsPerBeat val tlDataBytes = tlDataBits/8 - val tlDataBeats = p(TLDataBeats) - val tlWriteMaskBits = p(TLWriteMaskBits) + val tlWriteMaskBits = tlExternal.writeMaskBits val tlBeatAddrBits = log2Up(tlDataBeats) val tlByteAddrBits = log2Up(tlWriteMaskBits) val tlMemoryOpcodeBits = M_SZ @@ -69,7 +73,8 @@ trait HasTileLinkParameters { tlMemoryOpcodeBits)) + 1 val tlGrantTypeBits = max(log2Up(Grant.nBuiltInTypes), tlCoh.grantTypeWidth) + 1 - val tlNetworkPreservesPointToPointOrdering = p(TLNetworkIsOrderedP2P) +/** Whether the underlying physical network preserved point-to-point ordering of messages */ + val tlNetworkPreservesPointToPointOrdering = false val tlNetworkDoesNotInterleaveBeats = true val amoAluOperandBits = p(AmoAluOperandBits) } @@ -126,6 +131,11 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasMultibeatData(dummy: Int = 0): Bool } +/** An entire cache block of data */ +trait HasTileLinkBlock extends HasTileLinkParameters { + val data_buffer = Vec(tlDataBeats, UInt(width = tlDataBits)) +} + /** The id of a client source or destination. Used in managers. */ trait HasClientId extends HasTileLinkParameters { val client_id = UInt(width = tlClientIdBits) @@ -141,10 +151,10 @@ trait HasClientId extends HasTileLinkParameters { * PutAtomic built-in types. After sending an Acquire, clients must * wait for a manager to send them a [[uncore.Grant]] message in response. */ -class Acquire(implicit p: Parameters) extends ClientToManagerChannel()(p) +class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p) with HasCacheBlockAddress - with HasClientTransactionId - with HasTileLinkData { + with HasClientTransactionId + with HasTileLinkBeatId { // Actual bundle fields: val is_builtin_type = Bool() val a_type = UInt(width = tlAcquireTypeBits) @@ -221,9 +231,18 @@ class Acquire(implicit p: Parameters) extends ClientToManagerChannel()(p) } } +/** [[uncore.AcquireMetadata]] with an extra field containing the data beat */ +class Acquire(implicit p: Parameters) extends AcquireMetadata()(p) with HasTileLinkData + +/** [[uncore.AcquireMetadata]] with an extra field containing the entire cache block */ +class BufferedAcquire(implicit p: Parameters) extends AcquireMetadata()(p) with HasTileLinkBlock + /** [[uncore.Acquire]] with an extra field stating its source id */ class AcquireFromSrc(implicit p: Parameters) extends Acquire()(p) with HasClientId +/** [[uncore.BufferedAcquire]] with an extra field stating its source id */ +class BufferedAcquireFromSrc(implicit p: Parameters) extends BufferedAcquire()(p) with HasClientId + /** Contains definitions of the the built-in Acquire types and a factory * for [[uncore.Acquire]] * @@ -252,7 +271,7 @@ object Acquire { def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) - def fullWriteMask(implicit p: Parameters) = SInt(-1, width = p(TLWriteMaskBits)).toUInt + def fullWriteMask(implicit p: Parameters) = SInt(-1, width = p(TLKey(p(TLId))).writeMaskBits).toUInt // Most generic constructor def apply( @@ -550,10 +569,10 @@ object Probe { * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grant Grants]]. */ -class Release(implicit p: Parameters) extends ClientToManagerChannel()(p) +class ReleaseMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p) + with HasTileLinkBeatId with HasCacheBlockAddress - with HasClientTransactionId - with HasTileLinkData { + with HasClientTransactionId { val r_type = UInt(width = tlCoh.releaseTypeWidth) val voluntary = Bool() @@ -567,9 +586,18 @@ class Release(implicit p: Parameters) extends ClientToManagerChannel()(p) def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, UInt(0, width = tlByteAddrBits)) } +/** [[uncore.ReleaseMetadata]] with an extra field containing the data beat */ +class Release(implicit p: Parameters) extends ReleaseMetadata()(p) with HasTileLinkData + +/** [[uncore.ReleaseMetadata]] with an extra field containing the entire cache block */ +class BufferedRelease(implicit p: Parameters) extends ReleaseMetadata()(p) with HasTileLinkBlock + /** [[uncore.Release]] with an extra field stating its source id */ class ReleaseFromSrc(implicit p: Parameters) extends Release()(p) with HasClientId +/** [[uncore.BufferedRelease]] with an extra field stating its source id */ +class BufferedReleaseFromSrc(implicit p: Parameters) extends BufferedRelease()(p) with HasClientId + /** Contains a [[uncore.Release]] factory * * In general you should avoid using this factory directly and use @@ -609,8 +637,8 @@ object Release { * coherence policies may also define custom Grant types. Grants may contain data * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. */ -class Grant(implicit p: Parameters) extends ManagerToClientChannel()(p) - with HasTileLinkData +class GrantMetadata(implicit p: Parameters) extends ManagerToClientChannel()(p) + with HasTileLinkBeatId with HasClientTransactionId with HasManagerTransactionId { val is_builtin_type = Bool() @@ -636,9 +664,18 @@ class Grant(implicit p: Parameters) extends ManagerToClientChannel()(p) } } +/** [[uncore.GrantMetadata]] with an extra field containing a single beat of data */ +class Grant(implicit p: Parameters) extends GrantMetadata()(p) with HasTileLinkData + /** [[uncore.Grant]] with an extra field stating its destination */ class GrantToDst(implicit p: Parameters) extends Grant()(p) with HasClientId +/** [[uncore.GrantMetadata]] with an extra field containing an entire cache block */ +class BufferedGrant(implicit p: Parameters) extends GrantMetadata()(p) with HasTileLinkBlock + +/** [[uncore.BufferedGrant]] with an extra field stating its destination */ +class BufferedGrantToDst(implicit p: Parameters) extends BufferedGrant()(p) with HasClientId + /** Contains definitions of the the built-in grant types and factories * for [[uncore.Grant]] and [[uncore.GrantToDst]] * @@ -1491,17 +1528,24 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) data = Bits(0)) } -class TileLinkIONarrower(factor: Int) extends TLModule { - val outerParams = params.alterPartial({ - case TLDataBeats => tlDataBeats * factor - }) - val outerDataBeats = outerParams(TLDataBeats) - val outerDataBits = outerParams(TLDataBits) - val outerWriteMaskBits = outerParams(TLWriteMaskBits) +class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Parameters) extends Module { + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + require(outerDataBeats >= innerDataBeats) + require(outerDataBeats % innerDataBeats == 0) + require(outerDataBits >= innerDataBits) + + val factor = outerDataBeats / innerDataBeats val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = Bundle(new ClientUncachedTileLinkIO)(outerParams) + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) } if (factor > 1) { @@ -1511,8 +1555,8 @@ class TileLinkIONarrower(factor: Int) extends TLModule { val stretch = iacq.a_type === Acquire.putBlockType val shrink = iacq.a_type === Acquire.getBlockType - val acq_data_buffer = Reg(UInt(width = tlDataBits)) - val acq_wmask_buffer = Reg(UInt(width = tlWriteMaskBits)) + val acq_data_buffer = Reg(UInt(width = innerDataBits)) + val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) val acq_client_id = Reg(iacq.client_xact_id) val acq_addr_block = Reg(iacq.addr_block) val acq_addr_beat = Reg(iacq.addr_beat) @@ -1560,7 +1604,7 @@ class TileLinkIONarrower(factor: Int) extends TLModule { val gnt_client_id = Reg(ognt.client_xact_id) val gnt_manager_id = Reg(ognt.manager_xact_id) - val ignt_ctr = Counter(tlDataBeats) + val ignt_ctr = Counter(innerDataBeats) val ognt_ctr = Counter(factor) val sending_get = Reg(init = Bool(false)) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index cee7e392..bfdbe681 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -7,23 +7,31 @@ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] +/** Identifies the TLId of the inner network in a hierarchical cache controller */ +case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ +case object OuterTLId extends Field[String] + trait HasCoherenceAgentParameters { implicit val p: Parameters val nReleaseTransactors = 1 val nAcquireTransactors = p(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors - val outerTLParams = p.alterPartial({ case TLId => p(OuterTLId)}) - val outerDataBeats = outerTLParams(TLDataBeats) - val outerDataBits = outerTLParams(TLDataBits) + val outerTLId = p(OuterTLId) + val outerTLParams = p(TLKey(outerTLId)) + val outerDataBeats = outerTLParams.dataBeats + val outerDataBits = outerTLParams.dataBits val outerBeatAddrBits = log2Up(outerDataBeats) val outerByteAddrBits = log2Up(outerDataBits/8) - val innerTLParams = p.alterPartial({case TLId => p(InnerTLId)}) - val innerDataBeats = innerTLParams(TLDataBeats) - val innerDataBits = innerTLParams(TLDataBits) - val innerWriteMaskBits = innerTLParams(TLWriteMaskBits) + val outerWriteMaskBits = outerTLParams.writeMaskBits + val innerTLId = p(InnerTLId) + val innerTLParams = p(TLKey(innerTLId)) + val innerDataBeats = innerTLParams.dataBeats + val innerDataBits = innerTLParams.dataBits + val innerWriteMaskBits = innerTLParams.writeMaskBits val innerBeatAddrBits = log2Up(innerDataBeats) val innerByteAddrBits = log2Up(innerDataBits/8) - require(outerDataBeats == innerDataBeats) //TODO: must fix all xact_data Vecs to remove this requirement + require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement } abstract class CoherenceAgentModule(implicit val p: Parameters) extends Module @@ -52,7 +60,7 @@ trait HasCoherenceAgentWiringHelpers { } trait HasInnerTLIO extends HasCoherenceAgentParameters { - val inner = Bundle(new ManagerTileLinkIO)(innerTLParams) + val inner = Bundle(new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)}))) val incoherent = Vec(Bool(), inner.tlNCachingClients).asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits @@ -62,13 +70,13 @@ trait HasInnerTLIO extends HasCoherenceAgentParameters { } trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters { - val outer = Bundle(new ClientUncachedTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends HasCoherenceAgentParameters { - val outer = Bundle(new ClientTileLinkIO)(outerTLParams) + val outer = Bundle(new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits def orel(dummy: Int = 0) = outer.release.bits @@ -89,7 +97,7 @@ abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAg with HasCoherenceAgentWiringHelpers { val io = new ManagerTLIO def innerTL = io.inner - def outerTL = TileLinkIOWrapper(io.outer)(outerTLParams) + def outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) def incoherent = io.incoherent } From 1d362d6d3a0ddefe1770a8f9885300c0511a0b38 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 14 Oct 2015 17:58:35 -0700 Subject: [PATCH 453/688] make sure correct parameters are used for TileLink constructors --- uncore/src/main/scala/broadcast.scala | 12 +++++++----- uncore/src/main/scala/tilelink.scala | 13 +++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index ef3cbc79..912e863d 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -250,24 +250,26 @@ class BroadcastAcquireTracker(trackerId: Int) !io.irel().isVoluntary() && (state === s_probe) + val outerParams = p.alterPartial({ case TLId => outerTLId }) + val oacq_probe = PutBlock( client_xact_id = UInt(trackerId), addr_block = io.irel().addr_block, addr_beat = io.irel().addr_beat, - data = io.irel().data) + data = io.irel().data)(outerParams) val oacq_write_beat = Put( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, addr_beat = xact.addr_beat, data = xact.data_buffer(0), - wmask = xact.wmask()) + wmask = xact.wmask())(outerParams) val oacq_write_block = PutBlock( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, addr_beat = oacq_data_cnt, - data = xact.data_buffer(oacq_data_cnt)) + data = xact.data_buffer(oacq_data_cnt))(outerParams) val oacq_read_beat = Get( client_xact_id = UInt(trackerId), @@ -275,11 +277,11 @@ class BroadcastAcquireTracker(trackerId: Int) addr_beat = xact.addr_beat, addr_byte = xact.addr_byte(), operand_size = xact.op_size(), - alloc = Bool(false)) + alloc = Bool(false))(outerParams) val oacq_read_block = GetBlock( client_xact_id = UInt(trackerId), - addr_block = xact.addr_block) + addr_block = xact.addr_block)(outerParams) io.outer.acquire.valid := Bool(false) io.outer.acquire.bits := Mux(state === s_probe, oacq_probe, diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 5c02b8b7..feb930c2 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1537,9 +1537,11 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param val outerDataBeats = outerParams.dataBeats val outerDataBits = outerParams.dataBitsPerBeat val outerWriteMaskBits = outerParams.writeMaskBits + require(outerDataBeats >= innerDataBeats) require(outerDataBeats % innerDataBeats == 0) - require(outerDataBits >= innerDataBits) + require(outerDataBits <= innerDataBits) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) val factor = outerDataBeats / innerDataBeats @@ -1562,10 +1564,13 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param val acq_addr_beat = Reg(iacq.addr_beat) val oacq_ctr = Counter(factor) + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + val get_block_acquire = GetBlock( client_xact_id = iacq.client_xact_id, addr_block = iacq.addr_block, - alloc = iacq.allocate()) + alloc = iacq.allocate())(outerConfig) val put_block_acquire = PutBlock( client_xact_id = acq_client_id, @@ -1574,7 +1579,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param Cat(acq_addr_beat, oacq_ctr.value) else acq_addr_beat, data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0)) + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) val sending_put = Reg(init = Bool(false)) @@ -1614,7 +1619,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param client_xact_id = gnt_client_id, manager_xact_id = gnt_manager_id, addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits) + data = gnt_data_buffer.toBits)(innerConfig) io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) From 49667aa4b049917b136349e8bb5c66550604e303 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 14 Oct 2015 18:56:13 -0700 Subject: [PATCH 454/688] make sure broadcast acquire tracker doesn't try to send requests back-to-back --- uncore/src/main/scala/broadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 912e863d..b7b4f9a7 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -377,7 +377,7 @@ class BroadcastAcquireTracker(trackerId: Int) } } is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := !pending_ognt_ack || !collect_iacq_data || iacq_data_valid(oacq_data_cnt) + io.outer.acquire.valid := !pending_ognt_ack && (!collect_iacq_data || iacq_data_valid(oacq_data_cnt)) when(oacq_data_done) { pending_ognt_ack := Bool(true) state := Mux(pending_outer_read, s_mem_read, s_mem_resp) From e1f573918de245a36d807b9d751f8f585c52c398 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 16 Oct 2015 18:24:02 -0700 Subject: [PATCH 455/688] simplify TileLinkParameters with Option --- uncore/src/main/scala/broadcast.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index b7b4f9a7..7570d37b 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -32,8 +32,8 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) val usingStoreDataQueue = p.alterPartial({ - case TLKey(`innerTLId`) => innerTLParams.copy()(internalDataBits, innerWriteMaskBits) - case TLKey(`outerTLId`) => outerTLParams.copy()(internalDataBits, outerWriteMaskBits) + case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) + case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) }) // Create SHRs for outstanding transactions diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index feb930c2..7d7c7ffd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -33,10 +33,12 @@ case class TileLinkParameters( maxManagerXacts: Int, addrBits: Int, dataBits: Int, - dataBeats: Int = 4) - (val dataBitsPerBeat: Int = dataBits / dataBeats, - val writeMaskBits: Int = ((dataBits / dataBeats) - 1) / 8 + 1) { + dataBeats: Int = 4, + overrideDataBitsPerBeat: Option[Int] = None + ) { val nClients = nCachingClients + nCachelessClients + val writeMaskBits: Int = ((dataBits / dataBeats) - 1) / 8 + 1 + val dataBitsPerBeat: Int = overrideDataBitsPerBeat.getOrElse(dataBits / dataBeats) } From d391f979531be44d5985353042ee792f4e231ef6 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 16 Oct 2015 19:11:06 -0700 Subject: [PATCH 456/688] Minor refactor of StoreGen/AMOALU. Bugfix for 32b ops in L2's AMOALU. --- uncore/src/main/scala/amoalu.scala | 123 +++++++++++++++++++++++++++++ uncore/src/main/scala/cache.scala | 77 +----------------- 2 files changed, 125 insertions(+), 75 deletions(-) create mode 100644 uncore/src/main/scala/amoalu.scala diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala new file mode 100644 index 00000000..e3250d34 --- /dev/null +++ b/uncore/src/main/scala/amoalu.scala @@ -0,0 +1,123 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ + +abstract class StoreGen(typ: UInt, addr: UInt, dat: UInt) { + val byte = typ === MT_B || typ === MT_BU + val half = typ === MT_H || typ === MT_HU + val word = typ === MT_W || typ === MT_WU + + def mask: UInt + def data: UInt + def wordData: UInt +} + +class StoreGen64(typ: UInt, addr: UInt, dat: UInt) extends StoreGen(typ, addr, dat) { + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), + Bits(255)))) + + def data = + Mux(byte, Fill(8, dat( 7,0)), + Mux(half, Fill(4, dat(15,0)), + wordData)) + def wordData = + Mux(word, Fill(2, dat(31,0)), + dat) +} + +class StoreGenAligned64(typ: UInt, addr: UInt, dat: UInt) extends StoreGen64(typ, addr, dat) { + override def data = dat + override def wordData = dat +} + +class StoreGen32(typ: UInt, addr: UInt, dat: UInt) extends StoreGen(typ, addr, dat){ + override val word = typ === MT_W + + def mask = + Mux(byte, Bits( 1) << addr(2,0), + Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), + Bits( 15))) + + def data = + Mux(byte, Fill(4, dat( 7,0)), + Mux(half, Fill(2, dat(15,0)), + wordData)) + + def wordData = dat + + def size = + Mux(byte, UInt("b000"), + Mux(half, UInt("b001"), + UInt("b010"))) +} + +class LoadGen64(typ: UInt, addr: UInt, dat: UInt, zero: Bool) { + val t = new StoreGen64(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D + + val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) + val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) + val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) + val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) +} + +class LoadGen32(typ: UInt, addr: UInt, dat: UInt) { + val t = new StoreGen32(typ, addr, dat) + val sign = typ === MT_B || typ === MT_H || typ === MT_W + + val word = dat + val halfShift = Mux(addr(1), word(31,16), word(15,0)) + val half = Cat(Mux(t.half, Fill(16, sign && halfShift(15)), word(31,16)), halfShift) + val byteShift = Mux(addr(0), half(15,8), half(7,0)) + val byte = Cat(Mux(t.byte, Fill(24, sign && byteShift(7)), half(31,8)), byteShift) +} + +class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends CacheModule()(p) { + val operandBits = p(AmoAluOperandBits) + require(operandBits == 64) + val io = new Bundle { + val addr = Bits(INPUT, blockOffBits) + val cmd = Bits(INPUT, M_SZ) + val typ = Bits(INPUT, MT_SZ) + val lhs = Bits(INPUT, operandBits) + val rhs = Bits(INPUT, operandBits) + val out = Bits(OUTPUT, operandBits) + } + + val storegen = if(rhsIsAligned) new StoreGenAligned64(io.typ, io.addr, io.rhs) + else new StoreGen64(io.typ, io.addr, io.rhs) + val rhs = storegen.wordData + + val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX + val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU + val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU + val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: + io.typ === MT_B || io.typ === MT_BU + + val mask = ~UInt(0,64) ^ (io.addr(2) << 31) + val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + + val out = Mux(io.cmd === M_XA_ADD, adder_out, + Mux(io.cmd === M_XA_AND, io.lhs & rhs, + Mux(io.cmd === M_XA_OR, io.lhs | rhs, + Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, + Mux(Mux(less, min, max), io.lhs, + storegen.data))))) + + val wmask = FillInterleaved(8, storegen.mask) + io.out := wmask & out | ~wmask & io.lhs +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 143f68e1..f0ada529 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -38,79 +38,6 @@ abstract class CacheModule(implicit val p: Parameters) extends Module abstract class CacheBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasCacheParameters -class StoreGen(typ: UInt, addr: UInt, dat: UInt) { - val byte = typ === MT_B || typ === MT_BU - val half = typ === MT_H || typ === MT_HU - val word = typ === MT_W || typ === MT_WU - def mask = - Mux(byte, Bits( 1) << addr(2,0), - Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), - Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), - Bits(255)))) - def data = - Mux(byte, Fill(8, dat( 7,0)), - Mux(half, Fill(4, dat(15,0)), - wordData)) - lazy val wordData = - Mux(word, Fill(2, dat(31,0)), - dat) -} - -class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool) { - val t = new StoreGen(typ, addr, dat) - val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D - - val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) - val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) - val halfShift = Mux(addr(1), word(31,16), word(15,0)) - val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) - val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) - val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) -} - -class AMOALU(implicit p: Parameters) extends CacheModule()(p) { - val operandBits = p(AmoAluOperandBits) - require(operandBits == 64) - val io = new Bundle { - val addr = Bits(INPUT, blockOffBits) - val cmd = Bits(INPUT, M_SZ) - val typ = Bits(INPUT, MT_SZ) - val lhs = Bits(INPUT, operandBits) - val rhs = Bits(INPUT, operandBits) - val out = Bits(OUTPUT, operandBits) - } - - val storegen = new StoreGen(io.typ, io.addr, io.rhs) - val rhs = storegen.wordData - - val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX - val max = io.cmd === M_XA_MAX || io.cmd === M_XA_MAXU - val min = io.cmd === M_XA_MIN || io.cmd === M_XA_MINU - val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: - io.typ === MT_B || io.typ === MT_BU - - val mask = ~UInt(0,64) ^ (io.addr(2) << 31) - val adder_out = (io.lhs & mask).toUInt + (rhs & mask) - - val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) - val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) - val lt_lo = io.lhs(31,0) < rhs(31,0) - val lt_hi = io.lhs(63,32) < rhs(63,32) - val eq_hi = io.lhs(63,32) === rhs(63,32) - val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) - val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) - - val out = Mux(io.cmd === M_XA_ADD, adder_out, - Mux(io.cmd === M_XA_AND, io.lhs & rhs, - Mux(io.cmd === M_XA_OR, io.lhs | rhs, - Mux(io.cmd === M_XA_XOR, io.lhs ^ rhs, - Mux(Mux(less, min, max), io.lhs, - storegen.data))))) - - val wmask = FillInterleaved(8, storegen.mask) - io.out := wmask & out | ~wmask & io.lhs -} - abstract class ReplacementPolicy { def way: UInt def miss: Unit @@ -652,12 +579,12 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Provide a single ALU per tracker to merge Puts and AMOs with data being // refilled, written back, or extant in the cache - val amoalu = Module(new AMOALU) + val amoalu = Module(new AMOALU(rhsIsAligned = true)) amoalu.io.addr := xact.full_addr() amoalu.io.cmd := xact.op_code() amoalu.io.typ := xact.op_size() amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData - amoalu.io.rhs := xact.data_buffer.head // default, overwritten by calls to mergeData + amoalu.io.rhs := xact.data_buffer.head // default, overwritten by calls to mergeData val amo_result = Reg(init = UInt(0, xact.tlDataBits)) // Utility functions for updating the data and metadata that will be kept in From d12403e7dc8eba10a3e0246eb7b50d73b2872d74 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 19 Oct 2015 13:47:13 -0700 Subject: [PATCH 457/688] fix up and simplify TL -> NASTI converter logic --- uncore/src/main/scala/tilelink.scala | 98 ++++++++++++---------------- 1 file changed, 43 insertions(+), 55 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7d7c7ffd..069f0dc9 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1418,20 +1418,16 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.w.valid := Bool(false) val dst_off = dstIdBits + tlClientXactIdBits - val acq_has_data = io.tl.acquire.bits.hasData() - val is_write = io.tl.acquire.valid && acq_has_data + val has_data = io.tl.acquire.bits.hasData() + val is_write = io.tl.acquire.valid && has_data - // Decompose outgoing TL Acquires into Nasti address and data channels - val active_out = Reg(init=Bool(false)) - val cmd_sent_out = Reg(init=Bool(false)) - val tag_out = Reg(UInt(width = nastiXIdBits)) - val addr_out = Reg(UInt(width = nastiXAddrBits)) - val has_data = Reg(init=Bool(false)) val is_subblock = io.tl.acquire.bits.isSubBlockType() + val is_multibeat = io.tl.acquire.bits.hasMultibeatData() val (tl_cnt_out, tl_wrap_out) = Counter( - io.tl.acquire.fire() && io.tl.acquire.bits.hasMultibeatData(), tlDataBeats) - val tl_done_out = Reg(init=Bool(false)) + io.tl.acquire.fire() && is_multibeat, tlDataBeats) + // Reorder queue saves extra information needed to send correct + // grant back to TL client val roq = Module(new ReorderQueue( new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxClientsPerPort)) @@ -1439,68 +1435,60 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val (nasti_cnt_out, nasti_wrap_out) = Counter( io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - roq.io.enq.valid := io.tl.acquire.fire() && !acq_has_data + roq.io.enq.valid := io.tl.acquire.fire() && !has_data roq.io.enq.bits.tag := io.nasti.ar.bits.id roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() roq.io.enq.bits.data.subblock := is_subblock roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) roq.io.deq.tag := io.nasti.r.bits.id + // Decompose outgoing TL Acquires into Nasti address and data channels io.nasti.ar.bits := NastiReadAddressChannel( - id = tag_out, - addr = addr_out, + id = io.tl.acquire.bits.client_xact_id, + addr = io.tl.acquire.bits.full_addr(), + size = Mux(is_subblock, + opSizeToXSize(io.tl.acquire.bits.op_size()), + UInt(log2Ceil(tlDataBytes))), + len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + + io.nasti.aw.bits := NastiWriteAddressChannel( + id = io.tl.acquire.bits.client_xact_id, + addr = io.tl.acquire.bits.full_addr(), size = UInt(log2Ceil(tlDataBytes)), - len = Mux(has_data, UInt(tlDataBeats - 1), UInt(0))) - io.nasti.aw.bits := io.nasti.ar.bits + len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) + io.nasti.w.bits := NastiWriteDataChannel( data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) - when(!active_out){ - io.tl.acquire.ready := io.nasti.w.ready - io.nasti.w.valid := is_write - when(io.tl.acquire.fire()) { - active_out := (!is_write && !io.nasti.ar.ready) || - (is_write && !(io.nasti.aw.ready && io.nasti.w.ready)) || - (io.nasti.w.valid && Bool(tlDataBeats > 1)) - io.nasti.aw.valid := is_write - io.nasti.ar.valid := !is_write - cmd_sent_out := (!is_write && io.nasti.ar.ready) || (is_write && io.nasti.aw.ready) - val tag = io.tl.acquire.bits.client_xact_id - val addr = io.tl.acquire.bits.full_addr() - when(is_write) { - io.nasti.aw.bits.id := tag - io.nasti.aw.bits.addr := addr - io.nasti.aw.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) - } .otherwise { - io.nasti.ar.bits.id := tag - io.nasti.ar.bits.addr := addr - io.nasti.ar.bits.len := Mux(!is_subblock, UInt(tlDataBeats-1), UInt(0)) - io.nasti.ar.bits.size := opSizeToXSize(io.tl.acquire.bits.op_size()) - } - tag_out := tag - addr_out := addr - has_data := acq_has_data - tl_done_out := tl_wrap_out || is_subblock - } - } - when(active_out) { - io.nasti.ar.valid := !cmd_sent_out && !has_data - io.nasti.aw.valid := !cmd_sent_out && has_data - cmd_sent_out := cmd_sent_out || io.nasti.ar.fire() || io.nasti.aw.fire() - when(has_data && !tl_done_out) { - io.tl.acquire.ready := io.nasti.w.ready - io.nasti.w.valid := io.tl.acquire.valid - } - when(tl_wrap_out) { tl_done_out := Bool(true) } - when(cmd_sent_out && roq.io.enq.ready && (!has_data || tl_done_out)) { - active_out := Bool(false) + val w_inflight = Reg(init = Bool(false)) + + when (!w_inflight && io.tl.acquire.valid) { + when (has_data) { + // For Put/PutBlock, make sure aw and w channel are both ready before + // we send the first beat + io.tl.acquire.ready := io.nasti.aw.ready && io.nasti.w.ready + io.nasti.aw.valid := io.nasti.w.ready + io.nasti.w.valid := io.nasti.aw.ready + // For Putblock, use a different state for the subsequent beats + when (io.tl.acquire.ready && is_multibeat) { w_inflight := Bool(true) } + } .otherwise { + // For Get/GetBlock, make sure Reorder queue can accept new entry + io.tl.acquire.ready := io.nasti.ar.ready && roq.io.enq.ready + io.nasti.ar.valid := roq.io.enq.ready } } + when (w_inflight) { + io.nasti.w.valid := io.tl.acquire.valid + io.tl.acquire.ready := io.nasti.w.ready + when (tl_wrap_out) { w_inflight := Bool(false) } + } + // Aggregate incoming NASTI responses into TL Grants - val (tl_cnt_in, tl_wrap_in) = Counter(io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val (tl_cnt_in, tl_wrap_in) = Counter( + io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out From 4389b9edb0dbb4020c21ccb9e976930b40d29145 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 20 Oct 2015 14:54:36 -0700 Subject: [PATCH 458/688] tilelink parameter tweak: addrBits now a constant --- uncore/src/main/scala/tilelink.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 069f0dc9..1a477bc7 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -31,7 +31,6 @@ case class TileLinkParameters( maxClientXacts: Int, maxClientsPerPort: Int, maxManagerXacts: Int, - addrBits: Int, dataBits: Int, dataBeats: Int = 4, overrideDataBitsPerBeat: Option[Int] = None @@ -58,7 +57,7 @@ trait HasTileLinkParameters { val tlMaxManagerXacts = tlExternal.maxManagerXacts val tlClientXactIdBits = log2Up(tlMaxClientXacts*tlMaxClientsPerPort) val tlManagerXactIdBits = log2Up(tlMaxManagerXacts) - val tlBlockAddrBits = tlExternal.addrBits + val tlBlockAddrBits = p(PAddrBits) - p(CacheBlockOffsetBits) val tlDataBeats = tlExternal.dataBeats val tlDataBits = tlExternal.dataBitsPerBeat val tlDataBytes = tlDataBits/8 From 1c135c162881faa8fbc2f9ca8eb7bfd765656f27 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 19 Oct 2015 17:25:33 -0700 Subject: [PATCH 459/688] fix ready-valid mixup in TileLink unwrapper --- uncore/src/main/scala/tilelink.scala | 29 ++++++++++++++++++++-------- uncore/src/main/scala/util.scala | 10 ++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1a477bc7..8748d637 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1320,15 +1320,28 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val irel = io.in.release.bits val ognt = io.out.grant.bits - val iacq_wait = needsRoqEnq(iacq) && !roqArb.io.in(0).ready - val irel_wait = needsRoqEnq(irel) && !roqArb.io.in(1).ready + val acq_roq_enq = needsRoqEnq(iacq) + val rel_roq_enq = needsRoqEnq(irel) - roqArb.io.in(0).valid := io.in.acquire.valid && needsRoqEnq(iacq) + val acq_roq_ready = !acq_roq_enq || roqArb.io.in(0).ready + val rel_roq_ready = !rel_roq_enq || roqArb.io.in(1).ready + + val acq_helper = DecoupledHelper( + io.in.acquire.valid, + acq_roq_ready, + acqArb.io.in(0).ready) + + val rel_helper = DecoupledHelper( + io.in.release.valid, + rel_roq_ready, + acqArb.io.in(1).ready) + + roqArb.io.in(0).valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) roqArb.io.in(0).bits.data.voluntary := Bool(false) roqArb.io.in(0).bits.data.builtin := iacq.isBuiltInType() roqArb.io.in(0).bits.tag := iacq.client_xact_id - acqArb.io.in(0).valid := io.in.acquire.valid && !iacq_wait + acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) acqArb.io.in(0).bits := Acquire( is_builtin_type = Bool(true), a_type = Mux(iacq.isBuiltInType(), @@ -1339,21 +1352,21 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { data = iacq.data, union = Mux(iacq.isBuiltInType(), iacq.union, Cat(Acquire.fullWriteMask, Bool(false)))) - io.in.acquire.ready := acqArb.io.in(0).ready && !iacq_wait + io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) - roqArb.io.in(1).valid := io.in.release.valid && needsRoqEnq(iacq) + roqArb.io.in(1).valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) roqArb.io.in(1).bits.data.voluntary := irel.isVoluntary() roqArb.io.in(1).bits.data.builtin := Bool(true) roqArb.io.in(1).bits.tag := irel.client_xact_id - acqArb.io.in(1).valid := io.in.release.valid && !irel_wait + acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) acqArb.io.in(1).bits := PutBlock( client_xact_id = irel.client_xact_id, addr_block = irel.addr_block, addr_beat = irel.addr_beat, data = irel.data, wmask = Acquire.fullWriteMask) - io.in.release.ready := acqArb.io.in(1).ready && !irel_wait + io.in.release.ready := rel_helper.fire(io.in.release.valid) io.out.acquire <> acqArb.io.out diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index bf05d9d3..ce7bcf12 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -145,3 +145,13 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) roq_free(roq_deq_addr) := Bool(true) } } + +object DecoupledHelper { + def apply(rvs: Bool*) = new DecoupledHelper(rvs) +} + +class DecoupledHelper(val rvs: Seq[Bool]) { + def fire(exclude: Bool, includes: Bool*) = { + (rvs.filter(_ ne exclude) ++ includes).reduce(_ && _) + } +} From ffe7df2fed9ddb4de05420baa4d2fccfe55cf508 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 19 Oct 2015 22:46:03 -0700 Subject: [PATCH 460/688] make sure TL -> NASTI converter acquire ready not dependent on valid --- uncore/src/main/scala/tilelink.scala | 63 ++++++++++++++-------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 8748d637..e81b2340 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1421,33 +1421,43 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NastiIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) - io.tl.acquire.ready := Bool(false) - - io.nasti.b.ready := Bool(false) - io.nasti.r.ready := Bool(false) - io.nasti.ar.valid := Bool(false) - io.nasti.aw.valid := Bool(false) - io.nasti.w.valid := Bool(false) - val dst_off = dstIdBits + tlClientXactIdBits val has_data = io.tl.acquire.bits.hasData() - val is_write = io.tl.acquire.valid && has_data val is_subblock = io.tl.acquire.bits.isSubBlockType() val is_multibeat = io.tl.acquire.bits.hasMultibeatData() val (tl_cnt_out, tl_wrap_out) = Counter( io.tl.acquire.fire() && is_multibeat, tlDataBeats) + val get_valid = io.tl.acquire.valid && !has_data + val put_valid = io.tl.acquire.valid && has_data + // Reorder queue saves extra information needed to send correct // grant back to TL client val roq = Module(new ReorderQueue( new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxClientsPerPort)) + // For Get/GetBlock, make sure Reorder queue can accept new entry + val get_helper = DecoupledHelper( + get_valid, + roq.io.enq.ready, + io.nasti.ar.ready) + + val w_inflight = Reg(init = Bool(false)) + + // For Put/PutBlock, make sure aw and w channel are both ready before + // we send the first beat + val aw_ready = w_inflight || io.nasti.aw.ready + val put_helper = DecoupledHelper( + put_valid, + aw_ready, + io.nasti.w.ready) + val (nasti_cnt_out, nasti_wrap_out) = Counter( io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - roq.io.enq.valid := io.tl.acquire.fire() && !has_data + roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) roq.io.enq.bits.tag := io.nasti.ar.bits.id roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() roq.io.enq.bits.data.subblock := is_subblock @@ -1455,46 +1465,37 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) roq.io.deq.tag := io.nasti.r.bits.id // Decompose outgoing TL Acquires into Nasti address and data channels + io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) io.nasti.ar.bits := NastiReadAddressChannel( id = io.tl.acquire.bits.client_xact_id, - addr = io.tl.acquire.bits.full_addr(), - size = Mux(is_subblock, - opSizeToXSize(io.tl.acquire.bits.op_size()), + addr = io.tl.acquire.bits.full_addr(), + size = Mux(is_subblock, + opSizeToXSize(io.tl.acquire.bits.op_size()), UInt(log2Ceil(tlDataBytes))), len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) io.nasti.aw.bits := NastiWriteAddressChannel( id = io.tl.acquire.bits.client_xact_id, - addr = io.tl.acquire.bits.full_addr(), + addr = io.tl.acquire.bits.full_addr(), size = UInt(log2Ceil(tlDataBytes)), len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) + io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) io.nasti.w.bits := NastiWriteDataChannel( data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) - val w_inflight = Reg(init = Bool(false)) + io.tl.acquire.ready := Mux(has_data, + put_helper.fire(put_valid), + get_helper.fire(get_valid)) - when (!w_inflight && io.tl.acquire.valid) { - when (has_data) { - // For Put/PutBlock, make sure aw and w channel are both ready before - // we send the first beat - io.tl.acquire.ready := io.nasti.aw.ready && io.nasti.w.ready - io.nasti.aw.valid := io.nasti.w.ready - io.nasti.w.valid := io.nasti.aw.ready - // For Putblock, use a different state for the subsequent beats - when (io.tl.acquire.ready && is_multibeat) { w_inflight := Bool(true) } - } .otherwise { - // For Get/GetBlock, make sure Reorder queue can accept new entry - io.tl.acquire.ready := io.nasti.ar.ready && roq.io.enq.ready - io.nasti.ar.valid := roq.io.enq.ready - } + when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { + w_inflight := Bool(true) } when (w_inflight) { - io.nasti.w.valid := io.tl.acquire.valid - io.tl.acquire.ready := io.nasti.w.ready when (tl_wrap_out) { w_inflight := Bool(false) } } From baf95533a4facf00cbaf0ccaafda8c4d7fc19b28 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 20 Oct 2015 23:26:11 -0700 Subject: [PATCH 461/688] fix combinational loop in TileLink Unwrapper --- uncore/src/main/scala/tilelink.scala | 60 ++++++++++++++++------------ uncore/src/main/scala/util.scala | 5 ++- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index e81b2340..0b9c296d 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1313,8 +1313,12 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, Some((acq: Acquire) => acq.hasMultibeatData()))) - val roqArb = Module(new RRArbiter(new ReorderQueueWrite( - new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits), 2)) + + val acqRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val relRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) val iacq = io.in.acquire.bits val irel = io.in.release.bits @@ -1323,8 +1327,8 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val acq_roq_enq = needsRoqEnq(iacq) val rel_roq_enq = needsRoqEnq(irel) - val acq_roq_ready = !acq_roq_enq || roqArb.io.in(0).ready - val rel_roq_ready = !rel_roq_enq || roqArb.io.in(1).ready + val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready + val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready val acq_helper = DecoupledHelper( io.in.acquire.valid, @@ -1336,10 +1340,9 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { rel_roq_ready, acqArb.io.in(1).ready) - roqArb.io.in(0).valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) - roqArb.io.in(0).bits.data.voluntary := Bool(false) - roqArb.io.in(0).bits.data.builtin := iacq.isBuiltInType() - roqArb.io.in(0).bits.tag := iacq.client_xact_id + acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) + acqRoq.io.enq.bits.data := iacq.isBuiltInType() + acqRoq.io.enq.bits.tag := iacq.client_xact_id acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) acqArb.io.in(0).bits := Acquire( @@ -1351,13 +1354,12 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { addr_beat = iacq.addr_beat, data = iacq.data, union = Mux(iacq.isBuiltInType(), - iacq.union, Cat(Acquire.fullWriteMask, Bool(false)))) + iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) - roqArb.io.in(1).valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) - roqArb.io.in(1).bits.data.voluntary := irel.isVoluntary() - roqArb.io.in(1).bits.data.builtin := Bool(true) - roqArb.io.in(1).bits.tag := irel.client_xact_id + relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) + relRoq.io.enq.bits.data := irel.isVoluntary() + relRoq.io.enq.bits.tag := irel.client_xact_id acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) acqArb.io.in(1).bits := PutBlock( @@ -1370,25 +1372,33 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { io.out.acquire <> acqArb.io.out - val roq = Module(new ReorderQueue( - new ClientTileLinkIOUnwrapperInfo, tlClientXactIdBits, tlMaxClientsPerPort)) - roq.io.enq <> roqArb.io.out - roq.io.deq.valid := io.out.grant.valid && needsRoqDeq(ognt) - roq.io.deq.tag := ognt.client_xact_id + acqRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + acqRoq.io.deq.tag := ognt.client_xact_id - val gnt_builtin = roq.io.deq.data.builtin - val gnt_voluntary = roq.io.deq.data.voluntary + relRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + relRoq.io.deq.tag := ognt.client_xact_id - io.in.grant.valid := io.out.grant.valid - io.in.grant.bits := Grant( + val gnt_builtin = acqRoq.io.deq.data + val gnt_voluntary = relRoq.io.deq.data + + val acq_grant = Grant( is_builtin_type = gnt_builtin, - g_type = MuxCase(ognt.g_type, Seq( - (!gnt_builtin, tlCoh.getExclusiveGrantType), - (gnt_voluntary, Grant.voluntaryAckType))), + g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, addr_beat = ognt.addr_beat, data = ognt.data) + + val rel_grant = Grant( + is_builtin_type = Bool(true), + g_type = Mux(gnt_voluntary, Grant.voluntaryAckType, ognt.g_type), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := io.out.grant.valid + io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) io.out.grant.ready := io.in.grant.ready io.in.probe.valid := Bool(false) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index ce7bcf12..80310966 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -121,6 +121,7 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val valid = Bool(INPUT) val tag = UInt(INPUT, tagWidth) val data = dType.cloneType.asOutput + val matches = Bool(OUTPUT) } val full = Bool(OUTPUT) } @@ -130,10 +131,12 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val roq_free = Reg(init = Vec.fill(size)(Bool(true))) val roq_enq_addr = PriorityEncoder(roq_free) - val roq_deq_addr = PriorityEncoder(roq_tags.map(_ === io.deq.tag)) + val roq_matches = roq_tags.map(_ === io.deq.tag) + val roq_deq_addr = PriorityEncoder(roq_matches) io.enq.ready := roq_free.reduce(_ || _) io.deq.data := roq_data(roq_deq_addr) + io.deq.matches := roq_matches.reduce(_ || _) when (io.enq.valid && io.enq.ready) { roq_data(roq_enq_addr) := io.enq.bits.data From 02d113b39fda1f47e141068e600a384a6f998e5f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 21 Oct 2015 11:31:13 -0700 Subject: [PATCH 462/688] outerDataBits / innerDataBits should be per beat, not per block --- uncore/src/main/scala/uncore.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index bfdbe681..d8f0683e 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -20,14 +20,14 @@ trait HasCoherenceAgentParameters { val outerTLId = p(OuterTLId) val outerTLParams = p(TLKey(outerTLId)) val outerDataBeats = outerTLParams.dataBeats - val outerDataBits = outerTLParams.dataBits + val outerDataBits = outerTLParams.dataBitsPerBeat val outerBeatAddrBits = log2Up(outerDataBeats) val outerByteAddrBits = log2Up(outerDataBits/8) val outerWriteMaskBits = outerTLParams.writeMaskBits val innerTLId = p(InnerTLId) val innerTLParams = p(TLKey(innerTLId)) val innerDataBeats = innerTLParams.dataBeats - val innerDataBits = innerTLParams.dataBits + val innerDataBits = innerTLParams.dataBitsPerBeat val innerWriteMaskBits = innerTLParams.writeMaskBits val innerBeatAddrBits = log2Up(innerDataBeats) val innerByteAddrBits = log2Up(innerDataBits/8) From f8594da1d32a5496c1e39cbb6c03a35bd51162ec Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 21 Oct 2015 18:16:44 -0700 Subject: [PATCH 463/688] depend on external cde library --- uncore/src/main/scala/amoalu.scala | 1 + uncore/src/main/scala/broadcast.scala | 1 + uncore/src/main/scala/cache.scala | 5 +- uncore/src/main/scala/htif.scala | 1 + uncore/src/main/scala/metadata.scala | 90 +++++++++++++-------------- uncore/src/main/scala/network.scala | 1 + uncore/src/main/scala/rtc.scala | 1 + uncore/src/main/scala/scr.scala | 1 + uncore/src/main/scala/tilelink.scala | 3 +- uncore/src/main/scala/uncore.scala | 7 ++- 10 files changed, 58 insertions(+), 53 deletions(-) diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala index e3250d34..033873fa 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/amoalu.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import cde.{Parameters, Field} abstract class StoreGen(typ: UInt, addr: UInt, dat: UInt) { val byte = typ === MT_B || typ === MT_BU diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 7570d37b..39aac120 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import cde.{Parameters, Field} case object L2StoreDataQueueDepth extends Field[Int] diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index f0ada529..20010406 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -4,6 +4,7 @@ package uncore import Chisel._ import scala.reflect.ClassTag import junctions._ +import cde.{Parameters, Field} case object CacheName extends Field[String] case object NSets extends Field[Int] @@ -436,7 +437,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) val state = Reg(init=s_idle) - val xact = Reg(Bundle(new BufferedReleaseFromSrc()(p.alterPartial({case TLId => p(InnerTLId)})))) + val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({case TLId => p(InnerTLId)}))) val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh @@ -530,7 +531,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val state = Reg(init=s_idle) // State holding transaction metadata - val xact = Reg(Bundle(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => p(InnerTLId) })))) + val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => p(InnerTLId) }))) val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index fedadf71..8dc0216c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -5,6 +5,7 @@ package uncore import Chisel._ import Chisel.ImplicitConversions._ import junctions._ +import cde.{Parameters, Field} case object HtifKey extends Field[HtifParameters] diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index ca8dda50..5a514941 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import cde.{Parameters, Field} /** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata(implicit p: Parameters) extends TLBundle()(p) { @@ -44,16 +45,15 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param op_code a memory operation from [[uncore.constants.MemoryOpConstants]] */ def makeAcquire( - op_code: UInt, - client_xact_id: UInt, - addr_block: UInt): Acquire = { - Bundle(Acquire( - is_builtin_type = Bool(false), - a_type = co.getAcquireType(op_code, this), - client_xact_id = client_xact_id, - addr_block = addr_block, - union = Cat(op_code, Bool(true)))(p)) - } + op_code: UInt, + client_xact_id: UInt, + addr_block: UInt): Acquire = + Acquire( + is_builtin_type = Bool(false), + a_type = co.getAcquireType(op_code, this), + client_xact_id = client_xact_id, + addr_block = addr_block, + union = Cat(op_code, Bool(true)))(p) /** Constructs a Release message based on this metadata on cache control op * @@ -63,19 +63,18 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being written back */ def makeVoluntaryRelease( - op_code: UInt, - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Bundle(Release( + op_code: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = + Release( voluntary = Bool(true), r_type = co.getReleaseType(op_code, this), client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - data = data)(p)) - } + data = data)(p) /** Constructs a Release message based on this metadata on an eviction * @@ -85,10 +84,10 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being written back */ def makeVoluntaryWriteback( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = makeVoluntaryRelease( op_code = M_FLUSH, client_xact_id = client_xact_id, @@ -103,17 +102,16 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being released */ def makeRelease( - prb: Probe, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): Release = { - Bundle(Release( + prb: Probe, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): Release = + Release( voluntary = Bool(false), r_type = co.getReleaseType(prb, this), client_xact_id = UInt(0), addr_block = prb.addr_block, addr_beat = addr_beat, - data = data)(p)) - } + data = data)(p) /** New metadata after receiving a [[uncore.Grant]] * @@ -189,7 +187,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param acq Acquire message triggering this Probe */ def makeProbe(dst: UInt, acq: AcquireMetadata): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p)) + Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p) /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op * @@ -198,7 +196,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param addr_block address of the cache block being probed */ def makeProbe(dst: UInt, op_code: UInt, addr_block: UInt): ProbeToDst = - Bundle(Probe(dst, co.getProbeType(op_code, this), addr_block)(p)) + Probe(dst, co.getProbeType(op_code, this), addr_block)(p) /** Construct an appropriate [[uncore.ProbeToDst]] for an eviction * @@ -213,14 +211,13 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param rel Release message being acknowledged by this Grant * @param manager_xact_id manager's transaction id */ - def makeGrant(rel: ReleaseMetadata with HasClientId, manager_xact_id: UInt): GrantToDst = { - Bundle(Grant( + def makeGrant(rel: ReleaseMetadata with HasClientId, manager_xact_id: UInt): GrantToDst = + Grant( dst = rel.client_id, is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, - manager_xact_id = manager_xact_id)(p)) - } + manager_xact_id = manager_xact_id)(p) /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] * @@ -232,11 +229,11 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being refilled to the original requestor */ def makeGrant( - acq: AcquireMetadata with HasClientId, - manager_xact_id: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)): GrantToDst = { - Bundle(Grant( + acq: AcquireMetadata with HasClientId, + manager_xact_id: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)): GrantToDst = + Grant( dst = acq.client_id, is_builtin_type = acq.isBuiltInType(), g_type = Mux(acq.isBuiltInType(), @@ -245,8 +242,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { client_xact_id = acq.client_xact_id, manager_xact_id = manager_xact_id, addr_beat = addr_beat, - data = data)(p)) - } + data = data)(p) /** Construct an [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] with some overrides * @@ -259,10 +255,10 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * @param data data being refilled to the original requestor */ def makeGrant( - pri: AcquireMetadata with HasClientId, - sec: SecondaryMissInfo, - manager_xact_id: UInt, - data: UInt): GrantToDst = { + pri: AcquireMetadata with HasClientId, + sec: SecondaryMissInfo, + manager_xact_id: UInt, + data: UInt): GrantToDst = { val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) g.client_xact_id := sec.client_xact_id g @@ -309,8 +305,8 @@ object ManagerMetadata { * [[uncore.InnerTLId]] or [[uncore.OuterTLId]]. */ class HierarchicalMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { - val inner: ManagerMetadata = Bundle(new ManagerMetadata()(p.alterPartial({case TLId => p(InnerTLId)}))) - val outer: ClientMetadata = Bundle(new ClientMetadata()(p.alterPartial({case TLId => p(OuterTLId)}))) + val inner: ManagerMetadata = new ManagerMetadata()(p.alterPartial({case TLId => p(InnerTLId)})) + val outer: ClientMetadata = new ClientMetadata()(p.alterPartial({case TLId => p(OuterTLId)})) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 928b3ee4..eee9aa6e 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import cde.{Parameters, Field} case object LNEndpoints extends Field[Int] case object LNHeaderBits extends Field[Int] diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index e292cb3d..93c8807c 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ import junctions._ +import cde.{Parameters, Field} case object RTCPeriod extends Field[Int] diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 365dedb5..5cf099a2 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ import junctions.{SMIIO, MMIOBase} +import cde.Parameters class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { val rdata = Vec(Bits(INPUT, scrDataBits), nSCR) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 0b9c296d..7b73511d 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -4,6 +4,7 @@ package uncore import Chisel._ import junctions._ import scala.math.max +import cde.{Parameters, Field} case object TLId extends Field[String] case class TLKey(id: String) extends Field[TileLinkParameters] @@ -659,7 +660,7 @@ class GrantMetadata(implicit p: Parameters) extends ManagerToClientChannel()(p) def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() def makeFinish(dummy: Int = 0): Finish = { - val f = Wire(Bundle(new Finish)) + val f = Wire(new Finish) f.manager_xact_id := this.manager_xact_id f } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d8f0683e..26bc3ff9 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ +import cde.{Parameters, Field} case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] @@ -60,7 +61,7 @@ trait HasCoherenceAgentWiringHelpers { } trait HasInnerTLIO extends HasCoherenceAgentParameters { - val inner = Bundle(new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)}))) + val inner = new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)})) val incoherent = Vec(Bool(), inner.tlNCachingClients).asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits @@ -70,13 +71,13 @@ trait HasInnerTLIO extends HasCoherenceAgentParameters { } trait HasUncachedOuterTLIO extends HasCoherenceAgentParameters { - val outer = Bundle(new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))) + val outer = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)})) def oacq(dummy: Int = 0) = outer.acquire.bits def ognt(dummy: Int = 0) = outer.grant.bits } trait HasCachedOuterTLIO extends HasCoherenceAgentParameters { - val outer = Bundle(new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)}))) + val outer = new ClientTileLinkIO()(p.alterPartial({case TLId => p(OuterTLId)})) def oacq(dummy: Int = 0) = outer.acquire.bits def oprb(dummy: Int = 0) = outer.probe.bits def orel(dummy: Int = 0) = outer.release.bits From 4c2b0a90326e15b1052232e95159f5d5a93a986b Mon Sep 17 00:00:00 2001 From: Jim Lawson Date: Thu, 22 Oct 2015 09:57:02 -0700 Subject: [PATCH 464/688] Add ability to generate libraryDependency on cde. --- uncore/build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/build.sbt b/uncore/build.sbt index a889e7a6..120670b5 100644 --- a/uncore/build.sbt +++ b/uncore/build.sbt @@ -7,7 +7,7 @@ name := "uncore" scalaVersion := "2.11.6" // Provide a managed dependency on X if -DXVersion="" is supplied on the command line. -libraryDependencies ++= (Seq("chisel","junctions").map { +libraryDependencies ++= (Seq("chisel","junctions","cde").map { dep: String => sys.props.get(dep + "Version") map { "edu.berkeley.cs" %% dep % _ }}).flatten site.settings From 6403f27fbe5112a39eb5559beee5199d78e3b104 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 22 Oct 2015 15:51:40 -0700 Subject: [PATCH 465/688] fix bug in ReorderQueue breaking TileLink Unwrapper --- uncore/src/main/scala/tilelink.scala | 5 ----- uncore/src/main/scala/util.scala | 3 ++- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7b73511d..445dea95 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1295,11 +1295,6 @@ trait HasDataBeatCounters { } } -class ClientTileLinkIOUnwrapperInfo extends Bundle { - val voluntary = Bool() - val builtin = Bool() -} - class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val io = new Bundle { val in = new ClientTileLinkIO().flip diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 80310966..a86a7525 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -131,7 +131,8 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val roq_free = Reg(init = Vec.fill(size)(Bool(true))) val roq_enq_addr = PriorityEncoder(roq_free) - val roq_matches = roq_tags.map(_ === io.deq.tag) + val roq_matches = roq_tags.zip(roq_free) + .map { case (tag, free) => tag === io.deq.tag && !free } val roq_deq_addr = PriorityEncoder(roq_matches) io.enq.ready := roq_free.reduce(_ || _) From 9fa4541916649effb7d74a7b1df8f52aafbfb29a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 26 Oct 2015 12:17:25 -0700 Subject: [PATCH 466/688] get rid of unused full signal in ReorderQueue --- uncore/src/main/scala/util.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index a86a7525..e5b14e7f 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -123,7 +123,6 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) val data = dType.cloneType.asOutput val matches = Bool(OUTPUT) } - val full = Bool(OUTPUT) } val roq_data = Reg(Vec(dType.cloneType, size)) From c10870a87ced5cbdd27eace24cd32d4e15751f75 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 27 Oct 2015 13:25:29 -0700 Subject: [PATCH 467/688] make sure ID width requirement in TL -> NASTI converter is correct --- uncore/src/main/scala/tilelink.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 445dea95..9c7bffb3 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1422,12 +1422,10 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) MT_Q -> UInt(log2Up(tlDataBytes)))) val dataBits = tlDataBits*tlDataBeats - val dstIdBits = p(LNHeaderBits) require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(dstIdBits + tlClientXactIdBits < nastiXIdBits, "NastiIO converter is going truncate tags: " + dstIdBits + " + " + tlClientXactIdBits + " >= " + nastiXIdBits) + require(tlClientXactIdBits <= nastiXIdBits, "NastiIO converter is going truncate tags: " + tlClientXactIdBits + " > " + nastiXIdBits) - val dst_off = dstIdBits + tlClientXactIdBits val has_data = io.tl.acquire.bits.hasData() val is_subblock = io.tl.acquire.bits.isSubBlockType() From d4b8653002876e6202a61bd5fc019b86b3f89cf2 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 31 Oct 2015 15:58:10 -0700 Subject: [PATCH 468/688] fix too strict assertion in broadcast hub --- uncore/src/main/scala/broadcast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 39aac120..cc908348 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -310,7 +310,7 @@ class BroadcastAcquireTracker(trackerId: Int) "AcquireTracker accepted data beat from different client transaction than initial request.") assert(!(state === s_idle && io.inner.acquire.fire() && - io.iacq().addr_beat != UInt(0)), + io.iacq().hasMultibeatData() && io.iacq().addr_beat != UInt(0)), "AcquireTracker initialized with a tail data beat.") when(collect_iacq_data) { From 812c5bcc559fafc06499aaf57c315d5c7877932b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 31 Oct 2015 15:58:36 -0700 Subject: [PATCH 469/688] make sure narrower can handle sub-block level requests correctly --- uncore/src/main/scala/tilelink.scala | 107 +++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 9c7bffb3..46469ebd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1535,15 +1535,20 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) data = Bits(0)) } -class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Parameters) extends Module { +class TileLinkIONarrower(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + val innerParams = p(TLKey(innerTLId)) val outerParams = p(TLKey(outerTLId)) val innerDataBeats = innerParams.dataBeats val innerDataBits = innerParams.dataBitsPerBeat val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) val outerDataBeats = outerParams.dataBeats val outerDataBits = outerParams.dataBitsPerBeat val outerWriteMaskBits = outerParams.writeMaskBits + val outerBlockOffset = log2Up(outerDataBits / 8) + val outerByteAddrBits = log2Up(outerWriteMaskBits) require(outerDataBeats >= innerDataBeats) require(outerDataBeats % innerDataBeats == 0) @@ -1563,6 +1568,8 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param val stretch = iacq.a_type === Acquire.putBlockType val shrink = iacq.a_type === Acquire.getBlockType + val smallput = iacq.a_type === Acquire.putType + val smallget = iacq.a_type === Acquire.getType val acq_data_buffer = Reg(UInt(width = innerDataBits)) val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) @@ -1571,6 +1578,44 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param val acq_addr_beat = Reg(iacq.addr_beat) val oacq_ctr = Counter(factor) + // this part of the address shifts from the inner byte address + // to the outer beat address + val readshift = iacq.full_addr()(innerByteAddrBits - 1, outerByteAddrBits) + val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) + val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) + + val mask_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerWriteMaskBits + val msb = (i + 1) * outerWriteMaskBits - 1 + iacq.wmask()(msb, lsb) + } + + val data_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerDataBits + val msb = (i + 1) * outerDataBits - 1 + iacq.data(msb, lsb) + } + + val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) + + val smallput_data = Mux1H(beat_sel, data_chunks) + val smallput_wmask = Mux1H(beat_sel, mask_chunks) + + assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), + "Can't perform Put wider than outer width") + + val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( + MT_B -> Bool(true), + MT_BU -> Bool(true), + MT_H -> Bool(outerDataBits >= 16), + MT_HU -> Bool(outerDataBits >= 16), + MT_W -> Bool(outerDataBits >= 32), + MT_D -> Bool(outerDataBits >= 64), + MT_Q -> Bool(false))) + + assert(!io.in.acquire.valid || !smallget || read_size_ok, + "Can't perform Get wider than outer width") + val outerConfig = p.alterPartial({ case TLId => outerTLId }) val innerConfig = p.alterPartial({ case TLId => innerTLId }) @@ -1588,13 +1633,48 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param data = acq_data_buffer(outerDataBits - 1, 0), wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + addr_byte = outer_byte_addr, + operand_size = iacq.op_size(), + alloc = iacq.allocate())(outerConfig) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + data = smallput_data, + wmask = Some(smallput_wmask))(outerConfig) + val sending_put = Reg(init = Bool(false)) + val pass_valid = io.in.acquire.valid && !stretch && !smallget + val smallget_valid = smallget && io.in.acquire.valid + + val smallget_roq = Module(new ReorderQueue( + readshift, tlClientXactIdBits, tlMaxClientsPerPort)) + + val smallget_helper = DecoupledHelper( + smallget_valid, + smallget_roq.io.enq.ready, + io.out.acquire.ready) + + smallget_roq.io.enq.valid := smallget_helper.fire(smallget_roq.io.enq.ready) + smallget_roq.io.enq.bits.data := readshift + smallget_roq.io.enq.bits.tag := iacq.client_xact_id + io.out.acquire.bits := MuxBundle(iacq, Seq( (sending_put, put_block_acquire), - (shrink, get_block_acquire))) - io.out.acquire.valid := sending_put || (io.in.acquire.valid && !stretch) - io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + (shrink, get_block_acquire), + (smallput, put_acquire), + (smallget, get_acquire))) + io.out.acquire.valid := sending_put || pass_valid || + smallget_helper.fire(io.out.acquire.ready) + io.in.acquire.ready := !sending_put && (stretch || + (!smallget && io.out.acquire.ready) || + smallget_helper.fire(smallget_valid)) when (io.in.acquire.fire() && stretch) { acq_data_buffer := iacq.data @@ -1628,9 +1708,26 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String)(implicit p: Param addr_beat = ignt_ctr.value, data = gnt_data_buffer.toBits)(innerConfig) + val smallget_grant = ognt.g_type === Grant.getDataBeatType + val get_grant_shift = Cat(smallget_roq.io.deq.data, + UInt(0, outerByteAddrBits + 3)) + + smallget_roq.io.deq.valid := io.out.grant.fire() && smallget_grant + smallget_roq.io.deq.tag := ognt.client_xact_id + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data << get_grant_shift)(innerConfig) + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - io.in.grant.bits := Mux(sending_get, get_block_grant, ognt) + io.in.grant.bits := MuxBundle(ognt, Seq( + sending_get -> get_block_grant, + smallget_grant -> get_grant)) when (io.out.grant.valid && ognt_block && !sending_get) { gnt_data_buffer(ognt_ctr.value) := ognt.data From baa2544651e117c35f6156338557eb0026a24681 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 31 Oct 2015 18:56:17 -0700 Subject: [PATCH 470/688] Fix some more issues with narrower --- uncore/src/main/scala/tilelink.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 46469ebd..b3a2464b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1547,8 +1547,11 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val outerDataBeats = outerParams.dataBeats val outerDataBits = outerParams.dataBitsPerBeat val outerWriteMaskBits = outerParams.writeMaskBits - val outerBlockOffset = log2Up(outerDataBits / 8) val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) require(outerDataBeats >= innerDataBeats) require(outerDataBeats % innerDataBeats == 0) @@ -1600,6 +1603,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val smallput_data = Mux1H(beat_sel, data_chunks) val smallput_wmask = Mux1H(beat_sel, mask_chunks) + val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), "Can't perform Put wider than outer width") @@ -1644,7 +1648,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val put_acquire = Put( client_xact_id = iacq.client_xact_id, addr_block = iacq.addr_block, - addr_beat = outer_beat_addr, + addr_beat = smallput_beat, data = smallput_data, wmask = Some(smallput_wmask))(outerConfig) @@ -1654,7 +1658,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val smallget_valid = smallget && io.in.acquire.valid val smallget_roq = Module(new ReorderQueue( - readshift, tlClientXactIdBits, tlMaxClientsPerPort)) + readshift, outerIdBits, outerMaxClients)) val smallget_helper = DecoupledHelper( smallget_valid, From 3698153535d5d278feda8d50ccc944c330182d4e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 3 Nov 2015 14:31:35 -0800 Subject: [PATCH 471/688] OHToUInt instead of PriorityEncoder on Acq/RelMatches signals in L2Bank --- uncore/src/main/scala/cache.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 20010406..a71568b4 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -316,7 +316,7 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits val acquire_idx = Mux(acquireMatches.orR, - PriorityEncoder(acquireMatches), + OHToUInt(acquireMatches), PriorityEncoder(acquireReadys)) val block_acquires = acquireConflicts.orR io.inner.acquire.ready := acquireReadys.orR && !block_acquires @@ -325,18 +325,22 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) tracker.bits := io.inner.acquire.bits tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) } + assert(PopCount(acquireMatches) <= UInt(1), + "At most a single tracker should match for any given Acquire") // Wire releases from clients val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits - val release_idx = PriorityEncoder(releaseMatches) + val release_idx = OHToUInt(releaseMatches) io.inner.release.ready := releaseReadys(release_idx) trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => tracker.bits := io.inner.release.bits tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) } + assert(PopCount(releaseMatches) <= UInt(nReleaseTransactors), + "At most a single tracker should match for any given Release") assert(!(io.inner.release.valid && !releaseMatches.orR), "Non-voluntary release should always have a Tracker waiting for it.") From e3efc09b5b8cba2f6b7f06dc199c27259ca0057a Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 5 Nov 2015 17:20:03 -0800 Subject: [PATCH 472/688] remove unnecessary UInt encode/decode on releaseMatches path --- uncore/src/main/scala/cache.scala | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index a71568b4..821495dc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -304,6 +304,7 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors)) + val trackerAndWbIOs = trackerList.map(_.io) :+ wb.io doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) @@ -329,15 +330,12 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) "At most a single tracker should match for any given Acquire") // Wire releases from clients - val trackerReleaseIOs = trackerList.map(_.io.inner.release) :+ wb.io.inner.release - val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits - val releaseMatches = Vec(trackerList.map(_.io.has_release_match) :+ wb.io.has_release_match).toBits - val release_idx = OHToUInt(releaseMatches) - io.inner.release.ready := releaseReadys(release_idx) - trackerReleaseIOs.zipWithIndex.foreach { - case(tracker, i) => - tracker.bits := io.inner.release.bits - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) + val releaseReadys = Vec(trackerAndWbIOs.map(_.inner.release.ready)).toBits + val releaseMatches = Vec(trackerAndWbIOs.map(_.has_release_match)).toBits + io.inner.release.ready := (releaseMatches & releaseReadys).orR + trackerAndWbIOs foreach { tracker => + tracker.inner.release.bits := io.inner.release.bits + tracker.inner.release.valid := io.inner.release.valid && tracker.has_release_match } assert(PopCount(releaseMatches) <= UInt(nReleaseTransactors), "At most a single tracker should match for any given Release") From 7942be4e018d3315897e9e84f13b20485e7a1383 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 9 Nov 2015 11:10:02 -0800 Subject: [PATCH 473/688] make sure outerTL method is idempotent --- uncore/src/main/scala/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 26bc3ff9..d4822a49 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -98,7 +98,7 @@ abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAg with HasCoherenceAgentWiringHelpers { val io = new ManagerTLIO def innerTL = io.inner - def outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) + lazy val outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) def incoherent = io.incoherent } From 42d3d09d7a4adda467efeb06afc15f18379c2963 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 9 Nov 2015 11:49:19 -0800 Subject: [PATCH 474/688] add a ClientTileLinkEnqueuer to complement the TileLinkEnqueuer --- uncore/src/main/scala/tilelink.scala | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b3a2464b..627c4e0a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1035,6 +1035,29 @@ object TileLinkEnqueuer { } } +class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { + val io = new Bundle { + val inner = new ClientTileLinkIO().flip + val outer = new ClientTileLinkIO + } + + io.outer.acquire <> (if(depths.acq > 0) Queue(io.inner.acquire, depths.acq) else io.inner.acquire) + io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) + io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) + io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) +} + +object ClientTileLinkEnqueuer { + def apply(in: ClientTileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): ClientTileLinkIO = { + val t = Module(new ClientTileLinkEnqueuer(depths)) + t.io.inner <> in + t.io.outer + } + def apply(in: ClientTileLinkIO, depth: Int)(implicit p: Parameters): ClientTileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) + } +} + /** Utility functions for constructing TileLinkIO arbiters */ trait TileLinkArbiterLike extends HasTileLinkParameters { // Some shorthand type variables From b59ce5fed4f9470353b72e43278f883243bb71a3 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 10 Nov 2015 16:06:14 -0800 Subject: [PATCH 475/688] make sure L2 waits for outer grant before sending grant for write request --- uncore/src/main/scala/cache.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 821495dc..388d16bf 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -568,6 +568,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_ignt_ack = Reg(init = Bool(false)) val pending_meta_write = Reg{ Bool() } val all_pending_done = @@ -731,10 +732,15 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) + pending_ignt_ack := pending_ignt_ack | + io.data.write.fire() | + io.outer.grant.fire() && io.outer.grant.bits.hasData() ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && - (!io.ignt().hasData() || pending_ignt_data(ignt_data_idx)) + Mux(io.ignt().hasData(), + pending_ignt_data(ignt_data_idx), + pending_ignt_ack) // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( pri = xact, @@ -833,6 +839,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) pending_resps := UInt(0) pending_ignt_data := UInt(0) + pending_ignt_ack := Bool(false) pending_meta_write := UInt(0) state := s_meta_read } From 7733fbe6a31a6cf3076f3fe2afb2882559d5fdee Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 12 Nov 2015 11:39:36 -0800 Subject: [PATCH 476/688] make sure no-alloc write still updates data array if there is a cache hit --- uncore/src/main/scala/cache.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 388d16bf..c2535d9e 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -628,6 +628,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } } + def addPendingBitFromBufferedAcquire(xact: AcquireMetadata): UInt = + Mux(xact.hasMultibeatData(), + Fill(innerDataBeats, UInt(1, 1)), + UIntToOH(xact.addr_beat)) + // Actual transaction processing logic begins here: // // First, take care of accpeting new requires or secondary misses @@ -732,9 +737,10 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) + // We can issue a grant for a pending write once the write is committed pending_ignt_ack := pending_ignt_ack | io.data.write.fire() | - io.outer.grant.fire() && io.outer.grant.bits.hasData() + io.outer.grant.fire() && !io.outer.grant.bits.hasData() ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && @@ -869,6 +875,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } + // If the write is marked no-allocate but is already in the cache, + // we do, in fact, need to write the data to the cache + when (is_hit && !xact.allocate() && xact.hasData()) { + pending_writes := addPendingBitFromBufferedAcquire(xact) + } state := Mux(needs_writeback, s_wb_req, Mux(needs_inner_probes, s_inner_probe, Mux(!is_hit, s_outer_acquire, s_busy))) From f397d610338999de3c33df4a6729a94f3ef3c74b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 12 Nov 2015 11:39:59 -0800 Subject: [PATCH 477/688] add alloc option to Put constructor --- uncore/src/main/scala/tilelink.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 627c4e0a..d33a5cf0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -402,6 +402,7 @@ object GetPrefetch { * @param addr_beat sub-block address (which beat) * @param data data being refilled to the original requestor * @param wmask per-byte write mask for this beat + * @param alloc hint whether the block should be allocated in intervening caches */ object Put { def apply( @@ -409,7 +410,8 @@ object Put { addr_block: UInt, addr_beat: UInt, data: UInt, - wmask: Option[UInt]= None) + wmask: Option[UInt]= None, + alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { Acquire( is_builtin_type = Bool(true), @@ -418,7 +420,7 @@ object Put { addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(wmask.getOrElse(Acquire.fullWriteMask), Bool(true))) + union = Cat(wmask.getOrElse(Acquire.fullWriteMask), alloc)) } } From b3865c370a1025abab743aabc563049ccfcc3d5b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 12 Nov 2015 15:40:38 -0800 Subject: [PATCH 478/688] make sure correct addr_beat is sent for Get response by narrower/converter --- uncore/src/main/scala/tilelink.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d33a5cf0..dea63245 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1426,6 +1426,7 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { } class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { + val addr_beat = UInt(width = tlBeatAddrBits) val byteOff = UInt(width = tlByteAddrBits) val subblock = Bool() } @@ -1488,6 +1489,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() roq.io.enq.bits.data.subblock := is_subblock roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) @@ -1546,7 +1548,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) Grant.getDataBeatType, Grant.getDataBlockType), client_xact_id = io.nasti.r.bits.id, manager_xact_id = UInt(0), - addr_beat = tl_cnt_in, + addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), data = r_aligned_data) gnt_arb.io.in(1).valid := io.nasti.b.valid @@ -1749,7 +1751,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) g_type = Grant.getDataBeatType, client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, + addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), data = ognt.data << get_grant_shift)(innerConfig) io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) From 7e7d688a016bfa99b66f5a29aec77da57dab7692 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 12 Nov 2015 15:40:58 -0800 Subject: [PATCH 479/688] make sure L2 passes no-alloc acquires through to outer memory --- uncore/src/main/scala/cache.scala | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c2535d9e..dd1f731b 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -709,6 +709,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) + val outerParams = p.alterPartial({ case TLId => p(OuterTLId) }) // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // // If we're allocating in this cache, we can use the current metadata @@ -716,11 +717,19 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // built-in Acquire from the inner TL to the outer TL io.outer.acquire.valid := state === s_outer_acquire && (xact.allocate() || !pending_puts(oacq_data_idx)) - io.outer.acquire.bits := Mux(xact.allocate(), xact_old_meta.coh.outer, ClientMetadata.onReset) - .makeAcquire( + io.outer.acquire.bits := Mux(xact.allocate(), + xact_old_meta.coh.outer.makeAcquire( client_xact_id = UInt(0), addr_block = xact.addr_block, - op_code = xact.op_code()) + op_code = xact.op_code()), + Acquire( + is_builtin_type = Bool(true), + a_type = xact.a_type, + client_xact_id = UInt(0), + addr_block = xact.addr_block, + addr_beat = xact.addr_beat, + union = xact.union)(outerParams)) + io.oacq().data := xact.data_buffer(oacq_data_idx) // Handle the response from outer memory @@ -792,7 +801,8 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_writes := (pending_writes & dropPendingBit(io.data.write)) | addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | - addPendingBitWhenBeatHasData(io.outer.grant) + (Fill(refillCycles, xact.allocate()) & + addPendingBitWhenBeatHasData(io.outer.grant)) val curr_write_beat = PriorityEncoder(pending_writes) io.data.write.valid := state === s_busy && pending_writes.orR && @@ -807,7 +817,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.data.write.bits.data := xact.data_buffer(curr_write_beat) // End a transaction by updating the block metadata - io.meta.write.valid := state === s_meta_write + io.meta.write.valid := (state === s_meta_write) && pending_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en @@ -864,9 +874,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra (coh.outer.requiresVoluntaryWriteback() || coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) - when(!tag_match || is_hit && pending_coh_on_hit != coh) { pending_meta_write := Bool(true) } + val should_update_meta = !tag_match && xact.allocate() || + is_hit && pending_coh_on_hit != coh + when (should_update_meta) { pending_meta_write := Bool(true) } pending_coh := Mux(is_hit, pending_coh_on_hit, Mux(tag_match, coh, pending_coh_on_miss)) - when(needs_inner_probes) { + when (needs_inner_probes) { val full_sharers = coh.inner.full() val mask_self = Mux( xact.requiresSelfProbe(), From e12efab423777365953e77f6ee3d18b8885d9f03 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 13 Nov 2015 13:50:35 -0800 Subject: [PATCH 480/688] skip meta_write state if no meta write pending --- uncore/src/main/scala/cache.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dd1f731b..df2ce03a 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -569,7 +569,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_ignt_ack = Reg(init = Bool(false)) - val pending_meta_write = Reg{ Bool() } + val pending_meta_write = Reg(Bool()) val all_pending_done = !(pending_reads.orR || @@ -817,7 +817,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.data.write.bits.data := xact.data_buffer(curr_write_beat) // End a transaction by updating the block metadata - io.meta.write.valid := (state === s_meta_write) && pending_meta_write + io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en @@ -856,7 +856,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_resps := UInt(0) pending_ignt_data := UInt(0) pending_ignt_ack := Bool(false) - pending_meta_write := UInt(0) + pending_meta_write := Bool(false) state := s_meta_read } when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } @@ -911,11 +911,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_outer_acquire && oacq_data_done) { state := s_busy } - when(state === s_busy && all_pending_done) { state := s_meta_write } - when(state === s_meta_write && (io.meta.write.ready || !pending_meta_write)) { + when(state === s_busy && all_pending_done) { wmask_buffer.foreach { w => w := UInt(0) } - state := s_idle + state := Mux(pending_meta_write, s_meta_write, s_idle) } + when(state === s_meta_write && io.meta.write.ready) { state := s_idle } // These IOs are used for routing in the parent val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) From 03fa06e6e7c409ff760ef7af2b9e9917f8d5986f Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 15 Nov 2015 12:51:34 -0800 Subject: [PATCH 481/688] fix prefetch lockup on L2 hit --- uncore/src/main/scala/cache.scala | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index df2ce03a..876e2041 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -876,8 +876,10 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) val should_update_meta = !tag_match && xact.allocate() || is_hit && pending_coh_on_hit != coh + // Determine any changes to the coherence metadata when (should_update_meta) { pending_meta_write := Bool(true) } pending_coh := Mux(is_hit, pending_coh_on_hit, Mux(tag_match, coh, pending_coh_on_miss)) + // If we need to probe some clients, make a bitmask identifying them when (needs_inner_probes) { val full_sharers = coh.inner.full() val mask_self = Mux( @@ -886,12 +888,17 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra coh.inner.full() & ~UIntToOH(xact.client_id)) val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent - } + } + // If a prefetch is a hit, note that we have to ack it + when (is_hit && xact.isPrefetch()) { + pending_ignt_ack := Bool(true) + } // If the write is marked no-allocate but is already in the cache, // we do, in fact, need to write the data to the cache when (is_hit && !xact.allocate() && xact.hasData()) { pending_writes := addPendingBitFromBufferedAcquire(xact) } + // Next: request writeback, issue probes, query outer memory, or respond state := Mux(needs_writeback, s_wb_req, Mux(needs_inner_probes, s_inner_probe, Mux(!is_hit, s_outer_acquire, s_busy))) From 64aaf71b06adc61e010f071e2df4398e24a0cce9 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Nov 2015 13:23:17 -0800 Subject: [PATCH 482/688] L2AcquireTracker refactor to support merging Gets and Puts into Prefetches of the correct type. Transaction metadata for primary and seconday misses now stored in the secondary miss queue. Added BuiltInAcquireBuilder factory. --- uncore/src/main/scala/cache.scala | 346 +++++++++++++------------- uncore/src/main/scala/coherence.scala | 107 ++++---- uncore/src/main/scala/metadata.scala | 34 ++- uncore/src/main/scala/tilelink.scala | 320 +++++++++++++++--------- uncore/src/main/scala/uncore.scala | 9 +- 5 files changed, 460 insertions(+), 356 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 876e2041..baa7fed8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -414,7 +414,7 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { val a = in.bits - val isPartial = a.wmask() != Acquire.fullWriteMask + val isPartial = a.wmask() =/= Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } @@ -533,44 +533,62 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val state = Reg(init=s_idle) // State holding transaction metadata - val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => p(InnerTLId) }))) + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) + val xact_addr_block = Reg{ io.inner.acquire.bits.addr_block } val xact_tag_match = Reg{ Bool() } val xact_way_en = Reg{ Bits(width = nWays) } val xact_old_meta = Reg{ new L2Metadata } val pending_coh = Reg{ xact_old_meta.coh } + val xact_allocate = Reg{ Bool() } + val xact_amo_shift_bits = Reg{ UInt() } + val xact_op_code = Reg{ UInt() } + val xact_addr_byte = Reg{ UInt() } + val xact_op_size = Reg{ UInt() } - // Secondary miss queue - val ignt_q = Module(new Queue(new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })), - nSecondaryMisses)) + // Miss queue holds transaction metadata used to make grants + val ignt_q = Module(new Queue( + new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })), + 1 + nSecondaryMisses)) - // State holding progress made on processing this transaction - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) + // Some accessor wires derived from the the above state + val xact = ignt_q.io.deq.bits + val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) + val xact_addr_tag = xact_addr_block >> UInt(idxBits) + + // Counters and scoreboard tracking progress made on processing this transaction val pending_irels = connectTwoWayBeatCounter( max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 + val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectTwoWayBeatCounter( max = 1, up = io.outer.acquire, down = io.outer.grant, beat = xact.addr_beat) - val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + + val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter( + out = io.inner.grant, + beat = ignt_q.io.deq.bits.addr_beat) + val pending_ifins = connectTwoWayBeatCounter( max = nSecondaryMisses, up = io.inner.grant, down = io.inner.finish, track = (g: Grant) => g.requiresAck())._1 + val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_ignt_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_ignt_ack = Reg(init = Bool(false)) - val pending_meta_write = Reg(Bool()) + val ignt_data_ready = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val ignt_ack_ready = Reg(init = Bool(false)) + val pending_meta_write = Reg(init = Bool(false)) + // Used to decide when to escape from s_busy val all_pending_done = !(pending_reads.orR || pending_writes.orR || @@ -584,16 +602,14 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Provide a single ALU per tracker to merge Puts and AMOs with data being // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU(rhsIsAligned = true)) - amoalu.io.addr := xact.full_addr() - amoalu.io.cmd := xact.op_code() - amoalu.io.typ := xact.op_size() + amoalu.io.addr := Cat(xact_addr_block, xact.addr_beat, xact_addr_byte) + amoalu.io.cmd := xact_op_code + amoalu.io.typ := xact_op_size amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData - amoalu.io.rhs := xact.data_buffer.head // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData val amo_result = Reg(init = UInt(0, xact.tlDataBits)) - // Utility functions for updating the data and metadata that will be kept in - // the cache or granted to the original requestor after this transaction: - + // Utility function for updating the metadata that will be kept in this cache def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { when(flag && pending_coh != next) { pending_meta_write := Bool(true) @@ -601,79 +617,88 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } } + // Defined here because of Chisel default wire demands, used in s_meta_resp + val pending_coh_on_hit = HierarchicalMetadata( + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact_op_code)) + + val pending_coh_on_miss = HierarchicalMetadata.onReset + + // Utility function for applying any buffered stored data to the cache line + // before storing it back into the data array def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data - val new_data = xact.data_buffer(beat) // Newly Put data is already in the buffer - amoalu.io.lhs := old_data >> xact.amo_shift_bits() - amoalu.io.rhs := new_data >> xact.amo_shift_bits() + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + amoalu.io.lhs := old_data >> xact_amo_shift_bits + amoalu.io.rhs := new_data >> xact_amo_shift_bits val wmask = FillInterleaved(8, wmask_buffer(beat)) - xact.data_buffer(beat) := ~wmask & old_data | + data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), - amoalu.io.out << xact.amo_shift_bits(), + amoalu.io.out << xact_amo_shift_bits, new_data) wmask_buffer(beat) := ~UInt(0, wmask_buffer.head.getWidth) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } + + // TODO: Deal with the possibility that rowBits != tlDataBits def mergeDataInternal[T <: L2HellaCacheBundle with HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } + def mergeDataInner[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { when(in.fire() && in.bits.hasData()) { mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) } } + def mergeDataOuter[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { when(in.fire() && in.bits.hasData()) { mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) } } - def addPendingBitFromBufferedAcquire(xact: AcquireMetadata): UInt = - Mux(xact.hasMultibeatData(), - Fill(innerDataBeats, UInt(1, 1)), - UIntToOH(xact.addr_beat)) + // and Puts-under-Put, and either may also merge witha preceding prefetch + // that requested the correct permissions (via op_code) + def acquiresAreMergeable(sec: AcquireMetadata): Bool = { + val allowedTypes = List((Acquire.getType, Acquire.getType), + (Acquire.putType, Acquire.putType), + (Acquire.putBlockType, Acquire.putBlockType), + (Acquire.prefetchType, Acquire.prefetchType), + (Acquire.prefetchType, Acquire.getType), + (Acquire.prefetchType, Acquire.putType), + (Acquire.prefetchType, Acquire.putBlockType)) + allowedTypes.map { case(a, b) => xact.isBuiltInType(a) && sec.isBuiltInType(b) }.reduce(_||_) && + xact_op_code === sec.op_code() && + sec.conflicts(xact_addr_block) && + (xact_allocate || xact.isBuiltInType(Acquire.putBlockType)) + } // Actual transaction processing logic begins here: // - // First, take care of accpeting new requires or secondary misses - // For now, the only allowed secondary miss types are Gets-under-Get - // and Puts-under-Put from the same client - val can_merge_iacq_get = (xact.isBuiltInType(Acquire.getType) && - io.iacq().isBuiltInType(Acquire.getType)) && - xact.client_id === io.iacq().client_id && //TODO remove - xact.conflicts(io.iacq()) && - state != s_idle && state != s_meta_write && - !all_pending_done && - xact.allocate() && - !io.inner.release.fire() && - !io.outer.grant.fire() && - !io.data.resp.valid && - ignt_q.io.enq.ready + // First, take care of accpeting new acquires or secondary misses + val iacq_can_merge = acquiresAreMergeable(io.iacq()) && + state =/= s_idle && state =/= s_meta_write && + !all_pending_done && + !io.inner.release.fire() && + !io.outer.grant.fire() && + !io.data.resp.valid && + ignt_q.io.enq.ready && ignt_q.io.deq.valid - // This logic also allows the tail beats of a PutBlock to be merged in - val can_merge_iacq_put = ((xact.isBuiltInType(Acquire.putType) && - io.iacq().isBuiltInType(Acquire.putType)) || - (xact.isBuiltInType(Acquire.putBlockType) && - io.iacq().isBuiltInType(Acquire.putBlockType))) && - xact.client_id === io.iacq().client_id && //TODO remove - xact.conflicts(io.iacq()) && - state != s_idle && state != s_meta_write && - !all_pending_done && - (xact.allocate() || xact.isBuiltInType(Acquire.putBlockType)) && - !io.inner.release.fire() && - !io.outer.grant.fire() && - !io.data.resp.valid && - ignt_q.io.enq.ready + io.inner.acquire.ready := state === s_idle || iacq_can_merge - io.inner.acquire.ready := state === s_idle || - can_merge_iacq_put || - can_merge_iacq_get + // Handling of primary and secondary misses' data and write mask merging + when(io.inner.acquire.fire() && io.iacq().hasData()) { + val beat = io.iacq().addr_beat + val full = FillInterleaved(8, io.iacq().wmask()) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) + wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed + } - // Enqueue secondary miss information - ignt_q.io.enq.valid := iacq_data_done - ignt_q.io.enq.bits.client_xact_id := io.iacq().client_xact_id - ignt_q.io.enq.bits.addr_beat := io.iacq().addr_beat - // TODO add ignt.dst <- iacq.src + // Enqueue some metadata information that we'll use to make coherence updates with later + ignt_q.io.enq.valid := Mux(io.iacq().hasMultibeatData(), + io.inner.acquire.fire() && state === s_idle, + io.inner.acquire.fire()) + ignt_q.io.enq.bits := io.iacq() // Track whether any beats are missing from a PutBlock pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) @@ -681,13 +706,13 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Begin a transaction by getting the current block metadata io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.read.bits.idx := xact_addr_idx + io.meta.read.bits.tag := xact_addr_tag // Issue a request to the writeback unit io.wb.req.valid := state === s_wb_req io.wb.req.bits.id := UInt(trackerId) - io.wb.req.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.wb.req.bits.idx := xact_addr_idx io.wb.req.bits.tag := xact_old_meta.tag io.wb.req.bits.coh := xact_old_meta.coh io.wb.req.bits.way_en := xact_way_en @@ -696,7 +721,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) val curr_probe_dst = PriorityEncoder(pending_iprbs) io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact) + io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact, xact_addr_block) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data @@ -709,61 +734,101 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) - val outerParams = p.alterPartial({ case TLId => p(OuterTLId) }) // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // // If we're allocating in this cache, we can use the current metadata // to make an appropriate custom Acquire, otherwise we copy over the // built-in Acquire from the inner TL to the outer TL io.outer.acquire.valid := state === s_outer_acquire && - (xact.allocate() || !pending_puts(oacq_data_idx)) - io.outer.acquire.bits := Mux(xact.allocate(), + (xact_allocate || !pending_puts(oacq_data_idx)) + io.outer.acquire.bits := Mux(xact_allocate, xact_old_meta.coh.outer.makeAcquire( + op_code = xact_op_code, client_xact_id = UInt(0), - addr_block = xact.addr_block, - op_code = xact.op_code()), - Acquire( - is_builtin_type = Bool(true), + addr_block = xact_addr_block), + BuiltInAcquireBuilder( a_type = xact.a_type, client_xact_id = UInt(0), - addr_block = xact.addr_block, - addr_beat = xact.addr_beat, - union = xact.union)(outerParams)) - - io.oacq().data := xact.data_buffer(oacq_data_idx) + addr_block = xact_addr_block, + addr_beat = oacq_data_idx, + data = data_buffer(oacq_data_idx), + addr_byte = xact_addr_byte, + operand_size = xact_op_size, + opcode = xact_op_code, + wmask = wmask_buffer(oacq_data_idx), + alloc = Bool(false)) + (p.alterPartial({ case TLId => p(OuterTLId)}))) // Handle the response from outer memory io.outer.grant.ready := state === s_busy val pending_coh_on_ognt = HierarchicalMetadata( ManagerMetadata.onReset, - pending_coh.outer.onGrant(io.outer.grant.bits, xact.op_code())) + pending_coh.outer.onGrant(io.outer.grant.bits, xact_op_code)) updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) mergeDataOuter(io.outer.grant) - // Going back to the original inner transaction, we can issue a Grant as + // Going back to the original inner transaction: + // We read from the the cache at this level if data wasn't written back or refilled. + // We may still merge further Gets, requiring further beats to be read. + // If ECC requires a full writemask, we'll read out data on partial writes as well. + pending_reads := (pending_reads & + dropPendingBit(io.data.read) & + dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)) | + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) + val curr_read_beat = PriorityEncoder(pending_reads) + io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt + io.data.read.bits.id := UInt(trackerId) + io.data.read.bits.way_en := xact_way_en + io.data.read.bits.addr_idx := xact_addr_idx + io.data.read.bits.addr_beat := curr_read_beat + + pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | + addPendingBitInternal(io.data.read) + mergeDataInternal(io.data.resp) + + // We write data to the cache at this level if it was Put here with allocate flag, + // written back dirty, or refilled from outer memory. + pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant, xact_allocate) + val curr_write_beat = PriorityEncoder(pending_writes) + io.data.write.valid := state === s_busy && + pending_writes.orR && + !pending_ognt && + !pending_reads(curr_write_beat) && + !pending_resps(curr_write_beat) + io.data.write.bits.id := UInt(trackerId) + io.data.write.bits.way_en := xact_way_en + io.data.write.bits.addr_idx := xact_addr_idx + io.data.write.bits.addr_beat := curr_write_beat + io.data.write.bits.wmask := wmask_buffer(curr_write_beat) + io.data.write.bits.data := data_buffer(curr_write_beat) + // soon as the data is released, granted, put, or read from the cache - pending_ignt_data := pending_ignt_data | + ignt_data_ready := ignt_data_ready | addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) // We can issue a grant for a pending write once the write is committed - pending_ignt_ack := pending_ignt_ack | + ignt_ack_ready := ignt_ack_ready | io.data.write.fire() | io.outer.grant.fire() && !io.outer.grant.bits.hasData() ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && Mux(io.ignt().hasData(), - pending_ignt_data(ignt_data_idx), - pending_ignt_ack) + ignt_data_ready(ignt_data_idx), + ignt_ack_ready) // Make the Grant message using the data stored in the secondary miss queue io.inner.grant.bits := pending_coh.inner.makeGrant( - pri = xact, sec = ignt_q.io.deq.bits, manager_xact_id = UInt(trackerId), data = Mux(xact.is(Acquire.putAtomicType), amo_result, - xact.data_buffer(ignt_data_idx))) + data_buffer(ignt_data_idx))) io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( @@ -776,72 +841,22 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // We must wait for as many Finishes as we sent Grants io.inner.finish.ready := state === s_busy - // We read from the the cache at this level if data wasn't written back or refilled. - // We may merge Gets, requiring further beats to be read. - // If ECC requires a full writemask, we'll read out data on partial writes as well. - pending_reads := (pending_reads & - dropPendingBit(io.data.read) & - dropPendingBitWhenBeatHasData(io.inner.release) & - dropPendingBitWhenBeatHasData(io.outer.grant)) | - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | - addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) - val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.read.bits.addr_beat := curr_read_beat - - pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | - addPendingBitInternal(io.data.read) - mergeDataInternal(io.data.resp) - - // We write data to the cache at this level if it was Put here with allocate flag, - // written back dirty, or refilled from outer memory. - pending_writes := (pending_writes & dropPendingBit(io.data.write)) | - addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | - addPendingBitWhenBeatHasData(io.inner.release) | - (Fill(refillCycles, xact.allocate()) & - addPendingBitWhenBeatHasData(io.outer.grant)) - val curr_write_beat = PriorityEncoder(pending_writes) - io.data.write.valid := state === s_busy && - pending_writes.orR && - !pending_ognt && - !pending_reads(curr_write_beat) && - !pending_resps(curr_write_beat) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := wmask_buffer(curr_write_beat) - io.data.write.bits.data := xact.data_buffer(curr_write_beat) - // End a transaction by updating the block metadata io.meta.write.valid := state === s_meta_write io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) + io.meta.write.bits.idx := xact_addr_idx io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.tag := xact_addr_tag io.meta.write.bits.data.coh := pending_coh - // Handling of secondary misses (Gets and Puts only for now) - when(io.inner.acquire.fire() && io.iacq().hasData()) { // state <= s_meta_wrtie - val beat = io.iacq().addr_beat - val wmask = io.iacq().wmask() - val full = FillInterleaved(8, wmask) - xact.data_buffer(beat) := (~full & xact.data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := wmask | Mux(state === s_idle, Bits(0), wmask_buffer(beat)) - } - - // Defined here because of Chisel default wire demands, used in s_meta_resp - val pending_coh_on_hit = HierarchicalMetadata( - io.meta.resp.bits.meta.coh.inner, - io.meta.resp.bits.meta.coh.outer.onHit(xact.op_code())) - val pending_coh_on_miss = HierarchicalMetadata.onReset - // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.acquire.valid) { - xact := io.iacq() + xact_addr_block := io.iacq().addr_block + xact_allocate := io.iacq().allocate() + xact_amo_shift_bits := io.iacq().amo_shift_bits() + xact_op_code := io.iacq().op_code() + xact_addr_byte := io.iacq().addr_byte() + xact_op_size := io.iacq().op_size() amo_result := UInt(0) pending_puts := Mux( // Make sure to collect all data from a PutBlock io.iacq().isBuiltInType(Acquire.putBlockType), @@ -854,8 +869,8 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toSInt).toUInt pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) pending_resps := UInt(0) - pending_ignt_data := UInt(0) - pending_ignt_ack := Bool(false) + ignt_data_ready := UInt(0) + ignt_ack_ready := Bool(false) pending_meta_write := Bool(false) state := s_meta_read } @@ -866,15 +881,15 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra xact_way_en := io.meta.resp.bits.way_en val coh = io.meta.resp.bits.meta.coh val tag_match = io.meta.resp.bits.tag_match - val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact.op_code()) + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact_op_code) else xact.isBuiltInType(Acquire.putBlockType) || tag_match && coh.outer.isValid()) val needs_writeback = !tag_match && - xact.allocate() && + xact_allocate && (coh.outer.requiresVoluntaryWriteback() || coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) - val should_update_meta = !tag_match && xact.allocate() || + val should_update_meta = !tag_match && xact_allocate || is_hit && pending_coh_on_hit != coh // Determine any changes to the coherence metadata when (should_update_meta) { pending_meta_write := Bool(true) } @@ -889,14 +904,14 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } - // If a prefetch is a hit, note that we have to ack it + // If a prefetch is a hit, note that we should ack it ASAP when (is_hit && xact.isPrefetch()) { - pending_ignt_ack := Bool(true) + ignt_ack_ready := Bool(true) } - // If the write is marked no-allocate but is already in the cache, - // we do, in fact, need to write the data to the cache - when (is_hit && !xact.allocate() && xact.hasData()) { - pending_writes := addPendingBitFromBufferedAcquire(xact) + // If some kind of Put is marked no-allocate but is already in the cache, + // we need to write its data to the data array + when (is_hit && !xact_allocate && xact.hasData()) { + pending_writes := addPendingBitsFromAcquire(xact) } // Next: request writeback, issue probes, query outer memory, or respond state := Mux(needs_writeback, s_wb_req, @@ -913,28 +928,25 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { // Tag matches, so if this is the last level cache we can use the data without upgrading permissions val skip_outer_acquire = - (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact.op_code()) + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact_op_code) else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_outer_acquire && oacq_data_done) { state := s_busy } when(state === s_busy && all_pending_done) { - wmask_buffer.foreach { w => w := UInt(0) } + wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle state := Mux(pending_meta_write, s_meta_write, s_idle) } when(state === s_meta_write && io.meta.write.ready) { state := s_idle } // These IOs are used for routing in the parent - val in_same_set = xact.addr_block(idxMSB,idxLSB) === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := xact.conflicts(io.irel()) && !io.irel().isVoluntary() && io.inner.release.ready - io.has_acquire_match := can_merge_iacq_put || can_merge_iacq_get - io.has_acquire_conflict := in_same_set && (state != s_idle) && !io.has_acquire_match + val in_same_set = xact_addr_idx === io.iacq().addr_block(idxMSB,idxLSB) + io.has_release_match := io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() && + io.inner.release.ready + io.has_acquire_match := iacq_can_merge + io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? - - // Checks for illegal behavior - assert(!(state != s_idle && io.inner.acquire.fire() && - io.inner.acquire.bits.client_id != xact.client_id), - "AcquireTracker accepted data beat from different network source than initial request.") } class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index fb6ec049..4f4e40a8 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -70,13 +70,13 @@ trait HasClientSideCoherencePolicy { // Determine which custom message type to use def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt - def getReleaseType(p: Probe, meta: ClientMetadata): UInt + def getReleaseType(p: HasProbeType, meta: ClientMetadata): UInt // Mutate ClientMetadata based on messages or cmds def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata): ClientMetadata def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata): ClientMetadata - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata): ClientMetadata + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata): ClientMetadata + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata): ClientMetadata } /** This API contains all functions required for manager coherence agents. @@ -90,18 +90,18 @@ trait HasManagerSideCoherencePolicy extends HasDirectoryRepresentation { def masterStateWidth = log2Ceil(nManagerStates) // Transaction probing logic - def requiresProbes(acq: AcquireMetadata, meta: ManagerMetadata): Bool + def requiresProbes(acq: HasAcquireType, meta: ManagerMetadata): Bool def requiresProbes(cmd: UInt, meta: ManagerMetadata): Bool // Determine which custom message type to use in response def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt - def getProbeType(acq: AcquireMetadata, meta: ManagerMetadata): UInt - def getGrantType(acq: AcquireMetadata, meta: ManagerMetadata): UInt + def getProbeType(acq: HasAcquireType, meta: ManagerMetadata): UInt + def getGrantType(acq: HasAcquireType, meta: ManagerMetadata): UInt def getExclusiveGrantType(): UInt // Mutate ManagerMetadata based on messages or cmds - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata): ManagerMetadata - def managerMetadataOnGrant(outgoing: GrantMetadata, dst: UInt, meta: ManagerMetadata) = + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata): ManagerMetadata + def managerMetadataOnGrant(outgoing: HasGrantType, dst: UInt, meta: ManagerMetadata) = ManagerMetadata(sharers=Mux(outgoing.isBuiltInType(), // Assumes all built-ins are uncached meta.sharers, dir.push(meta.sharers, dst)))(meta.p) @@ -151,7 +151,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeCopy -> getReleaseType(M_CLEAN, meta))) @@ -161,17 +161,17 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientMetadataOnCacheControl(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(cmd === M_FLUSH, clientInvalid, meta.state))(meta.p) - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(incoming.isBuiltInType(), clientInvalid, clientValid))(meta.p) - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = ClientMetadata(Mux(incoming.p_type === probeInvalidate, clientInvalid, meta.state))(meta.p) // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol - def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = !dir.none(meta.sharers) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -179,7 +179,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(cmd, probeCopy, Array( M_FLUSH -> probeInvalidate)) - def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -189,10 +189,11 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) - def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = grantExclusive + def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -240,7 +241,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeDowngrade -> getReleaseType(M_PRODUCE, meta), @@ -255,12 +256,12 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> clientInvalid, M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state))))(meta.p) - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(incoming.isBuiltInType(), clientInvalid, Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean)))(meta.p) - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, @@ -270,7 +271,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol - def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = !dir.none(meta.sharers) + def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = !dir.none(meta.sharers) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) def getProbeType(cmd: UInt, meta: ManagerMetadata): UInt = @@ -278,7 +279,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -288,10 +289,11 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) - def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = grantExclusive + def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), grantExclusive) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -339,7 +341,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeDowngrade -> getReleaseType(M_PRODUCE, meta), @@ -355,7 +357,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), clientShared, meta.state))))(meta.p) - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( @@ -363,7 +365,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { grantExclusive -> clientExclusiveDirty, grantExclusiveAck -> clientExclusiveDirty))))(meta.p) - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, @@ -376,7 +378,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // notification msg to track clean drops) // Also could avoid probes on outer WBs. - def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = + def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -388,7 +390,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -400,13 +402,14 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { acquireShared -> probeDowngrade, acquireExclusive -> probeInvalidate))) - def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive) + def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -441,7 +444,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) def clientStatesWithDirtyData = Vec(clientExclusiveDirty) - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) @@ -454,7 +457,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = + def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeDowngrade -> getReleaseType(M_PRODUCE, meta), @@ -472,7 +475,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state))))(meta.p) - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( @@ -480,7 +483,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { grantExclusive -> Mux(isWrite(cmd), clientExclusiveDirty, clientExclusiveClean), grantExclusiveAck -> clientExclusiveDirty))))(meta.p) - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, @@ -493,7 +496,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { // notification msg to track clean drops) // Also could avoid probes on outer WBs. - def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = + def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -505,7 +508,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -517,13 +520,14 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { acquireShared -> probeDowngrade, acquireExclusive -> probeInvalidate))) - def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = - Mux(a.a_type === acquireShared, - Mux(!dir.none(meta.sharers), grantShared, grantExclusive), - grantExclusive) + def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), + Mux(a.a_type === acquireShared, + Mux(!dir.none(meta.sharers), grantShared, grantExclusive), + grantExclusive)) def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, @@ -569,7 +573,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_CLEAN -> Mux(dirty, releaseCopyData, releaseCopyAck))) } - def getReleaseType(incoming: Probe, meta: ClientMetadata): UInt = { + def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = { val dirty = clientStatesWithDirtyData.contains(meta.state) val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), @@ -605,7 +609,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d clientExclusiveDirty -> clientExclusiveClean, clientMigratoryDirty -> clientMigratoryClean)))))(meta.p) - def clientMetadataOnGrant(incoming: Grant, cmd: UInt, meta: ClientMetadata) = + def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = ClientMetadata( Mux(incoming.isBuiltInType(), clientInvalid, MuxLookup(incoming.g_type, clientInvalid, Array( @@ -615,7 +619,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean)))))(meta.p) - def clientMetadataOnProbe(incoming: Probe, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, @@ -631,7 +635,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d // Manager states and functions: val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types - def requiresProbes(a: AcquireMetadata, meta: ManagerMetadata) = + def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) @@ -643,7 +647,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d M_FLUSH -> probeInvalidate, M_PRODUCE -> probeDowngrade)) - def getProbeType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getProbeType(a: HasAcquireType, meta: ManagerMetadata): UInt = Mux(a.isBuiltInType(), MuxLookup(a.a_type, probeCopy, Array( Acquire.getBlockType -> probeCopy, @@ -656,14 +660,15 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireExclusive -> probeInvalidate, acquireInvalidateOthers -> probeInvalidateOthers))) - def getGrantType(a: AcquireMetadata, meta: ManagerMetadata): UInt = + def getGrantType(a: HasAcquireType, meta: ManagerMetadata): UInt = + Mux(a.isBuiltInType(), Acquire.getBuiltInGrantType(a.a_type), MuxLookup(a.a_type, grantShared, Array( acquireShared -> Mux(!dir.none(meta.sharers), grantShared, grantExclusive), acquireExclusive -> grantExclusive, - acquireInvalidateOthers -> grantExclusiveAck)) //TODO: add this to MESI for broadcast? + acquireInvalidateOthers -> grantExclusiveAck))) //TODO: add this to MESI for broadcast? def getExclusiveGrantType(): UInt = grantExclusive - def managerMetadataOnRelease(incoming: ReleaseMetadata, src: UInt, meta: ManagerMetadata) = { + def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) MuxBundle(meta, Array( incoming.is(releaseInvalidateData) -> popped, diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 5a514941..8efad8bf 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -174,7 +174,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) /** Does this [[uncore.Acquire]] require [[uncore.Probe Probes]] to be sent */ - def requiresProbes(acq: AcquireMetadata): Bool = co.requiresProbes(acq, this) + def requiresProbes(acq: HasAcquireType): Bool = co.requiresProbes(acq, this) /** Does this memory op require [[uncore.Probe Probes]] to be sent */ def requiresProbes(op_code: UInt): Bool = co.requiresProbes(op_code, this) /** Does an eviction require [[uncore.Probe Probes]] to be sent */ @@ -185,8 +185,17 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * * @param dst Destination client id for this Probe * @param acq Acquire message triggering this Probe + * @param addr_block address of the cache block being probed */ - def makeProbe(dst: UInt, acq: AcquireMetadata): ProbeToDst = + def makeProbe(dst: UInt, acq: HasAcquireType, addr_block: UInt): ProbeToDst = + Probe(dst, co.getProbeType(acq, this), addr_block)(p) + + /** Construct an appropriate [[uncore.ProbeToDst]] for a given [[uncore.Acquire]] + * + * @param dst Destination client id for this Probe + * @param acq Acquire message triggering this Probe + */ + def makeProbe(dst: UInt, acq: AcquireMetadata): ProbeToDst = Probe(dst, co.getProbeType(acq, this), acq.addr_block)(p) /** Construct an appropriate [[uncore.ProbeToDst]] for a given mem op @@ -236,9 +245,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { Grant( dst = acq.client_id, is_builtin_type = acq.isBuiltInType(), - g_type = Mux(acq.isBuiltInType(), - acq.getBuiltInGrantType(), - co.getGrantType(acq, this)), + g_type = co.getGrantType(acq, this), client_xact_id = acq.client_xact_id, manager_xact_id = manager_xact_id, addr_beat = addr_beat, @@ -249,19 +256,22 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { * Used to respond to secondary misses merged into this transaction. * May contain single or multiple beats of data. * - * @param pri Primary miss's Acquire message, used to get g_type and dst - * @param sec Secondary miss info, used to get beat and client_xact_id + * @param sec Secondary miss info * @param manager_xact_id manager's transaction id * @param data data being refilled to the original requestor */ def makeGrant( - pri: AcquireMetadata with HasClientId, sec: SecondaryMissInfo, - manager_xact_id: UInt, + manager_xact_id: UInt, data: UInt): GrantToDst = { - val g = makeGrant(pri, manager_xact_id, sec.addr_beat, data) - g.client_xact_id := sec.client_xact_id - g + Grant( + dst = sec.client_id, + is_builtin_type = sec.isBuiltInType(), + g_type = co.getGrantType(sec, this), + client_xact_id = sec.client_xact_id, + manager_xact_id = manager_xact_id, + addr_beat = sec.addr_beat, + data = data)(p) } /** New metadata after receiving a [[uncore.ReleaseFromSrc]] diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index dea63245..f76b4f28 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -143,26 +143,11 @@ trait HasClientId extends HasTileLinkParameters { val client_id = UInt(width = tlClientIdBits) } -/** TileLink channel bundle definitions */ - -/** The Acquire channel is used to intiate coherence protocol transactions in - * order to gain access to a cache block's data with certain permissions - * enabled. Messages sent over this channel may be custom types defined by - * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types - * used for uncached data accesses. Acquires may contain data for Put or - * PutAtomic built-in types. After sending an Acquire, clients must - * wait for a manager to send them a [[uncore.Grant]] message in response. - */ -class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p) - with HasCacheBlockAddress - with HasClientTransactionId - with HasTileLinkBeatId { - // Actual bundle fields: - val is_builtin_type = Bool() - val a_type = UInt(width = tlAcquireTypeBits) +trait HasAcquireUnion extends HasTileLinkParameters { val union = Bits(width = tlAcquireUnionBits) // Utility funcs for accessing subblock union: + def isBuiltInType(t: UInt): Bool val opCodeOff = 1 val opSizeOff = tlMemoryOpcodeBits + opCodeOff val addrByteOff = tlMemoryOperandSizeBits + opSizeOff @@ -177,22 +162,25 @@ class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - private def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) /** Bit offset of [[uncore.PutAtomic]] operand */ def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ - def wmask(dummy: Int = 0) = + def wmask(dummy: Int = 0): UInt = { Mux(isBuiltInType(Acquire.putAtomicType), FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) + } /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) - /** Complete physical address for block, beat or operand */ - def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) +} + +trait HasAcquireType extends HasTileLinkParameters { + val is_builtin_type = Bool() + val a_type = UInt(width = tlAcquireTypeBits) - // Other helper functions: /** Message type equality */ def is(t: UInt) = a_type === t //TODO: make this more opaque; def ===? @@ -219,31 +207,93 @@ class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p */ def requiresSelfProbe(dummy: Int = 0) = Bool(false) - /** Mapping between each built-in Acquire type (defined in companion object) - * and a built-in Grant type. - */ - def getBuiltInGrantType(dummy: Int = 0): UInt = { - MuxLookup(this.a_type, Grant.putAckType, Array( - Acquire.getType -> Grant.getDataBeatType, - Acquire.getBlockType -> Grant.getDataBlockType, - Acquire.putType -> Grant.putAckType, - Acquire.putBlockType -> Grant.putAckType, - Acquire.putAtomicType -> Grant.getDataBeatType, - Acquire.prefetchType -> Grant.prefetchAckType)) - } + /** Mapping between each built-in Acquire type and a built-in Grant type. */ + def getBuiltInGrantType(dummy: Int = 0): UInt = Acquire.getBuiltInGrantType(this.a_type) +} + +trait HasProbeType extends HasTileLinkParameters { + val p_type = UInt(width = tlCoh.probeTypeWidth) + + def is(t: UInt) = p_type === t + def hasData(dummy: Int = 0) = Bool(false) + def hasMultibeatData(dummy: Int = 0) = Bool(false) +} + +trait HasReleaseType extends HasTileLinkParameters { + val voluntary = Bool() + val r_type = UInt(width = tlCoh.releaseTypeWidth) + + def is(t: UInt) = r_type === t + def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) + def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && + tlCoh.releaseTypesWithData.contains(r_type) + def isVoluntary(dummy: Int = 0) = voluntary + def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) +} + +trait HasGrantType extends HasTileLinkParameters { + val is_builtin_type = Bool() + val g_type = UInt(width = tlGrantTypeBits) + + // Helper funcs + def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type + def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t + def is(t: UInt):Bool = g_type === t + def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), + Grant.typesWithData.contains(g_type), + tlCoh.grantTypesWithData.contains(g_type)) + def hasMultibeatData(dummy: Int = 0): Bool = + Bool(tlDataBeats > 1) && Mux(isBuiltInType(), + Grant.typesWithMultibeatData.contains(g_type), + tlCoh.grantTypesWithData.contains(g_type)) + def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) + def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() +} + +/** TileLink channel bundle definitions */ + +/** The Acquire channel is used to intiate coherence protocol transactions in + * order to gain access to a cache block's data with certain permissions + * enabled. Messages sent over this channel may be custom types defined by + * a [[uncore.CoherencePolicy]] for cached data accesse or may be built-in types + * used for uncached data accesses. Acquires may contain data for Put or + * PutAtomic built-in types. After sending an Acquire, clients must + * wait for a manager to send them a [[uncore.Grant]] message in response. + */ +class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel + with HasCacheBlockAddress + with HasClientTransactionId + with HasTileLinkBeatId + with HasAcquireType + with HasAcquireUnion { + /** Complete physical address for block, beat or operand */ + def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) } /** [[uncore.AcquireMetadata]] with an extra field containing the data beat */ -class Acquire(implicit p: Parameters) extends AcquireMetadata()(p) with HasTileLinkData +class Acquire(implicit p: Parameters) extends AcquireMetadata + with HasTileLinkData /** [[uncore.AcquireMetadata]] with an extra field containing the entire cache block */ -class BufferedAcquire(implicit p: Parameters) extends AcquireMetadata()(p) with HasTileLinkBlock +class BufferedAcquire(implicit p: Parameters) extends AcquireMetadata + with HasTileLinkBlock /** [[uncore.Acquire]] with an extra field stating its source id */ -class AcquireFromSrc(implicit p: Parameters) extends Acquire()(p) with HasClientId +class AcquireFromSrc(implicit p: Parameters) extends Acquire + with HasClientId /** [[uncore.BufferedAcquire]] with an extra field stating its source id */ -class BufferedAcquireFromSrc(implicit p: Parameters) extends BufferedAcquire()(p) with HasClientId +class BufferedAcquireFromSrc(implicit p: Parameters) extends BufferedAcquire + with HasClientId + +/** Used to track metadata for transactions where multiple secondary misses have been merged + * and handled by a single transaction tracker. + */ +class SecondaryMissInfo(implicit p: Parameters) extends TLBundle + with HasClientTransactionId + with HasTileLinkBeatId + with HasClientId + with HasAcquireType /** Contains definitions of the the built-in Acquire types and a factory * for [[uncore.Acquire]] @@ -273,6 +323,33 @@ object Acquire { def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) + /** Mapping between each built-in Acquire type and a built-in Grant type. */ + def getBuiltInGrantType(a_type: UInt): UInt = { + MuxLookup(a_type, Grant.putAckType, Array( + Acquire.getType -> Grant.getDataBeatType, + Acquire.getBlockType -> Grant.getDataBlockType, + Acquire.putType -> Grant.putAckType, + Acquire.putBlockType -> Grant.putAckType, + Acquire.putAtomicType -> Grant.getDataBeatType, + Acquire.prefetchType -> Grant.prefetchAckType)) + } + + def makeUnion( + a_type: UInt, + addr_byte: UInt, + operand_size: UInt, + opcode: UInt, + wmask: UInt, + alloc: Bool): UInt = { + MuxLookup(a_type, UInt(0), Array( + Acquire.getType -> Cat(addr_byte, operand_size, opcode, alloc), + Acquire.getBlockType -> Cat(operand_size, opcode, alloc), + Acquire.putType -> Cat(wmask, alloc), + Acquire.putBlockType -> Cat(wmask, alloc), + Acquire.putAtomicType -> Cat(addr_byte, operand_size, opcode, alloc), + Acquire.prefetchType -> Cat(opcode, alloc))) + } + def fullWriteMask(implicit p: Parameters) = SInt(-1, width = p(TLKey(p(TLId))).writeMaskBits).toUInt // Most generic constructor @@ -295,6 +372,7 @@ object Acquire { acq.union := union acq } + // Copy constructor def apply(a: Acquire): Acquire = { val acq = Wire(new Acquire()(a.p)) @@ -303,6 +381,30 @@ object Acquire { } } +object BuiltInAcquireBuilder { + def apply( + a_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0), + addr_byte: UInt = UInt(0), + operand_size: UInt = MT_Q, + opcode: UInt = UInt(0), + wmask: UInt = UInt(0), + alloc: Bool = Bool(true)) + (implicit p: Parameters): Acquire = { + Acquire( + is_builtin_type = Bool(true), + a_type = a_type, + client_xact_id = client_xact_id, + addr_block = addr_block, + addr_beat = addr_beat, + data = data, + union = Acquire.makeUnion(a_type, addr_byte, operand_size, opcode, wmask, alloc)) + } +} + /** Get a single beat of data from the outer memory hierarchy * * The client can hint whether he block containing this beat should be @@ -322,13 +424,13 @@ object Get { addr_beat: UInt, alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - union = Cat(MT_Q, M_XRD, alloc)) + opcode = M_XRD, + alloc = alloc) } def apply( client_xact_id: UInt, @@ -338,13 +440,15 @@ object Get { operand_size: UInt, alloc: Bool) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.getType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, - union = Cat(addr_byte, operand_size, M_XRD, alloc)) + addr_byte = addr_byte, + operand_size = operand_size, + opcode = M_XRD, + alloc = alloc) } } @@ -363,12 +467,12 @@ object GetBlock { addr_block: UInt, alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.getBlockType, client_xact_id = client_xact_id, addr_block = addr_block, - union = Cat(MT_Q, M_XRD, alloc)) + opcode = M_XRD, + alloc = alloc) } } @@ -383,13 +487,11 @@ object GetPrefetch { client_xact_id: UInt, addr_block: UInt) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.prefetchType, client_xact_id = client_xact_id, addr_block = addr_block, - addr_beat = UInt(0), - union = Cat(MT_Q, M_XRD, Bool(true))) + opcode = M_XRD) } } @@ -413,14 +515,14 @@ object Put { wmask: Option[UInt]= None, alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.putType, addr_block = addr_block, addr_beat = addr_beat, client_xact_id = client_xact_id, data = data, - union = Cat(wmask.getOrElse(Acquire.fullWriteMask), alloc)) + wmask = wmask.getOrElse(Acquire.fullWriteMask), + alloc = alloc) } } @@ -445,14 +547,14 @@ object PutBlock { data: UInt, wmask: UInt) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(wmask, (wmask != Acquire.fullWriteMask))) + wmask = wmask, + alloc = (wmask != Acquire.fullWriteMask)) } def apply( client_xact_id: UInt, @@ -461,14 +563,14 @@ object PutBlock { data: UInt, alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.putBlockType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(Acquire.fullWriteMask, alloc)) + wmask = Acquire.fullWriteMask, + alloc = alloc) } } @@ -483,13 +585,11 @@ object PutPrefetch { client_xact_id: UInt, addr_block: UInt) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.prefetchType, client_xact_id = client_xact_id, addr_block = addr_block, - addr_beat = UInt(0), - union = Cat(M_XWR, Bool(true))) + opcode = M_XWR) } } @@ -513,14 +613,15 @@ object PutAtomic { operand_size: UInt, data: UInt) (implicit p: Parameters): Acquire = { - Acquire( - is_builtin_type = Bool(true), + BuiltInAcquireBuilder( a_type = Acquire.putAtomicType, client_xact_id = client_xact_id, addr_block = addr_block, addr_beat = addr_beat, data = data, - union = Cat(addr_byte, operand_size, atomic_opcode, Bool(true))) + addr_byte = addr_byte, + operand_size = operand_size, + opcode = atomic_opcode) } } @@ -529,14 +630,9 @@ object PutAtomic { * The available types of Probes are customized by a particular * [[uncore.CoherencePolicy]]. */ -class Probe(implicit p: Parameters) extends ManagerToClientChannel()(p) - with HasCacheBlockAddress { - val p_type = UInt(width = tlCoh.probeTypeWidth) - - def is(t: UInt) = p_type === t - def hasData(dummy: Int = 0) = Bool(false) - def hasMultibeatData(dummy: Int = 0) = Bool(false) -} +class Probe(implicit p: Parameters) extends ManagerToClientChannel + with HasCacheBlockAddress + with HasProbeType /** [[uncore.Probe]] with an extra field stating its destination id */ class ProbeToDst(implicit p: Parameters) extends Probe()(p) with HasClientId @@ -573,34 +669,29 @@ object Probe { * a particular [[uncore.CoherencePolicy]]. Releases may contain data or may be * simple acknowledgements. Voluntary Releases are acknowledged with [[uncore.Grant Grants]]. */ -class ReleaseMetadata(implicit p: Parameters) extends ClientToManagerChannel()(p) +class ReleaseMetadata(implicit p: Parameters) extends ClientToManagerChannel with HasTileLinkBeatId with HasCacheBlockAddress - with HasClientTransactionId { - val r_type = UInt(width = tlCoh.releaseTypeWidth) - val voluntary = Bool() - - // Helper funcs - def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) - //TODO: Assumes all releases write back full cache blocks: - def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && tlCoh.releaseTypesWithData.contains(r_type) - def isVoluntary(dummy: Int = 0) = voluntary - def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) + with HasClientTransactionId + with HasReleaseType { def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, UInt(0, width = tlByteAddrBits)) } /** [[uncore.ReleaseMetadata]] with an extra field containing the data beat */ -class Release(implicit p: Parameters) extends ReleaseMetadata()(p) with HasTileLinkData +class Release(implicit p: Parameters) extends ReleaseMetadata + with HasTileLinkData /** [[uncore.ReleaseMetadata]] with an extra field containing the entire cache block */ -class BufferedRelease(implicit p: Parameters) extends ReleaseMetadata()(p) with HasTileLinkBlock +class BufferedRelease(implicit p: Parameters) extends ReleaseMetadata + with HasTileLinkBlock /** [[uncore.Release]] with an extra field stating its source id */ -class ReleaseFromSrc(implicit p: Parameters) extends Release()(p) with HasClientId +class ReleaseFromSrc(implicit p: Parameters) extends Release + with HasClientId /** [[uncore.BufferedRelease]] with an extra field stating its source id */ -class BufferedReleaseFromSrc(implicit p: Parameters) extends BufferedRelease()(p) with HasClientId +class BufferedReleaseFromSrc(implicit p: Parameters) extends BufferedRelease + with HasClientId /** Contains a [[uncore.Release]] factory * @@ -641,26 +732,11 @@ object Release { * coherence policies may also define custom Grant types. Grants may contain data * or may be simple acknowledgements. Grants are responded to with [[uncore.Finish]]. */ -class GrantMetadata(implicit p: Parameters) extends ManagerToClientChannel()(p) +class GrantMetadata(implicit p: Parameters) extends ManagerToClientChannel with HasTileLinkBeatId with HasClientTransactionId - with HasManagerTransactionId { - val is_builtin_type = Bool() - val g_type = UInt(width = tlGrantTypeBits) - - // Helper funcs - def isBuiltInType(dummy: Int = 0): Bool = is_builtin_type - def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t - def is(t: UInt):Bool = g_type === t - def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), - Grant.typesWithData.contains(g_type), - tlCoh.grantTypesWithData.contains(g_type)) - def hasMultibeatData(dummy: Int = 0): Bool = - Bool(tlDataBeats > 1) && Mux(isBuiltInType(), - Grant.typesWithMultibeatData.contains(g_type), - tlCoh.grantTypesWithData.contains(g_type)) - def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) - def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() + with HasManagerTransactionId + with HasGrantType { def makeFinish(dummy: Int = 0): Finish = { val f = Wire(new Finish) f.manager_xact_id := this.manager_xact_id @@ -669,16 +745,20 @@ class GrantMetadata(implicit p: Parameters) extends ManagerToClientChannel()(p) } /** [[uncore.GrantMetadata]] with an extra field containing a single beat of data */ -class Grant(implicit p: Parameters) extends GrantMetadata()(p) with HasTileLinkData +class Grant(implicit p: Parameters) extends GrantMetadata + with HasTileLinkData /** [[uncore.Grant]] with an extra field stating its destination */ -class GrantToDst(implicit p: Parameters) extends Grant()(p) with HasClientId +class GrantToDst(implicit p: Parameters) extends Grant + with HasClientId /** [[uncore.GrantMetadata]] with an extra field containing an entire cache block */ -class BufferedGrant(implicit p: Parameters) extends GrantMetadata()(p) with HasTileLinkBlock +class BufferedGrant(implicit p: Parameters) extends GrantMetadata + with HasTileLinkBlock /** [[uncore.BufferedGrant]] with an extra field stating its destination */ -class BufferedGrantToDst(implicit p: Parameters) extends BufferedGrant()(p) with HasClientId +class BufferedGrantToDst(implicit p: Parameters) extends BufferedGrant + with HasClientId /** Contains definitions of the the built-in grant types and factories * for [[uncore.Grant]] and [[uncore.GrantToDst]] @@ -854,14 +934,6 @@ class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { io.out.release.valid := Bool(false) } -/** Used to track metadata for transactions where multiple secondary misses have been merged - * and handled by a single transaction tracker. - */ -class SecondaryMissInfo(implicit p: Parameters) extends TLBundle()(p) - with HasTileLinkBeatId - with HasClientTransactionId -// TODO: add a_type to merge e.g. Get+GetBlocks, and/or HasClientId - /** A helper module that automatically issues [[uncore.Finish]] messages in repsonse * to [[uncore.Grant]] that it receives from a manager and forwards to a client */ @@ -1264,8 +1336,10 @@ trait HasDataBeatCounters { } /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ - def connectOutgoingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T], beat: UInt = UInt(0)): (UInt, Bool) = - connectDataBeatCounter(in.fire(), in.bits, beat) + def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( + out: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(out.fire(), out.bits, beat) /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on * incoming channels in case of network reordering. diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index d4822a49..04a38c71 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -132,11 +132,11 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) - def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = - addPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = - addPendingBitWhenBeat(in.fire() && in.bits.hasData() && in.bits.allocate(), in.bits) + addPendingBitWhenBeatHasData(in, in.bits.allocate()) def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { val a = in.bits @@ -145,6 +145,9 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) } + def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt = + Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat)) + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) From 0290635454be0c67902448d5b6f83f756194d489 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Nov 2015 19:07:58 -0800 Subject: [PATCH 483/688] amo_shift_bits -> amo_shift_bytes --- uncore/src/main/scala/cache.scala | 10 +++++----- uncore/src/main/scala/tilelink.scala | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index baa7fed8..c5322e32 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -541,7 +541,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val xact_old_meta = Reg{ new L2Metadata } val pending_coh = Reg{ xact_old_meta.coh } val xact_allocate = Reg{ Bool() } - val xact_amo_shift_bits = Reg{ UInt() } + val xact_amo_shift_bytes = Reg{ UInt() } val xact_op_code = Reg{ UInt() } val xact_addr_byte = Reg{ UInt() } val xact_op_size = Reg{ UInt() } @@ -629,12 +629,12 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { val old_data = incoming // Refilled, written back, or de-cached data val new_data = data_buffer(beat) // Newly Put data is already in the buffer - amoalu.io.lhs := old_data >> xact_amo_shift_bits - amoalu.io.rhs := new_data >> xact_amo_shift_bits + amoalu.io.lhs := old_data >> (xact_amo_shift_bytes << 3) + amoalu.io.rhs := new_data >> (xact_amo_shift_bytes << 3) val wmask = FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), - amoalu.io.out << xact_amo_shift_bits, + amoalu.io.out << (xact_amo_shift_bytes << 3), new_data) wmask_buffer(beat) := ~UInt(0, wmask_buffer.head.getWidth) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } @@ -853,7 +853,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra when(state === s_idle && io.inner.acquire.valid) { xact_addr_block := io.iacq().addr_block xact_allocate := io.iacq().allocate() - xact_amo_shift_bits := io.iacq().amo_shift_bits() + xact_amo_shift_bytes := io.iacq().amo_shift_bytes() xact_op_code := io.iacq().op_code() xact_addr_byte := io.iacq().addr_byte() xact_op_size := io.iacq().op_size() diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index f76b4f28..32aa96f0 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -79,6 +79,7 @@ trait HasTileLinkParameters { val tlNetworkPreservesPointToPointOrdering = false val tlNetworkDoesNotInterleaveBeats = true val amoAluOperandBits = p(AmoAluOperandBits) + val amoAluOperandBytes = amoAluOperandBits/8 } abstract class TLModule(implicit val p: Parameters) extends Module @@ -162,9 +163,9 @@ trait HasAcquireUnion extends HasTileLinkParameters { def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBits/8)) + def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBytes)) /** Bit offset of [[uncore.PutAtomic]] operand */ - def amo_shift_bits(dummy: Int = 0) = UInt(amoAluOperandBits)*amo_offset() + def amo_shift_bytes(dummy: Int = 0) = UInt(amoAluOperandBytes)*amo_offset() /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0): UInt = { Mux(isBuiltInType(Acquire.putAtomicType), From d426ecee78cb7ef8d707cae94990da3d640bed87 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 16 Nov 2015 21:52:24 -0800 Subject: [PATCH 484/688] Remove IPI network This is now provided via MMIO. --- uncore/src/main/scala/htif.scala | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 8dc0216c..d5b1df3d 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -41,8 +41,6 @@ class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) val csr = new SMIIO(scrDataBits, 12).flip - val ipi_req = Decoupled(Bits(width = log2Up(nCores))) - val ipi_rep = Decoupled(Bool()).flip val debug_stats_csr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work @@ -176,7 +174,6 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val csrReadData = Reg(Bits(width = io.cpu(0).csr.resp.bits.getWidth)) for (i <- 0 until nCores) { val my_reset = Reg(init=Bool(true)) - val my_ipi = Reg(init=Bool(false)) val cpu = io.cpu(i) val me = csr_coreid === UInt(i) @@ -186,17 +183,6 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt cpu.csr.req.bits.data := csr_wdata cpu.reset := my_reset - when (cpu.ipi_rep.ready) { - my_ipi := Bool(false) - } - cpu.ipi_rep.valid := my_ipi - cpu.ipi_req.ready := Bool(true) - for (j <- 0 until nCores) { - when (io.cpu(j).ipi_req.valid && io.cpu(j).ipi_req.bits === UInt(i)) { - my_ipi := Bool(true) - } - } - when (cpu.csr.req.fire()) { state := state_csr_resp } when (state === state_csr_req && me && csr_addr === UInt(csr_RESET)) { From 2b977325e3e7a6f1a85039605c7d6bf839db34f8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 16 Nov 2015 23:26:13 -0800 Subject: [PATCH 485/688] Make prefetch type available in a_type, issue probeInvalidates for putPrefetches --- uncore/src/main/scala/broadcast.scala | 2 +- uncore/src/main/scala/cache.scala | 9 ++++---- uncore/src/main/scala/coherence.scala | 10 +++++++++ uncore/src/main/scala/tilelink.scala | 32 ++++++++++++++------------- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index cc908348..0b78b546 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -219,7 +219,7 @@ class BroadcastAcquireTracker(trackerId: Int) val coh = ManagerMetadata.onReset assert(!(state != s_idle && xact.isBuiltInType() && - Vec(Acquire.putAtomicType, Acquire.prefetchType).contains(xact.a_type)), + Vec(Acquire.putAtomicType, Acquire.getPrefetchType, Acquire.putPrefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics or prefetches") // TODO val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index c5322e32..bdc92d29 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -663,10 +663,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val allowedTypes = List((Acquire.getType, Acquire.getType), (Acquire.putType, Acquire.putType), (Acquire.putBlockType, Acquire.putBlockType), - (Acquire.prefetchType, Acquire.prefetchType), - (Acquire.prefetchType, Acquire.getType), - (Acquire.prefetchType, Acquire.putType), - (Acquire.prefetchType, Acquire.putBlockType)) + (Acquire.getPrefetchType, Acquire.getPrefetchType), + (Acquire.putPrefetchType, Acquire.putPrefetchType), + (Acquire.getPrefetchType, Acquire.getType), + (Acquire.putPrefetchType, Acquire.putType), + (Acquire.putPrefetchType, Acquire.putBlockType)) allowedTypes.map { case(a, b) => xact.isBuiltInType(a) && sec.isBuiltInType(b) }.reduce(_||_) && xact_op_code === sec.op_code() && sec.conflicts(xact_addr_block) && diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 4f4e40a8..2ae5653d 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -186,6 +186,8 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putBlockType -> probeInvalidate, Acquire.getType -> probeCopy, Acquire.putType -> probeInvalidate, + Acquire.getPrefetchType -> probeCopy, + Acquire.putPrefetchType -> probeInvalidate, Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) @@ -286,6 +288,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putBlockType -> probeInvalidate, Acquire.getType -> probeCopy, Acquire.putType -> probeInvalidate, + Acquire.getPrefetchType -> probeCopy, + Acquire.putPrefetchType -> probeInvalidate, Acquire.putAtomicType -> probeInvalidate)), probeInvalidate) @@ -397,6 +401,8 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putBlockType -> probeInvalidate, Acquire.getType -> probeCopy, Acquire.putType -> probeInvalidate, + Acquire.getPrefetchType -> probeCopy, + Acquire.putPrefetchType -> probeInvalidate, Acquire.putAtomicType -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( acquireShared -> probeDowngrade, @@ -515,6 +521,8 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Acquire.putBlockType -> probeInvalidate, Acquire.getType -> probeCopy, Acquire.putType -> probeInvalidate, + Acquire.getPrefetchType -> probeCopy, + Acquire.putPrefetchType -> probeInvalidate, Acquire.putAtomicType -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( acquireShared -> probeDowngrade, @@ -654,6 +662,8 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d Acquire.putBlockType -> probeInvalidate, Acquire.getType -> probeCopy, Acquire.putType -> probeInvalidate, + Acquire.getPrefetchType -> probeCopy, + Acquire.putPrefetchType -> probeInvalidate, Acquire.putAtomicType -> probeInvalidate)), MuxLookup(a.a_type, probeCopy, Array( acquireShared -> probeDowngrade, diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 32aa96f0..7bee71dd 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -194,7 +194,8 @@ trait HasAcquireType extends HasTileLinkParameters { def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) /** Is this message a built-in prefetch message */ - def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.prefetchType) + def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && + (is(Acquire.getPrefetchType) || is(Acquire.putPrefetchType)) /** Does this message contain data? Assumes that no custom message types have data. */ def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) @@ -314,12 +315,13 @@ class SecondaryMissInfo(implicit p: Parameters) extends TLBundle object Acquire { val nBuiltInTypes = 5 //TODO: Use Enum - def getType = UInt("b000") // Get a single beat of data - def getBlockType = UInt("b001") // Get a whole block of data - def putType = UInt("b010") // Put a single beat of data - def putBlockType = UInt("b011") // Put a whole block of data - def putAtomicType = UInt("b100") // Perform an atomic memory op - def prefetchType = UInt("b101") // Prefetch a whole block of data + def getType = UInt("b000") // Get a single beat of data + def getBlockType = UInt("b001") // Get a whole block of data + def putType = UInt("b010") // Put a single beat of data + def putBlockType = UInt("b011") // Put a whole block of data + def putAtomicType = UInt("b100") // Perform an atomic memory op + def getPrefetchType = UInt("b101") // Prefetch a whole block of data + def putPrefetchType = UInt("b110") // Prefetch a whole block of data, with intent to write def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) @@ -332,7 +334,8 @@ object Acquire { Acquire.putType -> Grant.putAckType, Acquire.putBlockType -> Grant.putAckType, Acquire.putAtomicType -> Grant.getDataBeatType, - Acquire.prefetchType -> Grant.prefetchAckType)) + Acquire.getPrefetchType -> Grant.prefetchAckType, + Acquire.putPrefetchType -> Grant.prefetchAckType)) } def makeUnion( @@ -348,7 +351,8 @@ object Acquire { Acquire.putType -> Cat(wmask, alloc), Acquire.putBlockType -> Cat(wmask, alloc), Acquire.putAtomicType -> Cat(addr_byte, operand_size, opcode, alloc), - Acquire.prefetchType -> Cat(opcode, alloc))) + Acquire.getPrefetchType -> Cat(M_XRD, alloc), + Acquire.putPrefetchType -> Cat(M_XWR, alloc))) } def fullWriteMask(implicit p: Parameters) = SInt(-1, width = p(TLKey(p(TLId))).writeMaskBits).toUInt @@ -489,10 +493,9 @@ object GetPrefetch { addr_block: UInt) (implicit p: Parameters): Acquire = { BuiltInAcquireBuilder( - a_type = Acquire.prefetchType, + a_type = Acquire.getPrefetchType, client_xact_id = client_xact_id, - addr_block = addr_block, - opcode = M_XRD) + addr_block = addr_block) } } @@ -587,10 +590,9 @@ object PutPrefetch { addr_block: UInt) (implicit p: Parameters): Acquire = { BuiltInAcquireBuilder( - a_type = Acquire.prefetchType, + a_type = Acquire.putPrefetchType, client_xact_id = client_xact_id, - addr_block = addr_block, - opcode = M_XWR) + addr_block = addr_block) } } From e50c7ad30629f2ba7791cb78af898c731b758393 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 18 Nov 2015 17:05:54 -0800 Subject: [PATCH 486/688] add NASTI error assertions back in --- uncore/src/main/scala/tilelink.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 7bee71dd..422dd803 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1637,6 +1637,9 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) manager_xact_id = UInt(0), addr_beat = UInt(0), data = Bits(0)) + + assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") + assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") } class TileLinkIONarrower(innerTLId: String, outerTLId: String) From 24f7b9f472f50073fb8b5dcc93df6efa83d457d0 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 19 Nov 2015 10:45:54 -0800 Subject: [PATCH 487/688] make sure L2MetadataArray assigns unoccupied way if available --- uncore/src/main/scala/cache.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index bdc92d29..1ad5ceac 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -205,10 +205,14 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) + val s1_free_way = wayMap((w: Int) => !meta.io.resp(w).coh.outer.isValid()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits + val s2_free_way = RegEnable(s1_free_way, s1_clk_en) val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR + val s2_free = s2_free_way.toBits.orR + val s2_free_way_en = PriorityEncoderOH(s2_free_way).toBits val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) val replacer = p(Replacer)() @@ -216,7 +220,7 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - when(!s2_tag_match) { replacer.miss } + when(!s2_tag_match && !s2_free) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) @@ -224,7 +228,9 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { io.resp.bits.meta := Mux(s2_tag_match, L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) - io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) + io.resp.bits.way_en := MuxCase(s2_replaced_way_en, Seq( + s2_tag_match -> s2_tag_match_way, + s2_free -> s2_free_way_en)) } class L2DataReadReq(implicit p: Parameters) extends L2HellaCacheBundle()(p) From 941b64cd623bb376aea75c320055d2c311de8867 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 20 Nov 2015 13:34:07 -0800 Subject: [PATCH 488/688] make partial write-masking PutBlock constructor always set alloc bit --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 422dd803..deea74d8 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -558,7 +558,7 @@ object PutBlock { addr_beat = addr_beat, data = data, wmask = wmask, - alloc = (wmask != Acquire.fullWriteMask)) + alloc = Bool(true)) } def apply( client_xact_id: UInt, From 55a85cc67a8fc61e81d06b823cf4a4c8e3955ced Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 20 Nov 2015 14:09:24 -0800 Subject: [PATCH 489/688] make sure wmask is passed for PutBlock in broadcast hub --- uncore/src/main/scala/broadcast.scala | 5 ++++- uncore/src/main/scala/tilelink.scala | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 0b78b546..fbc52a11 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -270,7 +270,8 @@ class BroadcastAcquireTracker(trackerId: Int) client_xact_id = UInt(trackerId), addr_block = xact.addr_block, addr_beat = oacq_data_cnt, - data = xact.data_buffer(oacq_data_cnt))(outerParams) + data = xact.data_buffer(oacq_data_cnt), + wmask = xact.wmask_buffer(oacq_data_cnt))(outerParams) val oacq_read_beat = Get( client_xact_id = UInt(trackerId), @@ -317,6 +318,7 @@ class BroadcastAcquireTracker(trackerId: Int) io.inner.acquire.ready := Bool(true) when(io.inner.acquire.valid) { xact.data_buffer(io.iacq().addr_beat) := io.iacq().data + xact.wmask_buffer(io.iacq().addr_beat) := io.iacq().wmask() iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true)) } when(iacq_data_done) { collect_iacq_data := Bool(false) } @@ -334,6 +336,7 @@ class BroadcastAcquireTracker(trackerId: Int) when(io.inner.acquire.valid) { xact := io.iacq() xact.data_buffer(UInt(0)) := io.iacq().data + xact.wmask_buffer(UInt(0)) := io.iacq().wmask() collect_iacq_data := io.iacq().hasMultibeatData() iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat val needs_probes = mask_incoherent.orR diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index deea74d8..aea7e8e5 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -137,6 +137,7 @@ trait HasTileLinkData extends HasTileLinkBeatId { /** An entire cache block of data */ trait HasTileLinkBlock extends HasTileLinkParameters { val data_buffer = Vec(tlDataBeats, UInt(width = tlDataBits)) + val wmask_buffer = Vec(tlDataBeats, UInt(width = tlWriteMaskBits)) } /** The id of a client source or destination. Used in managers. */ From 3c95afebc65b5560c1a75a2102c4ae672ecbbd73 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 20 Nov 2015 23:21:14 -0800 Subject: [PATCH 490/688] Shift set index for multi-bank configurations Prior to this commit, the L2 cache banks used the lower bits of the block address as the set index. However, the lower bits are also used to route addresses to different banks. As a result, in multi-bank configurations, only a fraction of the sets in each bank could be accessed. This commit fixes that problem by using the bits ahead of the bank index as the set index, so that all sets in the cache can be accessed. --- uncore/src/main/scala/cache.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 1ad5ceac..0335a14c 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -17,13 +17,15 @@ case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] +case object SetIdxOffset extends Field[Int] trait HasCacheParameters { implicit val p: Parameters val nSets = p(NSets) val blockOffBits = p(CacheBlockOffsetBits) + val idxOffset = p(SetIdxOffset) val idxBits = log2Up(nSets) - val untagBits = blockOffBits + idxBits + val untagBits = blockOffBits + idxOffset + idxBits val tagBits = p(PAddrBits) - untagBits val nWays = p(NWays) val wayBits = log2Up(nWays) @@ -99,8 +101,9 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { - val idxMSB = idxBits-1 - val idxLSB = 0 + val idxLSB = idxOffset + val idxMSB = idxLSB + idxBits - 1 + val tagLSB = idxLSB + idxBits //val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats @@ -467,7 +470,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends io.meta.read.valid := state === s_meta_read io.meta.read.bits.id := UInt(trackerId) io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(idxBits) + io.meta.read.bits.tag := xact.addr_block >> UInt(tagLSB) // Write the voluntarily written back data to this cache pending_writes := (pending_writes & dropPendingBit(io.data.write)) | @@ -491,7 +494,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends io.meta.write.bits.id := UInt(trackerId) io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact.addr_block >> UInt(idxBits) + io.meta.write.bits.data.tag := xact.addr_block >> UInt(tagLSB) io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), xact_old_meta.coh.outer.onHit(M_XWR), @@ -560,7 +563,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Some accessor wires derived from the the above state val xact = ignt_q.io.deq.bits val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) - val xact_addr_tag = xact_addr_block >> UInt(idxBits) + val xact_addr_tag = xact_addr_block >> UInt(tagLSB) // Counters and scoreboard tracking progress made on processing this transaction val pending_irels = connectTwoWayBeatCounter( From 04383a31f578bef6dff37b09a4e16789073429ea Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 21 Nov 2015 10:35:40 -0800 Subject: [PATCH 491/688] Revert "make sure L2MetadataArray assigns unoccupied way if available" This reverts commit 1857f36c1e6f2b2859c724eea6ae3cfb2618f81b. --- uncore/src/main/scala/cache.scala | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0335a14c..58673537 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -208,14 +208,10 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) - val s1_free_way = wayMap((w: Int) => !meta.io.resp(w).coh.outer.isValid()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits - val s2_free_way = RegEnable(s1_free_way, s1_clk_en) val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR - val s2_free = s2_free_way.toBits.orR - val s2_free_way_en = PriorityEncoderOH(s2_free_way).toBits val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) val replacer = p(Replacer)() @@ -223,7 +219,7 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - when(!s2_tag_match && !s2_free) { replacer.miss } + when(!s2_tag_match) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) @@ -231,9 +227,7 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { io.resp.bits.meta := Mux(s2_tag_match, L2Metadata(s2_repl_meta.tag, s2_hit_coh), s2_repl_meta) - io.resp.bits.way_en := MuxCase(s2_replaced_way_en, Seq( - s2_tag_match -> s2_tag_match_way, - s2_free -> s2_free_way_en)) + io.resp.bits.way_en := Mux(s2_tag_match, s2_tag_match_way, s2_replaced_way_en) } class L2DataReadReq(implicit p: Parameters) extends L2HellaCacheBundle()(p) From ee6514e4f4e03b663e91a5b898defe40885ee0bc Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 21 Nov 2015 15:55:11 -0800 Subject: [PATCH 492/688] make sure WritebackUnit sends correct probe addresses --- uncore/src/main/scala/cache.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 58673537..77d5f83d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -17,15 +17,16 @@ case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] -case object SetIdxOffset extends Field[Int] +case object CacheIdBits extends Field[Int] +case object CacheId extends Field[Int] trait HasCacheParameters { implicit val p: Parameters val nSets = p(NSets) val blockOffBits = p(CacheBlockOffsetBits) - val idxOffset = p(SetIdxOffset) + val cacheIdBits = p(CacheIdBits) val idxBits = log2Up(nSets) - val untagBits = blockOffBits + idxOffset + idxBits + val untagBits = blockOffBits + cacheIdBits + idxBits val tagBits = p(PAddrBits) - untagBits val nWays = p(NWays) val wayBits = log2Up(nWays) @@ -101,7 +102,8 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { - val idxLSB = idxOffset + val cacheId = p(CacheId) + val idxLSB = cacheIdBits val idxMSB = idxLSB + idxBits - 1 val tagLSB = idxLSB + idxBits //val blockAddrBits = p(TLBlockAddrBits) @@ -979,7 +981,7 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac val xact = Reg(new L2WritebackReq) val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val xact_addr_block = Cat(xact.tag, xact.idx) + val xact_addr_block = Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) val pending_irels = connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 From 57e82442a180f9abccec842c69b706700e4b8cd6 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 24 Nov 2015 18:12:11 -0800 Subject: [PATCH 493/688] Make LoadGen and StoreGen generic --- uncore/src/main/scala/amoalu.scala | 110 +++++++++++------------------ 1 file changed, 41 insertions(+), 69 deletions(-) diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala index 033873fa..087e81c4 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/amoalu.scala @@ -4,79 +4,50 @@ package uncore import Chisel._ import cde.{Parameters, Field} -abstract class StoreGen(typ: UInt, addr: UInt, dat: UInt) { - val byte = typ === MT_B || typ === MT_BU - val half = typ === MT_H || typ === MT_HU - val word = typ === MT_W || typ === MT_WU - - def mask: UInt - def data: UInt - def wordData: UInt +class StoreGen(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) { + val size = typ(log2Up(log2Up(maxSize)+1)-1,0) + def misaligned = + (addr & ((UInt(1) << size) - UInt(1))(log2Up(maxSize)-1,0)).orR + + def mask = { + var res = UInt(1) + for (i <- 0 until log2Up(maxSize)) { + val upper = Mux(addr(i), res, UInt(0)) | Mux(size >= UInt(i+1), UInt((BigInt(1) << (1 << i))-1), UInt(0)) + val lower = Mux(addr(i), UInt(0), res) + res = Cat(upper, lower) + } + res + } + + protected def genData(i: Int): UInt = + if (i >= log2Up(maxSize)) dat + else Mux(size === UInt(i), Fill(1 << (log2Up(maxSize)-i), dat((8 << i)-1,0)), genData(i+1)) + + def data = genData(0) + def wordData = genData(2) } -class StoreGen64(typ: UInt, addr: UInt, dat: UInt) extends StoreGen(typ, addr, dat) { - def mask = - Mux(byte, Bits( 1) << addr(2,0), - Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), - Mux(word, Bits( 15) << Cat(addr(2), Bits(0,2)), - Bits(255)))) - - def data = - Mux(byte, Fill(8, dat( 7,0)), - Mux(half, Fill(4, dat(15,0)), - wordData)) - def wordData = - Mux(word, Fill(2, dat(31,0)), - dat) +class StoreGenAligned(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) extends StoreGen(typ, addr, dat, maxSize) { + override def genData(i: Int) = dat } -class StoreGenAligned64(typ: UInt, addr: UInt, dat: UInt) extends StoreGen64(typ, addr, dat) { - override def data = dat - override def wordData = dat -} +class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool, maxSize: Int) { + private val t = new StoreGen(typ, addr, dat, maxSize) + private val signed = typ.toSInt >= SInt(0) -class StoreGen32(typ: UInt, addr: UInt, dat: UInt) extends StoreGen(typ, addr, dat){ - override val word = typ === MT_W + private def genData(logMinSize: Int): UInt = { + var res = dat + for (i <- log2Up(maxSize)-1 to logMinSize by -1) { + val pos = 8 << i + val shifted = Mux(addr(i), res(2*pos-1,pos), res(pos-1,0)) + val zeroed = if (i > 0) shifted else Mux(zero, UInt(0), shifted) + res = Cat(Mux(t.size === UInt(i), Fill(8*maxSize-pos, signed && zeroed(pos-1)), res(8*maxSize-1,pos)), zeroed) + } + res + } - def mask = - Mux(byte, Bits( 1) << addr(2,0), - Mux(half, Bits( 3) << Cat(addr(2,1), Bits(0,1)), - Bits( 15))) - - def data = - Mux(byte, Fill(4, dat( 7,0)), - Mux(half, Fill(2, dat(15,0)), - wordData)) - - def wordData = dat - - def size = - Mux(byte, UInt("b000"), - Mux(half, UInt("b001"), - UInt("b010"))) -} - -class LoadGen64(typ: UInt, addr: UInt, dat: UInt, zero: Bool) { - val t = new StoreGen64(typ, addr, dat) - val sign = typ === MT_B || typ === MT_H || typ === MT_W || typ === MT_D - - val wordShift = Mux(addr(2), dat(63,32), dat(31,0)) - val word = Cat(Mux(t.word, Fill(32, sign && wordShift(31)), dat(63,32)), wordShift) - val halfShift = Mux(addr(1), word(31,16), word(15,0)) - val half = Cat(Mux(t.half, Fill(48, sign && halfShift(15)), word(63,16)), halfShift) - val byteShift = Mux(zero, UInt(0), Mux(addr(0), half(15,8), half(7,0))) - val byte = Cat(Mux(zero || t.byte, Fill(56, sign && byteShift(7)), half(63,8)), byteShift) -} - -class LoadGen32(typ: UInt, addr: UInt, dat: UInt) { - val t = new StoreGen32(typ, addr, dat) - val sign = typ === MT_B || typ === MT_H || typ === MT_W - - val word = dat - val halfShift = Mux(addr(1), word(31,16), word(15,0)) - val half = Cat(Mux(t.half, Fill(16, sign && halfShift(15)), word(31,16)), halfShift) - val byteShift = Mux(addr(0), half(15,8), half(7,0)) - val byte = Cat(Mux(t.byte, Fill(24, sign && byteShift(7)), half(31,8)), byteShift) + def wordData = genData(2) + def data = genData(0) } class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends CacheModule()(p) { @@ -91,8 +62,9 @@ class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends Cach val out = Bits(OUTPUT, operandBits) } - val storegen = if(rhsIsAligned) new StoreGenAligned64(io.typ, io.addr, io.rhs) - else new StoreGen64(io.typ, io.addr, io.rhs) + val storegen = + if(rhsIsAligned) new StoreGenAligned(io.typ, io.addr, io.rhs, operandBits/8) + else new StoreGen(io.typ, io.addr, io.rhs, operandBits/8) val rhs = storegen.wordData val sgned = io.cmd === M_XA_MIN || io.cmd === M_XA_MAX From 27df04354f359d3120f35d2c4381a48b8b63db62 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 25 Nov 2015 20:04:31 -0800 Subject: [PATCH 494/688] Add ROM with NASTI interface --- uncore/src/main/scala/NastiROM.scala | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 uncore/src/main/scala/NastiROM.scala diff --git a/uncore/src/main/scala/NastiROM.scala b/uncore/src/main/scala/NastiROM.scala new file mode 100644 index 00000000..ba1cef66 --- /dev/null +++ b/uncore/src/main/scala/NastiROM.scala @@ -0,0 +1,30 @@ +package uncore + +import Chisel._ +import junctions._ +import cde.{Parameters, Field} + +class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { + val io = new NastiIO().flip + val ar = Queue(io.ar, 1) + + // This assumes ROMs are in read-only parts of the address map. + // Reuse b_queue code from NastiErrorSlave if this assumption is bad. + when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } + assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") + io.aw.ready := Bool(false) + io.w.ready := Bool(false) + io.b.valid := Bool(false) + + val byteWidth = io.r.bits.nastiXDataBits / 8 + val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rom = Vec.tabulate(rows) { i => + val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }) + } + val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) + val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data + + io.r <> ar + io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) +} From e52685f2e9270a25a53834d87893836255cddd68 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 25 Nov 2015 20:52:30 -0800 Subject: [PATCH 495/688] Fix LoadGen zero flag --- uncore/src/main/scala/amoalu.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala index 087e81c4..6f083e9f 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/amoalu.scala @@ -40,8 +40,9 @@ class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool, maxSize: Int) { for (i <- log2Up(maxSize)-1 to logMinSize by -1) { val pos = 8 << i val shifted = Mux(addr(i), res(2*pos-1,pos), res(pos-1,0)) - val zeroed = if (i > 0) shifted else Mux(zero, UInt(0), shifted) - res = Cat(Mux(t.size === UInt(i), Fill(8*maxSize-pos, signed && zeroed(pos-1)), res(8*maxSize-1,pos)), zeroed) + val doZero = Bool(i == 0) && zero + val zeroed = Mux(doZero, UInt(0), shifted) + res = Cat(Mux(t.size === UInt(i) || doZero, Fill(8*maxSize-pos, signed && zeroed(pos-1)), res(8*maxSize-1,pos)), zeroed) } res } From 93aa370b87c9028b90fcf9bf80099cda6430393a Mon Sep 17 00:00:00 2001 From: Sagar Karandikar Date: Thu, 3 Dec 2015 14:03:10 -0800 Subject: [PATCH 496/688] yunsup's fix for dgemm-opt assertion failure --- uncore/src/main/scala/tilelink.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index aea7e8e5..21c38a36 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -981,7 +981,7 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e io.finish.valid := q.io.deq.valid q.io.deq.ready := io.finish.ready - io.refill.valid := io.grant.valid + io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid io.refill.bits := g io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready io.ready := q.io.enq.ready From e71293e2ae4ea174183399c19707aa180c7a9f52 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sun, 6 Dec 2015 02:58:12 -0800 Subject: [PATCH 497/688] fix bug in narrower logic --- uncore/src/main/scala/tilelink.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 21c38a36..6be218c3 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1773,7 +1773,8 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) smallget_roq.io.enq.ready, io.out.acquire.ready) - smallget_roq.io.enq.valid := smallget_helper.fire(smallget_roq.io.enq.ready) + smallget_roq.io.enq.valid := smallget_helper.fire( + smallget_roq.io.enq.ready, !sending_put) smallget_roq.io.enq.bits.data := readshift smallget_roq.io.enq.bits.tag := iacq.client_xact_id From ddc79674f9f265b98c611e45be2b214adb4e52cb Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 15 Dec 2015 18:17:19 -0800 Subject: [PATCH 498/688] fix some issues with cache request merging --- uncore/src/main/scala/cache.scala | 50 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 77d5f83d..31fbd0fa 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -423,6 +423,18 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } + def addOtherBits(en: Bool, nBits: Int): UInt = + Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits)) + + def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + addOtherBits(in.fire() && + in.bits.hasMultibeatData() && + in.bits.addr_beat === UInt(0), + in.bits.tlDataBeats) + + def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + ~addPendingBitsOnFirstBeat(in) + def pinAllReadyValidLow[T <: Data](b: Bundle) { b.elements.foreach { _._2 match { @@ -590,7 +602,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val ignt_data_ready = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val ignt_ack_ready = Reg(init = Bool(false)) val pending_meta_write = Reg(init = Bool(false)) // Used to decide when to escape from s_busy @@ -676,7 +687,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra allowedTypes.map { case(a, b) => xact.isBuiltInType(a) && sec.isBuiltInType(b) }.reduce(_||_) && xact_op_code === sec.op_code() && sec.conflicts(xact_addr_block) && - (xact_allocate || xact.isBuiltInType(Acquire.putBlockType)) + xact_allocate } // Actual transaction processing logic begins here: @@ -689,8 +700,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra !io.outer.grant.fire() && !io.data.resp.valid && ignt_q.io.enq.ready && ignt_q.io.deq.valid + val iacq_same_xact = xact.client_xact_id === io.iacq().client_xact_id && + xact.hasMultibeatData() && ignt_q.io.deq.valid && + pending_puts(io.iacq().addr_beat) - io.inner.acquire.ready := state === s_idle || iacq_can_merge + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact // Handling of primary and secondary misses' data and write mask merging when(io.inner.acquire.fire() && io.iacq().hasData()) { @@ -702,12 +716,14 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Enqueue some metadata information that we'll use to make coherence updates with later ignt_q.io.enq.valid := Mux(io.iacq().hasMultibeatData(), - io.inner.acquire.fire() && state === s_idle, + io.inner.acquire.fire() && io.iacq().addr_beat === UInt(0), io.inner.acquire.fire()) ignt_q.io.enq.bits := io.iacq() // Track whether any beats are missing from a PutBlock - pending_puts := (pending_puts & dropPendingBitWhenBeatHasData(io.inner.acquire)) + pending_puts := (pending_puts & + dropPendingBitWhenBeatHasData(io.inner.acquire)) | + addPendingBitsOnFirstBeat(io.inner.acquire) // Begin a transaction by getting the current block metadata io.meta.read.valid := state === s_meta_read @@ -796,7 +812,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // We write data to the cache at this level if it was Put here with allocate flag, // written back dirty, or refilled from outer memory. - pending_writes := (pending_writes & dropPendingBit(io.data.write)) | + pending_writes := (pending_writes & + dropPendingBit(io.data.write) & + dropPendingBitsOnFirstBeat(io.inner.acquire)) | addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant, xact_allocate) @@ -818,10 +836,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | addPendingBitInternal(io.data.resp) - // We can issue a grant for a pending write once the write is committed - ignt_ack_ready := ignt_ack_ready | - io.data.write.fire() | - io.outer.grant.fire() && !io.outer.grant.bits.hasData() + // We can issue a grant for a pending write once all data is + // received and committed to the data array or outer memory + val ignt_ack_ready = !(state === s_idle || state === s_meta_read || + pending_puts.orR || pending_writes.orR || pending_ognt) + ignt_q.io.deq.ready := ignt_data_done io.inner.grant.valid := state === s_busy && ignt_q.io.deq.valid && @@ -876,7 +895,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) pending_resps := UInt(0) ignt_data_ready := UInt(0) - ignt_ack_ready := Bool(false) pending_meta_write := Bool(false) state := s_meta_read } @@ -888,8 +906,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val coh = io.meta.resp.bits.meta.coh val tag_match = io.meta.resp.bits.tag_match val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact_op_code) - else xact.isBuiltInType(Acquire.putBlockType) || - tag_match && coh.outer.isValid()) + else tag_match && coh.outer.isValid()) val needs_writeback = !tag_match && xact_allocate && (coh.outer.requiresVoluntaryWriteback() || @@ -910,14 +927,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val mask_incoherent = mask_self & ~io.incoherent.toBits pending_iprbs := mask_incoherent } - // If a prefetch is a hit, note that we should ack it ASAP - when (is_hit && xact.isPrefetch()) { - ignt_ack_ready := Bool(true) - } // If some kind of Put is marked no-allocate but is already in the cache, // we need to write its data to the data array when (is_hit && !xact_allocate && xact.hasData()) { pending_writes := addPendingBitsFromAcquire(xact) + xact_allocate := Bool(true) } // Next: request writeback, issue probes, query outer memory, or respond state := Mux(needs_writeback, s_wb_req, @@ -950,7 +964,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready - io.has_acquire_match := iacq_can_merge + io.has_acquire_match := iacq_can_merge || iacq_same_xact io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? } From 922b1adc9c334aa9cb9bad510f9db0d5dad5a349 Mon Sep 17 00:00:00 2001 From: Albert Magyar Date: Thu, 10 Dec 2015 16:40:16 -0800 Subject: [PATCH 499/688] Add optional PLRU replacement to the L2 --- uncore/src/main/scala/cache.scala | 73 ++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 31fbd0fa..0fed70b0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -11,6 +11,7 @@ case object NSets extends Field[Int] case object NWays extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] +case object L2Replacer extends Field[() => SeqReplacementPolicy] case object AmoAluOperandBits extends Field[Int] case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] @@ -58,6 +59,68 @@ class RandomReplacement(ways: Int) extends ReplacementPolicy { def hit = {} } +abstract class SeqReplacementPolicy { + def access(set: UInt): Unit + def update(valid: Bool, hit: Bool, set: UInt, way: UInt): Unit + def way: UInt +} + +class SeqRandom(n_ways: Int) extends SeqReplacementPolicy { + val logic = new RandomReplacement(n_ways) + def access(set: UInt) = { } + def update(valid: Bool, hit: Bool, set: UInt, way: UInt) = { + when (valid && !hit) { logic.miss } + } + def way = logic.way +} + +class PseudoLRU(n: Int) +{ + val state_reg = Reg(Bits(width = n)) + def access(way: UInt) { + state_reg := get_next_state(state_reg,way) + } + def get_next_state(state: Bits, way: UInt) = { + var next_state = state + var idx = UInt(1,1) + for (i <- log2Up(n)-1 to 0 by -1) { + val bit = way(i) + val mask = (UInt(1,n) << idx)(n-1,0) + next_state = next_state & ~mask | Mux(bit, UInt(0), mask) + //next_state.bitSet(idx, !bit) + idx = Cat(idx, bit) + } + next_state + } + def replace = get_replace_way(state_reg) + def get_replace_way(state: Bits) = { + var idx = UInt(1,1) + for (i <- 0 until log2Up(n)) + idx = Cat(idx, state(idx)) + idx(log2Up(n)-1,0) + } +} + +class SeqPLRU(n_sets: Int, n_ways: Int) extends SeqReplacementPolicy { + val state = SeqMem(Bits(width = n_ways-1), n_sets) + val logic = new PseudoLRU(n_ways) + val current_state = Wire(Bits()) + val plru_way = logic.get_replace_way(current_state) + val next_state = Wire(Bits()) + + def access(set: UInt) = { + current_state := Cat(state.read(set), Bits(0, width = 1)) + } + + def update(valid: Bool, hit: Bool, set: UInt, way: UInt) = { + val update_way = Mux(hit, way, plru_way) + next_state := logic.get_next_state(current_state, update_way) + when (valid) { state.write(set, next_state(n_ways-1,1)) } + } + + def way = plru_way +} + abstract class Metadata(implicit p: Parameters) extends CacheBundle()(p) { val tag = Bits(width = tagBits) val coh: CoherenceMetadata @@ -212,16 +275,22 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits + val s1_idx = RegEnable(io.read.bits.idx, io.read.valid) // deal with stalls? val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR val s2_hit_coh = Mux1H(s2_tag_match_way, wayMap((w: Int) => RegEnable(meta.io.resp(w).coh, s1_clk_en))) - val replacer = p(Replacer)() + val replacer = p(L2Replacer)() + val s1_hit_way = Wire(Bits()) + s1_hit_way := Bits(0) + (0 until nWays).foreach(i => when (s1_tag_match_way(i)) { s1_hit_way := Bits(i) }) + replacer.access(io.read.bits.idx) + replacer.update(s1_clk_en, s1_tag_match_way.orR, s1_idx, s1_hit_way) + val s1_replaced_way_en = UIntToOH(replacer.way) val s2_replaced_way_en = UIntToOH(RegEnable(replacer.way, s1_clk_en)) val s2_repl_meta = Mux1H(s2_replaced_way_en, wayMap((w: Int) => RegEnable(meta.io.resp(w), s1_clk_en && s1_replaced_way_en(w))).toSeq) - when(!s2_tag_match) { replacer.miss } io.resp.valid := Reg(next = s1_clk_en) io.resp.bits.id := RegEnable(s1_id, s1_clk_en) From a48237f36d7064faf781e6bd279cd2a71420d756 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 16 Dec 2015 20:56:29 -0800 Subject: [PATCH 500/688] get rid of the rest of the PutBlock special casing in L2 --- uncore/src/main/scala/cache.scala | 33 +++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 0fed70b0..30624309 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -492,18 +492,6 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } - def addOtherBits(en: Bool, nBits: Int): UInt = - Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits)) - - def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = - addOtherBits(in.fire() && - in.bits.hasMultibeatData() && - in.bits.addr_beat === UInt(0), - in.bits.tlDataBeats) - - def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = - ~addPendingBitsOnFirstBeat(in) - def pinAllReadyValidLow[T <: Data](b: Bundle) { b.elements.foreach { _._2 match { @@ -702,6 +690,18 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } } + def addOtherBits(en: Bool, nBits: Int): UInt = + Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits)) + + def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + addOtherBits(in.fire() && + in.bits.hasMultibeatData() && + in.bits.addr_beat === UInt(0), + in.bits.tlDataBeats) + + def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + ~addPendingBitsOnFirstBeat(in) + // Defined here because of Chisel default wire demands, used in s_meta_resp val pending_coh_on_hit = HierarchicalMetadata( io.meta.resp.bits.meta.coh.inner, @@ -1008,17 +1008,12 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra Mux(!is_hit, s_outer_acquire, s_busy))) } when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } - when(state === s_wb_resp && io.wb.resp.valid) { - // If we're overwriting the whole block in a last level cache we can - // just do it without fetching any data from memory - val skip_outer_acquire = Bool(isLastLevelCache) && xact.isBuiltInType(Acquire.putBlockType) - state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) - } + when(state === s_wb_resp && io.wb.resp.valid) { state := s_outer_acquire } when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { // Tag matches, so if this is the last level cache we can use the data without upgrading permissions val skip_outer_acquire = (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact_op_code) - else xact.isBuiltInType(Acquire.putBlockType) || xact_old_meta.coh.outer.isValid()) + else xact_old_meta.coh.outer.isValid()) state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) } when(state === s_outer_acquire && oacq_data_done) { state := s_busy } From 8a611772245e8384f6eaf31afa8a990121d7576b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 19 Nov 2015 15:04:09 -0800 Subject: [PATCH 501/688] generalize TwoWayCounter --- uncore/src/main/scala/tilelink.scala | 5 +---- uncore/src/main/scala/util.scala | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 6be218c3..2e02962a 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1386,14 +1386,11 @@ trait HasDataBeatCounters { down: DecoupledIO[S], beat: UInt = UInt(0), track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val cnt = Reg(init = UInt(0, width = log2Up(max+1))) val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) val do_inc = up_done && track(up.bits) val do_dec = down_done - cnt := Mux(do_dec, - Mux(do_inc, cnt, cnt - UInt(1)), - Mux(do_inc, cnt + UInt(1), cnt)) + val cnt = TwoWayCounter(do_inc, do_dec, max) (cnt > UInt(0), up_idx, up_done, down_idx, down_done) } } diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index e5b14e7f..b0036b37 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -42,6 +42,15 @@ object ZCounter { } } +object TwoWayCounter { + def apply(up: Bool, down: Bool, max: Int): UInt = { + val cnt = Reg(init = UInt(0, log2Up(max+1))) + when (up && !down) { cnt := cnt + UInt(1) } + when (down && !up) { cnt := cnt - UInt(1) } + cnt + } +} + class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { val in = Decoupled(gen).flip From 872b162e1bff591e679bb50c2743d0e9e578725c Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 17 Nov 2015 18:13:50 -0800 Subject: [PATCH 502/688] implement DMA engine --- uncore/src/main/scala/dma.scala | 316 +++++++++++++++++++++++++++++++ uncore/src/main/scala/util.scala | 4 + 2 files changed, 320 insertions(+) create mode 100644 uncore/src/main/scala/dma.scala diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala new file mode 100644 index 00000000..3170eec3 --- /dev/null +++ b/uncore/src/main/scala/dma.scala @@ -0,0 +1,316 @@ +package uncore + +import Chisel._ +import cde.{Parameters, Field} +import junctions._ + +case object NDmaTransactors extends Field[Int] +case object NDmaClients extends Field[Int] +case object NDmaXactsPerClient extends Field[Int] + +trait HasDmaParameters { + implicit val p: Parameters + val nDmaTransactors = p(NDmaTransactors) + val nDmaClients = p(NDmaClients) + val nDmaXactsPerClient = p(NDmaXactsPerClient) + val dmaClientXactIdBits = log2Up(nDmaClients * nDmaXactsPerClient) + val addrBits = p(PAddrBits) + val dmaStatusBits = 2 +} + +abstract class DmaModule(implicit val p: Parameters) extends Module with HasDmaParameters +abstract class DmaBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasDmaParameters + +class DmaRequest(implicit p: Parameters) extends DmaBundle()(p) { + val client_xact_id = UInt(width = dmaClientXactIdBits) + val cmd = UInt(width = DmaRequest.DMA_CMD_SZ) + val source = UInt(width = addrBits) + val dest = UInt(width = addrBits) + val length = UInt(width = addrBits) +} + +class DmaResponse(implicit p: Parameters) extends DmaBundle()(p) { + val client_xact_id = UInt(width = dmaClientXactIdBits) + val status = UInt(width = dmaStatusBits) +} + +object DmaRequest { + val DMA_CMD_SZ = 2 + + val DMA_CMD_COPY = UInt(0, DMA_CMD_SZ) + val DMA_CMD_PFR = UInt(2, DMA_CMD_SZ) + val DMA_CMD_PFW = UInt(3, DMA_CMD_SZ) + + def apply(client_xact_id: UInt = UInt(0), + cmd: UInt, + source: UInt, + dest: UInt, + length: UInt)(implicit p: Parameters): DmaRequest = { + val req = Wire(new DmaRequest) + req.client_xact_id := client_xact_id + req.cmd := cmd + req.source := source + req.dest := dest + req.length := length + req + } +} +import DmaRequest._ + +class DmaIO(implicit p: Parameters) extends DmaBundle()(p) { + val req = Decoupled(new DmaRequest) + val resp = Decoupled(new DmaResponse).flip +} + +class DmaEngine(implicit p: Parameters) extends DmaModule()(p) { + val io = new Bundle { + val dma = (new DmaIO).flip + val mem = new ClientUncachedTileLinkIO + } + + val trackers = List.fill(nDmaTransactors) { Module(new DmaTracker) } + val reqReadys = Vec(trackers.map(_.io.dma.req.ready)).toBits + + if (nDmaTransactors > 1) { + val mem_arb = Module(new ClientUncachedTileLinkIOArbiter(nDmaTransactors)) + mem_arb.io.in <> trackers.map(_.io.mem) + io.mem <> mem_arb.io.out + + val resp_arb = Module(new RRArbiter(new DmaResponse, nDmaTransactors)) + resp_arb.io.in <> trackers.map(_.io.dma.resp) + io.dma.resp <> resp_arb.io.out + + val selection = PriorityEncoder(reqReadys) + trackers.zipWithIndex.foreach { case (tracker, i) => + tracker.io.dma.req.valid := io.dma.req.valid && selection === UInt(i) + tracker.io.dma.req.bits := io.dma.req.bits + } + io.dma.req.ready := reqReadys.orR + } else { + io.mem <> trackers.head.io.mem + io.dma <> trackers.head.io.dma + } +} + +class DmaTracker(implicit p: Parameters) extends DmaModule()(p) + with HasTileLinkParameters { + val io = new Bundle { + val dma = (new DmaIO).flip + val mem = new ClientUncachedTileLinkIO + } + + private val blockOffset = tlBeatAddrBits + tlByteAddrBits + private val blockBytes = tlDataBeats * tlDataBytes + + val data_buffer = Reg(Vec(2 * tlDataBeats, Bits(width = tlDataBits))) + val get_inflight = Reg(UInt(2 * tlDataBeats)) + val put_inflight = Reg(Bool()) + val put_half = Reg(UInt(width = 1)) + val get_half = Reg(UInt(width = 1)) + val prefetch_put = Reg(Bool()) + val get_done = !get_inflight.orR + + val src_block = Reg(UInt(width = tlBlockAddrBits)) + val dst_block = Reg(UInt(width = tlBlockAddrBits)) + val offset = Reg(UInt(width = blockOffset)) + val alignment = Reg(UInt(width = blockOffset)) + val shift_dir = Reg(Bool()) + + val bytes_left = Reg(UInt(width = addrBits)) + + val acq = io.mem.acquire.bits + val gnt = io.mem.grant.bits + + val (s_idle :: s_get :: s_put :: s_prefetch :: + s_wait :: s_resp :: Nil) = Enum(Bits(), 6) + val state = Reg(init = s_idle) + + val (put_beat, put_done) = Counter( + io.mem.acquire.fire() && acq.hasData(), tlDataBeats) + + val put_mask = Vec.tabulate(tlDataBytes) { i => + val byte_index = Cat(put_beat, UInt(i, tlByteAddrBits)) + byte_index >= offset && byte_index < bytes_left + }.toBits + + val prefetch_sent = io.mem.acquire.fire() && io.mem.acquire.bits.isPrefetch() + val prefetch_busy = Reg(init = UInt(0, tlMaxClientXacts)) + val (prefetch_id, _) = Counter(prefetch_sent, tlMaxClientXacts) + + val base_index = Cat(put_half, put_beat) + val put_data = Wire(init = Bits(0, tlDataBits)) + val beat_align = alignment(blockOffset - 1, tlByteAddrBits) + val bit_align = Cat(alignment(tlByteAddrBits - 1, 0), UInt(0, 3)) + val rev_align = UInt(tlDataBits) - bit_align + + def getBit(value: UInt, sel: UInt): Bool = + (value >> sel)(0) + + when (alignment === UInt(0)) { + put_data := data_buffer.read(base_index) + } .elsewhen (shift_dir) { + val shift_index = base_index - beat_align + when (bit_align === UInt(0)) { + put_data := data_buffer.read(shift_index) + } .otherwise { + val upper_bits = data_buffer.read(shift_index) + val lower_bits = data_buffer.read(shift_index - UInt(1)) + val upper_shifted = upper_bits << bit_align + val lower_shifted = lower_bits >> rev_align + put_data := upper_shifted | lower_shifted + } + } .otherwise { + val shift_index = base_index + beat_align + when (bit_align === UInt(0)) { + put_data := data_buffer.read(shift_index) + } .otherwise { + val upper_bits = data_buffer.read(shift_index + UInt(1)) + val lower_bits = data_buffer.read(shift_index) + val upper_shifted = upper_bits << rev_align + val lower_shifted = lower_bits >> bit_align + put_data := upper_shifted | lower_shifted + } + } + + val put_acquire = PutBlock( + client_xact_id = UInt(2), + addr_block = dst_block, + addr_beat = put_beat, + data = put_data, + wmask = put_mask) + + val get_acquire = GetBlock( + client_xact_id = get_half, + addr_block = src_block, + alloc = Bool(false)) + + val prefetch_acquire = Mux(prefetch_put, + PutPrefetch(client_xact_id = prefetch_id, addr_block = dst_block), + GetPrefetch(client_xact_id = prefetch_id, addr_block = dst_block)) + + val resp_id = Reg(UInt(width = dmaClientXactIdBits)) + + io.mem.acquire.valid := (state === s_get) || + (state === s_put && get_done) || + (state === s_prefetch && !prefetch_busy(prefetch_id)) + io.mem.acquire.bits := MuxBundle( + state, prefetch_acquire, Seq( + s_get -> get_acquire, + s_put -> put_acquire)) + io.mem.grant.ready := Bool(true) + io.dma.req.ready := state === s_idle + io.dma.resp.valid := state === s_resp + io.dma.resp.bits.client_xact_id := resp_id + io.dma.resp.bits.status := UInt(0) + + when (io.dma.req.fire()) { + val src_off = io.dma.req.bits.source(blockOffset - 1, 0) + val dst_off = io.dma.req.bits.dest(blockOffset - 1, 0) + val direction = src_off < dst_off + + resp_id := io.dma.req.bits.client_xact_id + src_block := io.dma.req.bits.source(addrBits - 1, blockOffset) + dst_block := io.dma.req.bits.dest(addrBits - 1, blockOffset) + alignment := Mux(direction, dst_off - src_off, src_off - dst_off) + shift_dir := direction + offset := dst_off + bytes_left := io.dma.req.bits.length + dst_off + get_inflight := UInt(0) + put_inflight := Bool(false) + get_half := UInt(0) + put_half := UInt(0) + + when (io.dma.req.bits.cmd === DMA_CMD_COPY) { + state := s_get + } .otherwise { + prefetch_put := io.dma.req.bits.cmd(0) + state := s_prefetch + } + } + + when (state === s_get && io.mem.acquire.ready) { + val bytes_in_buffer = UInt(blockBytes) - alignment + val extra_read = alignment > UInt(0) && !shift_dir && // dst_off < src_off + get_half === UInt(0) && // this is the first block + bytes_in_buffer < bytes_left // there is still more data left to fetch + get_inflight := get_inflight | FillInterleaved(tlDataBeats, UIntToOH(get_half)) + get_half := get_half + UInt(1) + src_block := src_block + UInt(1) + when (!extra_read) { + state := s_put + } + } + + when (prefetch_sent) { + prefetch_busy := prefetch_busy | UIntToOH(prefetch_id) + when (bytes_left < UInt(blockBytes)) { + bytes_left := UInt(0) + state := s_resp + } .otherwise { + bytes_left := bytes_left - UInt(blockBytes) + dst_block := dst_block + UInt(1) + } + } + + when (io.mem.grant.fire()) { + when (gnt.g_type === Grant.prefetchAckType) { + prefetch_busy := prefetch_busy & ~UIntToOH(gnt.client_xact_id) + } .elsewhen (gnt.hasData()) { + val write_half = gnt.client_xact_id(0) + val write_idx = Cat(write_half, gnt.addr_beat) + get_inflight := get_inflight & ~UIntToOH(write_idx) + data_buffer.write(write_idx, gnt.data) + } .otherwise { + put_inflight := Bool(false) + } + } + + when (put_done) { // state === s_put + put_half := put_half + UInt(1) + offset := UInt(0) + when (bytes_left < UInt(blockBytes)) { + bytes_left := UInt(0) + } .otherwise { + bytes_left := bytes_left - UInt(blockBytes) + } + put_inflight := Bool(true) + dst_block := dst_block + UInt(1) + state := s_wait + } + + when (state === s_wait && get_done && !put_inflight) { + state := Mux(bytes_left === UInt(0), s_resp, s_get) + } + + when (io.dma.resp.fire()) { state := s_idle } +} + +class DmaArbiter(arbN: Int)(implicit p: Parameters) extends DmaModule()(p) { + val io = new Bundle { + val in = Vec(arbN, new DmaIO).flip + val out = new DmaIO + } + + if (arbN > 1) { + val idBits = log2Up(arbN) + val req_arb = Module(new RRArbiter(new DmaRequest, arbN)) + val out_resp_client_id = io.out.resp.bits.client_xact_id(idBits - 1, 0) + + for (i <- 0 until arbN) { + req_arb.io.in(i) <> io.in(i).req + req_arb.io.in(i).bits.client_xact_id := Cat( + io.in(i).req.bits.client_xact_id, + UInt(i, idBits)) + + io.in(i).resp.valid := io.out.resp.valid && out_resp_client_id === UInt(i) + io.in(i).resp.bits := io.out.resp.bits + } + + val respReadys = Vec(io.in.map(_.resp.ready)) + + io.out.req <> req_arb.io.out + io.out.resp.ready := respReadys(out_resp_client_id) + } else { + io.out <> io.in.head + } +} diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index b0036b37..db1dd6f4 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -17,6 +17,10 @@ object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) } + + def apply[S <: Data, T <: Data] (key: S, default: T, mapping: Seq[(S, T)]): T = { + apply(default, mapping.map{ case (a, b) => (a === key, b) }) + } } // Produces 0-width value when counting to 1 From 46069ea13b09ea8b72c330e802490ada74522771 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Jan 2016 20:04:01 -0800 Subject: [PATCH 503/688] implement streaming DMA functionality --- uncore/src/main/scala/dma.scala | 177 +++++++++++++++++++++++++------- 1 file changed, 139 insertions(+), 38 deletions(-) diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala index 3170eec3..46794ad2 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/dma.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import cde.{Parameters, Field} import junctions._ +import junctions.NastiConstants._ case object NDmaTransactors extends Field[Int] case object NDmaClients extends Field[Int] @@ -16,6 +17,7 @@ trait HasDmaParameters { val dmaClientXactIdBits = log2Up(nDmaClients * nDmaXactsPerClient) val addrBits = p(PAddrBits) val dmaStatusBits = 2 + val dmaWordSizeBits = 2 } abstract class DmaModule(implicit val p: Parameters) extends Module with HasDmaParameters @@ -27,6 +29,7 @@ class DmaRequest(implicit p: Parameters) extends DmaBundle()(p) { val source = UInt(width = addrBits) val dest = UInt(width = addrBits) val length = UInt(width = addrBits) + val size = UInt(width = dmaWordSizeBits) } class DmaResponse(implicit p: Parameters) extends DmaBundle()(p) { @@ -35,23 +38,27 @@ class DmaResponse(implicit p: Parameters) extends DmaBundle()(p) { } object DmaRequest { - val DMA_CMD_SZ = 2 + val DMA_CMD_SZ = 3 - val DMA_CMD_COPY = UInt(0, DMA_CMD_SZ) - val DMA_CMD_PFR = UInt(2, DMA_CMD_SZ) - val DMA_CMD_PFW = UInt(3, DMA_CMD_SZ) + val DMA_CMD_COPY = UInt("b000") + val DMA_CMD_PFR = UInt("b010") + val DMA_CMD_PFW = UInt("b011") + val DMA_CMD_SIN = UInt("b100") + val DMA_CMD_SOUT = UInt("b101") def apply(client_xact_id: UInt = UInt(0), cmd: UInt, source: UInt, dest: UInt, - length: UInt)(implicit p: Parameters): DmaRequest = { + length: UInt, + size: UInt = UInt(0))(implicit p: Parameters): DmaRequest = { val req = Wire(new DmaRequest) req.client_xact_id := client_xact_id req.cmd := cmd req.source := source req.dest := dest req.length := length + req.size := size req } } @@ -62,19 +69,26 @@ class DmaIO(implicit p: Parameters) extends DmaBundle()(p) { val resp = Decoupled(new DmaResponse).flip } +class DmaTrackerIO(implicit p: Parameters) extends DmaBundle()(p) { + val dma = (new DmaIO).flip + val inner = new ClientUncachedTileLinkIO + val outer = new NastiIO +} + class DmaEngine(implicit p: Parameters) extends DmaModule()(p) { - val io = new Bundle { - val dma = (new DmaIO).flip - val mem = new ClientUncachedTileLinkIO - } + val io = new DmaTrackerIO val trackers = List.fill(nDmaTransactors) { Module(new DmaTracker) } val reqReadys = Vec(trackers.map(_.io.dma.req.ready)).toBits if (nDmaTransactors > 1) { - val mem_arb = Module(new ClientUncachedTileLinkIOArbiter(nDmaTransactors)) - mem_arb.io.in <> trackers.map(_.io.mem) - io.mem <> mem_arb.io.out + val inner_arb = Module(new ClientUncachedTileLinkIOArbiter(nDmaTransactors)) + inner_arb.io.in <> trackers.map(_.io.inner) + io.inner <> inner_arb.io.out + + val outer_arb = Module(new NastiArbiter(nDmaTransactors)) + outer_arb.io.master <> trackers.map(_.io.outer) + io.outer <> outer_arb.io.slave val resp_arb = Module(new RRArbiter(new DmaResponse, nDmaTransactors)) resp_arb.io.in <> trackers.map(_.io.dma.resp) @@ -87,17 +101,15 @@ class DmaEngine(implicit p: Parameters) extends DmaModule()(p) { } io.dma.req.ready := reqReadys.orR } else { - io.mem <> trackers.head.io.mem + io.inner <> trackers.head.io.inner + io.outer <> trackers.head.io.outer io.dma <> trackers.head.io.dma } } class DmaTracker(implicit p: Parameters) extends DmaModule()(p) - with HasTileLinkParameters { - val io = new Bundle { - val dma = (new DmaIO).flip - val mem = new ClientUncachedTileLinkIO - } + with HasTileLinkParameters with HasNastiParameters { + val io = new DmaTrackerIO private val blockOffset = tlBeatAddrBits + tlByteAddrBits private val blockBytes = tlDataBeats * tlDataBytes @@ -117,23 +129,46 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) val shift_dir = Reg(Bool()) val bytes_left = Reg(UInt(width = addrBits)) + val streaming = Reg(Bool()) + val stream_addr = Reg(UInt(width = nastiXAddrBits)) + val stream_len = Reg(UInt(width = nastiXLenBits)) + val stream_size = Reg(UInt(width = nastiXSizeBits)) + val stream_idx = Reg(UInt(width = blockOffset)) + val stream_bytesel = MuxLookup(stream_size, UInt("b11111111"), Seq( + UInt("b00") -> UInt("b00000001"), + UInt("b01") -> UInt("b00000011"), + UInt("b10") -> UInt("b00001111"))) + val stream_mask = FillInterleaved(8, stream_bytesel) + val stream_last = Reg(Bool()) - val acq = io.mem.acquire.bits - val gnt = io.mem.grant.bits + val stream_word_bytes = UInt(1) << stream_size + val stream_beat_idx = stream_idx(blockOffset - 1, tlByteAddrBits) + val stream_byte_idx = stream_idx(tlByteAddrBits - 1, 0) + val stream_bitshift = Cat(stream_byte_idx, UInt(0, 3)) + val stream_in_beat = + (((io.outer.r.bits.data & stream_mask) << stream_bitshift)) | + (data_buffer(stream_beat_idx) & ~(stream_mask << stream_bitshift)) + val stream_out_word = data_buffer(stream_beat_idx) >> stream_bitshift + val stream_out_last = bytes_left === stream_word_bytes + + val acq = io.inner.acquire.bits + val gnt = io.inner.grant.bits val (s_idle :: s_get :: s_put :: s_prefetch :: - s_wait :: s_resp :: Nil) = Enum(Bits(), 6) + s_stream_read_req :: s_stream_read_resp :: + s_stream_write_req :: s_stream_write_data :: s_stream_write_resp :: + s_wait :: s_resp :: Nil) = Enum(Bits(), 11) val state = Reg(init = s_idle) val (put_beat, put_done) = Counter( - io.mem.acquire.fire() && acq.hasData(), tlDataBeats) + io.inner.acquire.fire() && acq.hasData(), tlDataBeats) val put_mask = Vec.tabulate(tlDataBytes) { i => val byte_index = Cat(put_beat, UInt(i, tlByteAddrBits)) byte_index >= offset && byte_index < bytes_left }.toBits - val prefetch_sent = io.mem.acquire.fire() && io.mem.acquire.bits.isPrefetch() + val prefetch_sent = io.inner.acquire.fire() && io.inner.acquire.bits.isPrefetch() val prefetch_busy = Reg(init = UInt(0, tlMaxClientXacts)) val (prefetch_id, _) = Counter(prefetch_sent, tlMaxClientXacts) @@ -190,18 +225,39 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) val resp_id = Reg(UInt(width = dmaClientXactIdBits)) - io.mem.acquire.valid := (state === s_get) || + io.inner.acquire.valid := (state === s_get) || (state === s_put && get_done) || (state === s_prefetch && !prefetch_busy(prefetch_id)) - io.mem.acquire.bits := MuxBundle( + io.inner.acquire.bits := MuxBundle( state, prefetch_acquire, Seq( s_get -> get_acquire, s_put -> put_acquire)) - io.mem.grant.ready := Bool(true) + io.inner.grant.ready := Bool(true) io.dma.req.ready := state === s_idle io.dma.resp.valid := state === s_resp io.dma.resp.bits.client_xact_id := resp_id io.dma.resp.bits.status := UInt(0) + io.outer.ar.valid := (state === s_stream_read_req) + io.outer.ar.bits := NastiReadAddressChannel( + id = UInt(0), + addr = stream_addr, + size = stream_size, + len = stream_len, + burst = BURST_FIXED) + io.outer.r.ready := (state === s_stream_read_resp) + + io.outer.aw.valid := (state === s_stream_write_req) + io.outer.aw.bits := NastiWriteAddressChannel( + id = UInt(0), + addr = stream_addr, + size = stream_size, + len = stream_len, + burst = BURST_FIXED) + io.outer.w.valid := (state === s_stream_write_data) && get_done + io.outer.w.bits := NastiWriteDataChannel( + data = stream_out_word, + last = stream_out_last) + io.outer.b.ready := (state === s_stream_write_resp) when (io.dma.req.fire()) { val src_off = io.dma.req.bits.source(blockOffset - 1, 0) @@ -219,25 +275,65 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) put_inflight := Bool(false) get_half := UInt(0) put_half := UInt(0) + streaming := Bool(false) + stream_len := (io.dma.req.bits.length >> io.dma.req.bits.size) - UInt(1) + stream_size := io.dma.req.bits.size + stream_last := Bool(false) when (io.dma.req.bits.cmd === DMA_CMD_COPY) { state := s_get - } .otherwise { + } .elsewhen (io.dma.req.bits.cmd(2, 1) === UInt("b01")) { prefetch_put := io.dma.req.bits.cmd(0) state := s_prefetch + } .elsewhen (io.dma.req.bits.cmd === DMA_CMD_SIN) { + stream_addr := io.dma.req.bits.source + stream_idx := dst_off + streaming := Bool(true) + alignment := UInt(0) + state := s_stream_read_req + } .elsewhen (io.dma.req.bits.cmd === DMA_CMD_SOUT) { + stream_addr := io.dma.req.bits.dest + stream_idx := src_off + streaming := Bool(true) + bytes_left := io.dma.req.bits.length + state := s_stream_write_req } } - when (state === s_get && io.mem.acquire.ready) { - val bytes_in_buffer = UInt(blockBytes) - alignment - val extra_read = alignment > UInt(0) && !shift_dir && // dst_off < src_off - get_half === UInt(0) && // this is the first block - bytes_in_buffer < bytes_left // there is still more data left to fetch + when (io.outer.ar.fire()) { state := s_stream_read_resp } + + when (io.outer.r.fire()) { + data_buffer(stream_beat_idx) := stream_in_beat + stream_idx := stream_idx + stream_word_bytes + val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes + when (block_finished || io.outer.r.bits.last) { state := s_put } + } + + when (io.outer.aw.fire()) { state := s_get } + + when (io.outer.w.fire()) { + stream_idx := stream_idx + stream_word_bytes + bytes_left := bytes_left - stream_word_bytes + val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes + when (stream_out_last) { + state := s_resp + } .elsewhen (block_finished) { + state := s_get + } + } + + when (state === s_get && io.inner.acquire.ready) { get_inflight := get_inflight | FillInterleaved(tlDataBeats, UIntToOH(get_half)) - get_half := get_half + UInt(1) src_block := src_block + UInt(1) - when (!extra_read) { - state := s_put + when (streaming) { + state := s_stream_write_data + } .otherwise { + val bytes_in_buffer = UInt(blockBytes) - alignment + val extra_read = alignment > UInt(0) && !shift_dir && // dst_off < src_off + get_half === UInt(0) && // this is the first block + bytes_in_buffer < bytes_left // there is still more data left to fetch + get_half := get_half + UInt(1) + when (!extra_read) { state := s_put } } } @@ -252,7 +348,7 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) } } - when (io.mem.grant.fire()) { + when (io.inner.grant.fire()) { when (gnt.g_type === Grant.prefetchAckType) { prefetch_busy := prefetch_busy & ~UIntToOH(gnt.client_xact_id) } .elsewhen (gnt.hasData()) { @@ -266,8 +362,11 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) } when (put_done) { // state === s_put - put_half := put_half + UInt(1) + when (!streaming) { + put_half := put_half + UInt(1) + } offset := UInt(0) + stream_idx := UInt(0) when (bytes_left < UInt(blockBytes)) { bytes_left := UInt(0) } .otherwise { @@ -279,7 +378,9 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) } when (state === s_wait && get_done && !put_inflight) { - state := Mux(bytes_left === UInt(0), s_resp, s_get) + state := MuxCase(s_get, Seq( + (bytes_left === UInt(0)) -> s_resp, + streaming -> s_stream_read_resp)) } when (io.dma.resp.fire()) { state := s_idle } From d0a14c6de9f55e973afcb0ff71b3876a87d3ecee Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 11 Jan 2016 16:14:56 -0800 Subject: [PATCH 504/688] separate TileLink converter/wrapper/unwrapper/narrower into separate file --- uncore/src/main/scala/converters.scala | 730 +++++++++++++++++++++++++ uncore/src/main/scala/tilelink.scala | 725 ------------------------ 2 files changed, 730 insertions(+), 725 deletions(-) create mode 100644 uncore/src/main/scala/converters.scala diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala new file mode 100644 index 00000000..8806e5f2 --- /dev/null +++ b/uncore/src/main/scala/converters.scala @@ -0,0 +1,730 @@ +package uncore + +import Chisel._ +import junctions._ +import cde.Parameters + +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ +object TileLinkIOWrapper { + def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ +class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + } + + val g = io.grant.bits.payload + + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounterWithHeader(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits.fin := g.makeFinish() + q.io.enq.bits.dst := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.dst + io.finish.bits.payload := q.io.deq.bits.fin + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } +} + +class FinishQueueEntry(implicit p: Parameters) extends TLBundle()(p) { + val fin = new Finish + val dst = UInt(width = log2Up(p(LNEndpoints))) +} + +class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishQueueEntry()(p), entries) + +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probe Probes]]. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.network.release <> rel_with_header + io.client.probe <> prb_without_header + io.client.grant <> gnt_without_header +} + +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip + } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) +} + +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. + * + * Won't count message types that don't have data. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( + out: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(out.fire(), out.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param max max number of outstanding ups with no down + * @param up outgoing channel + * @param down incoming channel + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( + max: Int, + up: DecoupledIO[T], + down: DecoupledIO[S], + beat: UInt = UInt(0), + track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) + val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) + val do_inc = up_done && track(up.bits) + val do_dec = down_done + val cnt = TwoWayCounter(do_inc, do_dec, max) + (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + } +} + +class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientTileLinkIO().flip + val out = new ClientUncachedTileLinkIO + } + + def needsRoqEnq(channel: HasTileLinkData): Bool = + !channel.hasMultibeatData() || channel.addr_beat === UInt(0) + + def needsRoqDeq(channel: HasTileLinkData): Bool = + !channel.hasMultibeatData() || channel.addr_beat === UInt(tlDataBeats - 1) + + val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, + Some((acq: Acquire) => acq.hasMultibeatData()))) + + val acqRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val relRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val iacq = io.in.acquire.bits + val irel = io.in.release.bits + val ognt = io.out.grant.bits + + val acq_roq_enq = needsRoqEnq(iacq) + val rel_roq_enq = needsRoqEnq(irel) + + val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready + val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready + + val acq_helper = DecoupledHelper( + io.in.acquire.valid, + acq_roq_ready, + acqArb.io.in(0).ready) + + val rel_helper = DecoupledHelper( + io.in.release.valid, + rel_roq_ready, + acqArb.io.in(1).ready) + + acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) + acqRoq.io.enq.bits.data := iacq.isBuiltInType() + acqRoq.io.enq.bits.tag := iacq.client_xact_id + + acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) + acqArb.io.in(0).bits := Acquire( + is_builtin_type = Bool(true), + a_type = Mux(iacq.isBuiltInType(), + iacq.a_type, Acquire.getBlockType), + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = iacq.addr_beat, + data = iacq.data, + union = Mux(iacq.isBuiltInType(), + iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) + io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) + + relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) + relRoq.io.enq.bits.data := irel.isVoluntary() + relRoq.io.enq.bits.tag := irel.client_xact_id + + acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) + acqArb.io.in(1).bits := PutBlock( + client_xact_id = irel.client_xact_id, + addr_block = irel.addr_block, + addr_beat = irel.addr_beat, + data = irel.data, + wmask = Acquire.fullWriteMask) + io.in.release.ready := rel_helper.fire(io.in.release.valid) + + io.out.acquire <> acqArb.io.out + + acqRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + acqRoq.io.deq.tag := ognt.client_xact_id + + relRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + relRoq.io.deq.tag := ognt.client_xact_id + + val gnt_builtin = acqRoq.io.deq.data + val gnt_voluntary = relRoq.io.deq.data + + val acq_grant = Grant( + is_builtin_type = gnt_builtin, + g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + val rel_grant = Grant( + is_builtin_type = Bool(true), + g_type = Mux(gnt_voluntary, Grant.voluntaryAckType, ognt.g_type), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := io.out.grant.valid + io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) + io.out.grant.ready := io.in.grant.ready + + io.in.probe.valid := Bool(false) +} + +class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { + val addr_beat = UInt(width = tlBeatAddrBits) + val byteOff = UInt(width = tlByteAddrBits) + val subblock = Bool() +} + +class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val tl = new ClientUncachedTileLinkIO().flip + val nasti = new NastiIO + } + + private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( + MT_B -> UInt(0), + MT_BU -> UInt(0), + MT_H -> UInt(1), + MT_HU -> UInt(1), + MT_W -> UInt(2), + MT_D -> UInt(3), + MT_Q -> UInt(log2Up(tlDataBytes)))) + + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction + require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") + require(tlClientXactIdBits <= nastiXIdBits, "NastiIO converter is going truncate tags: " + tlClientXactIdBits + " > " + nastiXIdBits) + + val has_data = io.tl.acquire.bits.hasData() + + val is_subblock = io.tl.acquire.bits.isSubBlockType() + val is_multibeat = io.tl.acquire.bits.hasMultibeatData() + val (tl_cnt_out, tl_wrap_out) = Counter( + io.tl.acquire.fire() && is_multibeat, tlDataBeats) + + val get_valid = io.tl.acquire.valid && !has_data + val put_valid = io.tl.acquire.valid && has_data + + // Reorder queue saves extra information needed to send correct + // grant back to TL client + val roq = Module(new ReorderQueue( + new NastiIOTileLinkIOConverterInfo, + nastiRIdBits, tlMaxClientsPerPort)) + + // For Get/GetBlock, make sure Reorder queue can accept new entry + val get_helper = DecoupledHelper( + get_valid, + roq.io.enq.ready, + io.nasti.ar.ready) + + val w_inflight = Reg(init = Bool(false)) + + // For Put/PutBlock, make sure aw and w channel are both ready before + // we send the first beat + val aw_ready = w_inflight || io.nasti.aw.ready + val put_helper = DecoupledHelper( + put_valid, + aw_ready, + io.nasti.w.ready) + + val (nasti_cnt_out, nasti_wrap_out) = Counter( + io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) + + roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) + roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat + roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() + roq.io.enq.bits.data.subblock := is_subblock + roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) + roq.io.deq.tag := io.nasti.r.bits.id + + // Decompose outgoing TL Acquires into Nasti address and data channels + io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) + io.nasti.ar.bits := NastiReadAddressChannel( + id = io.tl.acquire.bits.client_xact_id, + addr = io.tl.acquire.bits.full_addr(), + size = Mux(is_subblock, + opSizeToXSize(io.tl.acquire.bits.op_size()), + UInt(log2Ceil(tlDataBytes))), + len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) + io.nasti.aw.bits := NastiWriteAddressChannel( + id = io.tl.acquire.bits.client_xact_id, + addr = io.tl.acquire.bits.full_addr(), + size = UInt(log2Ceil(tlDataBytes)), + len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) + + io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) + io.nasti.w.bits := NastiWriteDataChannel( + data = io.tl.acquire.bits.data, + strb = io.tl.acquire.bits.wmask(), + last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) + + io.tl.acquire.ready := Mux(has_data, + put_helper.fire(put_valid), + get_helper.fire(get_valid)) + + when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { + w_inflight := Bool(true) + } + + when (w_inflight) { + when (tl_wrap_out) { w_inflight := Bool(false) } + } + + // Aggregate incoming NASTI responses into TL Grants + val (tl_cnt_in, tl_wrap_in) = Counter( + io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + val r_aligned_data = Mux(roq.io.deq.data.subblock, + io.nasti.r.bits.data << Cat(roq.io.deq.data.byteOff, UInt(0, 3)), + io.nasti.r.bits.data) + + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + is_builtin_type = Bool(true), + g_type = Mux(roq.io.deq.data.subblock, + Grant.getDataBeatType, Grant.getDataBlockType), + client_xact_id = io.nasti.r.bits.id, + manager_xact_id = UInt(0), + addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), + data = r_aligned_data) + + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).bits := Grant( + is_builtin_type = Bool(true), + g_type = Grant.putAckType, + client_xact_id = io.nasti.b.bits.id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = Bits(0)) + + assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") + assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") +} + +class TileLinkIONarrower(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + + require(outerDataBeats >= innerDataBeats) + require(outerDataBeats % innerDataBeats == 0) + require(outerDataBits <= innerDataBits) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = outerDataBeats / innerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + if (factor > 1) { + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType + val smallput = iacq.a_type === Acquire.putType + val smallget = iacq.a_type === Acquire.getType + + val acq_data_buffer = Reg(UInt(width = innerDataBits)) + val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) + + // this part of the address shifts from the inner byte address + // to the outer beat address + val readshift = iacq.full_addr()(innerByteAddrBits - 1, outerByteAddrBits) + val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) + val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) + + val mask_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerWriteMaskBits + val msb = (i + 1) * outerWriteMaskBits - 1 + iacq.wmask()(msb, lsb) + } + + val data_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerDataBits + val msb = (i + 1) * outerDataBits - 1 + iacq.data(msb, lsb) + } + + val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) + + val smallput_data = Mux1H(beat_sel, data_chunks) + val smallput_wmask = Mux1H(beat_sel, mask_chunks) + val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) + + assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), + "Can't perform Put wider than outer width") + + val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( + MT_B -> Bool(true), + MT_BU -> Bool(true), + MT_H -> Bool(outerDataBits >= 16), + MT_HU -> Bool(outerDataBits >= 16), + MT_W -> Bool(outerDataBits >= 32), + MT_D -> Bool(outerDataBits >= 64), + MT_Q -> Bool(false))) + + assert(!io.in.acquire.valid || !smallget || read_size_ok, + "Can't perform Get wider than outer width") + + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate())(outerConfig) + + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + addr_byte = outer_byte_addr, + operand_size = iacq.op_size(), + alloc = iacq.allocate())(outerConfig) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = smallput_beat, + data = smallput_data, + wmask = Some(smallput_wmask))(outerConfig) + + val sending_put = Reg(init = Bool(false)) + + val pass_valid = io.in.acquire.valid && !stretch && !smallget + val smallget_valid = smallget && io.in.acquire.valid + + val smallget_roq = Module(new ReorderQueue( + readshift, outerIdBits, outerMaxClients)) + + val smallget_helper = DecoupledHelper( + smallget_valid, + smallget_roq.io.enq.ready, + io.out.acquire.ready) + + smallget_roq.io.enq.valid := smallget_helper.fire( + smallget_roq.io.enq.ready, !sending_put) + smallget_roq.io.enq.bits.data := readshift + smallget_roq.io.enq.bits.tag := iacq.client_xact_id + + io.out.acquire.bits := MuxBundle(iacq, Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire), + (smallput, put_acquire), + (smallget, get_acquire))) + io.out.acquire.valid := sending_put || pass_valid || + smallget_helper.fire(io.out.acquire.ready) + io.in.acquire.ready := !sending_put && (stretch || + (!smallget && io.out.acquire.ready) || + smallget_helper.fire(smallget_valid)) + + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) + } + + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(innerDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits)(innerConfig) + + val smallget_grant = ognt.g_type === Grant.getDataBeatType + val get_grant_shift = Cat(smallget_roq.io.deq.data, + UInt(0, outerByteAddrBits + 3)) + + smallget_roq.io.deq.valid := io.out.grant.fire() && smallget_grant + smallget_roq.io.deq.tag := ognt.client_xact_id + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), + data = ognt.data << get_grant_shift)(innerConfig) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + io.in.grant.bits := MuxBundle(ognt, Seq( + sending_get -> get_block_grant, + smallget_grant -> get_grant)) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) + } + } + + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } + } else { io.out <> io.in } +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 2e02962a..34756571 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -899,193 +899,6 @@ class ManagerTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val release = new DecoupledIO(new ReleaseFromSrc).flip } -/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ -object TileLinkIOWrapper { - def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { - val conv = Module(new ClientTileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl - def apply(tl: TileLinkIO): TileLinkIO = tl -} - -class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse - * to [[uncore.Grant]] that it receives from a manager and forwards to a client - */ -class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) - with HasDataBeatCounters { - val io = new Bundle { - val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip - val refill = Decoupled(new Grant) - val finish = Decoupled(new LogicalNetworkIO(new Finish)) - val ready = Bool(OUTPUT) - } - - val g = io.grant.bits.payload - - if(tlNetworkPreservesPointToPointOrdering) { - io.finish.valid := Bool(false) - io.refill.valid := io.grant.valid - io.refill.bits := g - io.grant.ready := io.refill.ready - io.ready := Bool(true) - } else { - // We only want to send Finishes after we have collected all beats of - // a multibeat Grant. But Grants from multiple managers or transactions may - // get interleaved, so we could need a counter for each. - val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounterWithHeader(io.grant) - } else { - val entries = 1 << tlClientXactIdBits - def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id - assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) - } - val q = Module(new FinishQueue(outstanding)) - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits.fin := g.makeFinish() - q.io.enq.bits.dst := io.grant.bits.header.src - - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.dst - io.finish.bits.payload := q.io.deq.bits.fin - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid - io.refill.bits := g - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready - io.ready := q.io.enq.ready - } -} - -class FinishQueueEntry(implicit p: Parameters) extends TLBundle()(p) { - val fin = new Finish - val dst = UInt(width = log2Up(p(LNEndpoints))) -} - -class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishQueueEntry()(p), entries) - -/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] - * - * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Probe Probes]]. - * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. - * - * @param clientId network port id of this agent - * @param addrConvert how a physical address maps to a destination manager port id - */ -class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val client = new ClientTileLinkIO().flip - val network = new TileLinkIO - } - - val finisher = Module(new FinishUnit(clientId)) - finisher.io.grant <> io.network.grant - io.network.finish <> finisher.io.finish - - val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) - val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) - val gnt_without_header = finisher.io.refill - - io.network.acquire.bits := acq_with_header.bits - io.network.acquire.valid := acq_with_header.valid && finisher.io.ready - acq_with_header.ready := io.network.acquire.ready && finisher.io.ready - io.network.release <> rel_with_header - io.client.probe <> prb_without_header - io.client.grant <> gnt_without_header -} - -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } -} - -/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip - * - * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], - * but supplies client_id instead. - * - * @param managerId the network port id of this agent - * @param idConvert how a sharer id maps to a destination client port id - */ -class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val manager = new ManagerTileLinkIO().flip - val network = new TileLinkIO().flip - } - io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) - io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) - io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src - io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) - io.manager.release.bits.client_id := io.network.release.bits.header.src - io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) - io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) -} - -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} - /** Struct for describing per-channel queue depths */ case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) @@ -1316,541 +1129,3 @@ class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) } - -/** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. - * - * Won't count message types that don't have data. - * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. - */ -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - - /** Returns the current count on this channel and when a message is done - * @param inc increment the counter (usually .valid or .fire()) - * @param data the actual channel data - * @param beat count to return for single-beat messages - */ - def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ - def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( - out: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = - connectDataBeatCounter(out.fire(), out.bits, beat) - - /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on - * incoming channels in case of network reordering. - */ - def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = - connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 - - /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ - def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - - /** If the network might interleave beats from different messages, we need a Vec of counters, - * one for every outstanding message id that might be interleaved. - * - * @param getId mapping from Message to counter id - */ - def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - /** Provides counters on two channels, as well a meta-counter that tracks how many - * messages have been sent over the up channel but not yet responded to over the down channel - * - * @param max max number of outstanding ups with no down - * @param up outgoing channel - * @param down incoming channel - * @param beat overrides cnts on single-beat messages - * @param track whether up's message should be tracked - * @return a tuple containing whether their are outstanding messages, up's count, - * up's done, down's count, down's done - */ - def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( - max: Int, - up: DecoupledIO[T], - down: DecoupledIO[S], - beat: UInt = UInt(0), - track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) - val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) - val do_inc = up_done && track(up.bits) - val do_dec = down_done - val cnt = TwoWayCounter(do_inc, do_dec, max) - (cnt > UInt(0), up_idx, up_done, down_idx, down_done) - } -} - -class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientTileLinkIO().flip - val out = new ClientUncachedTileLinkIO - } - - def needsRoqEnq(channel: HasTileLinkData): Bool = - !channel.hasMultibeatData() || channel.addr_beat === UInt(0) - - def needsRoqDeq(channel: HasTileLinkData): Bool = - !channel.hasMultibeatData() || channel.addr_beat === UInt(tlDataBeats - 1) - - val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, - Some((acq: Acquire) => acq.hasMultibeatData()))) - - val acqRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val relRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val iacq = io.in.acquire.bits - val irel = io.in.release.bits - val ognt = io.out.grant.bits - - val acq_roq_enq = needsRoqEnq(iacq) - val rel_roq_enq = needsRoqEnq(irel) - - val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready - val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready - - val acq_helper = DecoupledHelper( - io.in.acquire.valid, - acq_roq_ready, - acqArb.io.in(0).ready) - - val rel_helper = DecoupledHelper( - io.in.release.valid, - rel_roq_ready, - acqArb.io.in(1).ready) - - acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) - acqRoq.io.enq.bits.data := iacq.isBuiltInType() - acqRoq.io.enq.bits.tag := iacq.client_xact_id - - acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) - acqArb.io.in(0).bits := Acquire( - is_builtin_type = Bool(true), - a_type = Mux(iacq.isBuiltInType(), - iacq.a_type, Acquire.getBlockType), - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = iacq.addr_beat, - data = iacq.data, - union = Mux(iacq.isBuiltInType(), - iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) - io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) - - relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) - relRoq.io.enq.bits.data := irel.isVoluntary() - relRoq.io.enq.bits.tag := irel.client_xact_id - - acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) - acqArb.io.in(1).bits := PutBlock( - client_xact_id = irel.client_xact_id, - addr_block = irel.addr_block, - addr_beat = irel.addr_beat, - data = irel.data, - wmask = Acquire.fullWriteMask) - io.in.release.ready := rel_helper.fire(io.in.release.valid) - - io.out.acquire <> acqArb.io.out - - acqRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) - acqRoq.io.deq.tag := ognt.client_xact_id - - relRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) - relRoq.io.deq.tag := ognt.client_xact_id - - val gnt_builtin = acqRoq.io.deq.data - val gnt_voluntary = relRoq.io.deq.data - - val acq_grant = Grant( - is_builtin_type = gnt_builtin, - g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - val rel_grant = Grant( - is_builtin_type = Bool(true), - g_type = Mux(gnt_voluntary, Grant.voluntaryAckType, ognt.g_type), - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - io.in.grant.valid := io.out.grant.valid - io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) - io.out.grant.ready := io.in.grant.ready - - io.in.probe.valid := Bool(false) -} - -class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { - val addr_beat = UInt(width = tlBeatAddrBits) - val byteOff = UInt(width = tlByteAddrBits) - val subblock = Bool() -} - -class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) - with HasNastiParameters { - val io = new Bundle { - val tl = new ClientUncachedTileLinkIO().flip - val nasti = new NastiIO - } - - private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( - MT_B -> UInt(0), - MT_BU -> UInt(0), - MT_H -> UInt(1), - MT_HU -> UInt(1), - MT_W -> UInt(2), - MT_D -> UInt(3), - MT_Q -> UInt(log2Up(tlDataBytes)))) - - val dataBits = tlDataBits*tlDataBeats - require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction - require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(tlClientXactIdBits <= nastiXIdBits, "NastiIO converter is going truncate tags: " + tlClientXactIdBits + " > " + nastiXIdBits) - - val has_data = io.tl.acquire.bits.hasData() - - val is_subblock = io.tl.acquire.bits.isSubBlockType() - val is_multibeat = io.tl.acquire.bits.hasMultibeatData() - val (tl_cnt_out, tl_wrap_out) = Counter( - io.tl.acquire.fire() && is_multibeat, tlDataBeats) - - val get_valid = io.tl.acquire.valid && !has_data - val put_valid = io.tl.acquire.valid && has_data - - // Reorder queue saves extra information needed to send correct - // grant back to TL client - val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, - nastiRIdBits, tlMaxClientsPerPort)) - - // For Get/GetBlock, make sure Reorder queue can accept new entry - val get_helper = DecoupledHelper( - get_valid, - roq.io.enq.ready, - io.nasti.ar.ready) - - val w_inflight = Reg(init = Bool(false)) - - // For Put/PutBlock, make sure aw and w channel are both ready before - // we send the first beat - val aw_ready = w_inflight || io.nasti.aw.ready - val put_helper = DecoupledHelper( - put_valid, - aw_ready, - io.nasti.w.ready) - - val (nasti_cnt_out, nasti_wrap_out) = Counter( - io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - - roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) - roq.io.enq.bits.tag := io.nasti.ar.bits.id - roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat - roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() - roq.io.enq.bits.data.subblock := is_subblock - roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) - roq.io.deq.tag := io.nasti.r.bits.id - - // Decompose outgoing TL Acquires into Nasti address and data channels - io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) - io.nasti.ar.bits := NastiReadAddressChannel( - id = io.tl.acquire.bits.client_xact_id, - addr = io.tl.acquire.bits.full_addr(), - size = Mux(is_subblock, - opSizeToXSize(io.tl.acquire.bits.op_size()), - UInt(log2Ceil(tlDataBytes))), - len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) - - io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) - io.nasti.aw.bits := NastiWriteAddressChannel( - id = io.tl.acquire.bits.client_xact_id, - addr = io.tl.acquire.bits.full_addr(), - size = UInt(log2Ceil(tlDataBytes)), - len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) - - io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) - io.nasti.w.bits := NastiWriteDataChannel( - data = io.tl.acquire.bits.data, - strb = io.tl.acquire.bits.wmask(), - last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) - - io.tl.acquire.ready := Mux(has_data, - put_helper.fire(put_valid), - get_helper.fire(get_valid)) - - when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { - w_inflight := Bool(true) - } - - when (w_inflight) { - when (tl_wrap_out) { w_inflight := Bool(false) } - } - - // Aggregate incoming NASTI responses into TL Grants - val (tl_cnt_in, tl_wrap_in) = Counter( - io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - val r_aligned_data = Mux(roq.io.deq.data.subblock, - io.nasti.r.bits.data << Cat(roq.io.deq.data.byteOff, UInt(0, 3)), - io.nasti.r.bits.data) - - gnt_arb.io.in(0).valid := io.nasti.r.valid - io.nasti.r.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - is_builtin_type = Bool(true), - g_type = Mux(roq.io.deq.data.subblock, - Grant.getDataBeatType, Grant.getDataBlockType), - client_xact_id = io.nasti.r.bits.id, - manager_xact_id = UInt(0), - addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = r_aligned_data) - - gnt_arb.io.in(1).valid := io.nasti.b.valid - io.nasti.b.ready := gnt_arb.io.in(1).ready - gnt_arb.io.in(1).bits := Grant( - is_builtin_type = Bool(true), - g_type = Grant.putAckType, - client_xact_id = io.nasti.b.bits.id, - manager_xact_id = UInt(0), - addr_beat = UInt(0), - data = Bits(0)) - - assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") - assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") -} - -class TileLinkIONarrower(innerTLId: String, outerTLId: String) - (implicit p: Parameters) extends TLModule()(p) { - - val innerParams = p(TLKey(innerTLId)) - val outerParams = p(TLKey(outerTLId)) - val innerDataBeats = innerParams.dataBeats - val innerDataBits = innerParams.dataBitsPerBeat - val innerWriteMaskBits = innerParams.writeMaskBits - val innerByteAddrBits = log2Up(innerWriteMaskBits) - val outerDataBeats = outerParams.dataBeats - val outerDataBits = outerParams.dataBitsPerBeat - val outerWriteMaskBits = outerParams.writeMaskBits - val outerByteAddrBits = log2Up(outerWriteMaskBits) - val outerBeatAddrBits = log2Up(outerDataBeats) - val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits - val outerMaxClients = outerParams.maxClientsPerPort - val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - - require(outerDataBeats >= innerDataBeats) - require(outerDataBeats % innerDataBeats == 0) - require(outerDataBits <= innerDataBits) - require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) - - val factor = outerDataBeats / innerDataBeats - - val io = new Bundle { - val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip - val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) - } - - if (factor > 1) { - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits - - val stretch = iacq.a_type === Acquire.putBlockType - val shrink = iacq.a_type === Acquire.getBlockType - val smallput = iacq.a_type === Acquire.putType - val smallget = iacq.a_type === Acquire.getType - - val acq_data_buffer = Reg(UInt(width = innerDataBits)) - val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) - val acq_client_id = Reg(iacq.client_xact_id) - val acq_addr_block = Reg(iacq.addr_block) - val acq_addr_beat = Reg(iacq.addr_beat) - val oacq_ctr = Counter(factor) - - // this part of the address shifts from the inner byte address - // to the outer beat address - val readshift = iacq.full_addr()(innerByteAddrBits - 1, outerByteAddrBits) - val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) - val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) - - val mask_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerWriteMaskBits - val msb = (i + 1) * outerWriteMaskBits - 1 - iacq.wmask()(msb, lsb) - } - - val data_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerDataBits - val msb = (i + 1) * outerDataBits - 1 - iacq.data(msb, lsb) - } - - val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) - - val smallput_data = Mux1H(beat_sel, data_chunks) - val smallput_wmask = Mux1H(beat_sel, mask_chunks) - val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) - - assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), - "Can't perform Put wider than outer width") - - val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( - MT_B -> Bool(true), - MT_BU -> Bool(true), - MT_H -> Bool(outerDataBits >= 16), - MT_HU -> Bool(outerDataBits >= 16), - MT_W -> Bool(outerDataBits >= 32), - MT_D -> Bool(outerDataBits >= 64), - MT_Q -> Bool(false))) - - assert(!io.in.acquire.valid || !smallget || read_size_ok, - "Can't perform Get wider than outer width") - - val outerConfig = p.alterPartial({ case TLId => outerTLId }) - val innerConfig = p.alterPartial({ case TLId => innerTLId }) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - alloc = iacq.allocate())(outerConfig) - - val put_block_acquire = PutBlock( - client_xact_id = acq_client_id, - addr_block = acq_addr_block, - addr_beat = if (factor > 1) - Cat(acq_addr_beat, oacq_ctr.value) - else acq_addr_beat, - data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = outer_beat_addr, - addr_byte = outer_byte_addr, - operand_size = iacq.op_size(), - alloc = iacq.allocate())(outerConfig) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = smallput_beat, - data = smallput_data, - wmask = Some(smallput_wmask))(outerConfig) - - val sending_put = Reg(init = Bool(false)) - - val pass_valid = io.in.acquire.valid && !stretch && !smallget - val smallget_valid = smallget && io.in.acquire.valid - - val smallget_roq = Module(new ReorderQueue( - readshift, outerIdBits, outerMaxClients)) - - val smallget_helper = DecoupledHelper( - smallget_valid, - smallget_roq.io.enq.ready, - io.out.acquire.ready) - - smallget_roq.io.enq.valid := smallget_helper.fire( - smallget_roq.io.enq.ready, !sending_put) - smallget_roq.io.enq.bits.data := readshift - smallget_roq.io.enq.bits.tag := iacq.client_xact_id - - io.out.acquire.bits := MuxBundle(iacq, Seq( - (sending_put, put_block_acquire), - (shrink, get_block_acquire), - (smallput, put_acquire), - (smallget, get_acquire))) - io.out.acquire.valid := sending_put || pass_valid || - smallget_helper.fire(io.out.acquire.ready) - io.in.acquire.ready := !sending_put && (stretch || - (!smallget && io.out.acquire.ready) || - smallget_helper.fire(smallget_valid)) - - when (io.in.acquire.fire() && stretch) { - acq_data_buffer := iacq.data - acq_wmask_buffer := iacq.wmask() - acq_client_id := iacq.client_xact_id - acq_addr_block := iacq.addr_block - acq_addr_beat := iacq.addr_beat - sending_put := Bool(true) - } - - when (sending_put && io.out.acquire.ready) { - acq_data_buffer := acq_data_buffer >> outerDataBits - acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits - when (oacq_ctr.inc()) { sending_put := Bool(false) } - } - - val ognt_block = ognt.hasMultibeatData() - val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) - val gnt_client_id = Reg(ognt.client_xact_id) - val gnt_manager_id = Reg(ognt.manager_xact_id) - - val ignt_ctr = Counter(innerDataBeats) - val ognt_ctr = Counter(factor) - val sending_get = Reg(init = Bool(false)) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits)(innerConfig) - - val smallget_grant = ognt.g_type === Grant.getDataBeatType - val get_grant_shift = Cat(smallget_roq.io.deq.data, - UInt(0, outerByteAddrBits + 3)) - - smallget_roq.io.deq.valid := io.out.grant.fire() && smallget_grant - smallget_roq.io.deq.tag := ognt.client_xact_id - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), - data = ognt.data << get_grant_shift)(innerConfig) - - io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) - io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - io.in.grant.bits := MuxBundle(ognt, Seq( - sending_get -> get_block_grant, - smallget_grant -> get_grant)) - - when (io.out.grant.valid && ognt_block && !sending_get) { - gnt_data_buffer(ognt_ctr.value) := ognt.data - when (ognt_ctr.inc()) { - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - sending_get := Bool(true) - } - } - - when (io.in.grant.ready && sending_get) { - ignt_ctr.inc() - sending_get := Bool(false) - } - } else { io.out <> io.in } -} From c81745eb8e7e8913bdd2b0cf34ae8ad765601b62 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 11 Jan 2016 16:18:44 -0800 Subject: [PATCH 505/688] lowercase SMI to Smi --- uncore/src/main/scala/htif.scala | 4 ++-- uncore/src/main/scala/scr.scala | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index d5b1df3d..f392fffd 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -40,7 +40,7 @@ class HostIO(w: Int) extends Bundle { class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val csr = new SMIIO(scrDataBits, 12).flip + val csr = new SmiIO(scrDataBits, 12).flip val debug_stats_csr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work @@ -51,7 +51,7 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val host = new HostIO(w) val cpu = Vec(new HtifIO, nCores).flip val mem = new ClientUncachedTileLinkIO - val scr = new SMIIO(scrDataBits, scrAddrBits) + val scr = new SmiIO(scrDataBits, scrAddrBits) } io.host.debug_stats_csr := io.cpu.map(_.debug_stats_csr).reduce(_||_) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 5cf099a2..b716793d 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -import junctions.{SMIIO, MMIOBase} +import junctions.{SmiIO, MMIOBase} import cde.Parameters class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { @@ -13,7 +13,7 @@ class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { class SCRFile(implicit p: Parameters) extends HtifModule()(p) { val io = new Bundle { - val smi = new SMIIO(scrDataBits, scrAddrBits).flip + val smi = new SmiIO(scrDataBits, scrAddrBits).flip val scr = new SCRIO } From a953ff384aab119ea4fb84931cf5ae283f40f521 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 12 Jan 2016 15:30:26 -0800 Subject: [PATCH 506/688] Chisel3 compatibility: use more concrete types --- uncore/src/main/scala/cache.scala | 2 +- uncore/src/main/scala/util.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 30624309..89176769 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -80,7 +80,7 @@ class PseudoLRU(n: Int) def access(way: UInt) { state_reg := get_next_state(state_reg,way) } - def get_next_state(state: Bits, way: UInt) = { + def get_next_state(state: UInt, way: UInt) = { var next_state = state var idx = UInt(1,1) for (i <- log2Up(n)-1 to 0 by -1) { diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index db1dd6f4..cc5036bf 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -18,7 +18,7 @@ object MuxBundle { mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) } - def apply[S <: Data, T <: Data] (key: S, default: T, mapping: Seq[(S, T)]): T = { + def apply[T <: Data] (key: UInt, default: T, mapping: Seq[(UInt, T)]): T = { apply(default, mapping.map{ case (a, b) => (a === key, b) }) } } From 0b90b8fe5fe9c05770da21de2b14c58a388afc19 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 12 Jan 2016 15:32:29 -0800 Subject: [PATCH 507/688] Avoid zero-width wire case :-/ --- uncore/src/main/scala/tilelink.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 34756571..41d3d5ee 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -169,8 +169,11 @@ trait HasAcquireUnion extends HasTileLinkParameters { def amo_shift_bytes(dummy: Int = 0) = UInt(amoAluOperandBytes)*amo_offset() /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0): UInt = { + val amo_word_mask = + if (amoAluOperandBytes == tlWriteMaskBits) UInt(1) + else UIntToOH(amo_offset()) Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBits/8, UIntToOH(amo_offset())), + FillInterleaved(amoAluOperandBytes, amo_word_mask), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) From 4ff1aea288c008928cff650e118e4e70e7752288 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 14 Jan 2016 13:47:47 -0800 Subject: [PATCH 508/688] fix more Chisel3 deprecations --- uncore/src/main/scala/broadcast.scala | 18 +++++++++--------- uncore/src/main/scala/cache.scala | 18 +++++++++--------- uncore/src/main/scala/coherence.scala | 18 +++++++++--------- uncore/src/main/scala/htif.scala | 10 +++++----- uncore/src/main/scala/metadata.scala | 6 +++--- uncore/src/main/scala/network.scala | 4 ++-- uncore/src/main/scala/scr.scala | 4 ++-- uncore/src/main/scala/tilelink.scala | 8 ++++---- uncore/src/main/scala/uncore.scala | 2 +- uncore/src/main/scala/util.scala | 6 +++--- 10 files changed, 47 insertions(+), 47 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index fbc52a11..4b6627aa 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -48,7 +48,7 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) trackerList.map(_.io.incoherent := io.incoherent) // Queue to store impending Put data - val sdq = Reg(Vec(io.iacq().data, sdqDepth)) + val sdq = Reg(Vec(sdqDepth, io.iacq().data)) val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR @@ -77,7 +77,7 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) val voluntary = io.irel().isVoluntary() val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Reg(Vec(io.irel().data, innerDataBeats)) //TODO Assumes nReleaseTransactors == 1 + val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1 when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data @@ -218,7 +218,7 @@ class BroadcastAcquireTracker(trackerId: Int) val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val coh = ManagerMetadata.onReset - assert(!(state != s_idle && xact.isBuiltInType() && + assert(!(state =/= s_idle && xact.isBuiltInType() && Vec(Acquire.putAtomicType, Acquire.getPrefetchType, Acquire.putPrefetchType).contains(xact.a_type)), "Broadcast Hub does not support PutAtomics or prefetches") // TODO @@ -243,7 +243,7 @@ class BroadcastAcquireTracker(trackerId: Int) val subblock_type = xact.isSubBlockType() io.has_acquire_conflict := xact.conflicts(io.iacq()) && - (state != s_idle) && + (state =/= s_idle) && !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data @@ -302,16 +302,16 @@ class BroadcastAcquireTracker(trackerId: Int) io.inner.release.ready := Bool(false) io.inner.finish.ready := Bool(false) - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_id != xact.client_id), + assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_id =/= xact.client_id), "AcquireTracker accepted data beat from different network source than initial request.") - assert(!(state != s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id != xact.client_xact_id), + assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && + io.iacq().client_xact_id =/= xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") assert(!(state === s_idle && io.inner.acquire.fire() && - io.iacq().hasMultibeatData() && io.iacq().addr_beat != UInt(0)), + io.iacq().hasMultibeatData() && io.iacq().addr_beat =/= UInt(0)), "AcquireTracker initialized with a tail data beat.") when(collect_iacq_data) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 89176769..3192cb66 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -102,7 +102,7 @@ class PseudoLRU(n: Int) } class SeqPLRU(n_sets: Int, n_ways: Int) extends SeqReplacementPolicy { - val state = SeqMem(Bits(width = n_ways-1), n_sets) + val state = SeqMem(n_sets, Bits(width = n_ways-1)) val logic = new PseudoLRU(n_ways) val current_state = Wire(Bits()) val plru_way = logic.get_replace_way(current_state) @@ -141,7 +141,7 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext val io = new Bundle { val read = Decoupled(new MetaReadReq).flip val write = Decoupled(new MetaWriteReq(rstVal)).flip - val resp = Vec(rstVal.cloneType, nWays).asOutput + val resp = Vec(nWays, rstVal.cloneType).asOutput } val rst_cnt = Reg(init=UInt(0, log2Up(nSets+1))) val rst = rst_cnt < UInt(nSets) @@ -151,7 +151,7 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth - val tag_arr = SeqMem(Vec(UInt(width = metabits), nWays), nSets) + val tag_arr = SeqMem(nSets, Vec(nWays, UInt(width = metabits))) when (rst || io.write.valid) { tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) } @@ -333,7 +333,7 @@ class L2DataRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) class L2DataArray(delay: Int)(implicit p: Parameters) extends L2HellaCacheModule()(p) { val io = new L2DataRWIO().flip - val array = SeqMem(Vec(Bits(width=8), rowBits/8), nWays*nSets*refillCycles) + val array = SeqMem(nWays*nSets*refillCycles, Vec(rowBits/8, Bits(width=8))) val ren = !io.write.valid && io.read.valid val raddr = Cat(OHToUInt(io.read.bits.way_en), io.read.bits.addr_idx, io.read.bits.addr_beat) val waddr = Cat(OHToUInt(io.write.bits.way_en), io.write.bits.addr_idx, io.write.bits.addr_beat) @@ -449,9 +449,9 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) class CacheBlockBuffer { // TODO val buffer = Reg(Bits(width = p(CacheBlockBytes)*8)) - def internal = Vec(Bits(width = rowBits), internalDataBeats).fromBits(buffer) - def inner = Vec(Bits(width = innerDataBits), innerDataBeats).fromBits(buffer) - def outer = Vec(Bits(width = outerDataBits), outerDataBeats).fromBits(buffer) + def internal = Vec(internalDataBeats, Bits(width = rowBits)).fromBits(buffer) + def inner = Vec(innerDataBeats, Bits(width = innerDataBits)).fromBits(buffer) + def outer = Vec(outerDataBeats, Bits(width = outerDataBits)).fromBits(buffer) } def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { @@ -684,7 +684,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Utility function for updating the metadata that will be kept in this cache def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { - when(flag && pending_coh != next) { + when(flag && pending_coh =/= next) { pending_meta_write := Bool(true) pending_coh := next } @@ -982,7 +982,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra coh.inner.requiresProbesOnVoluntaryWriteback()) val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) val should_update_meta = !tag_match && xact_allocate || - is_hit && pending_coh_on_hit != coh + is_hit && pending_coh_on_hit =/= coh // Determine any changes to the coherence metadata when (should_update_meta) { pending_meta_write := Bool(true) } pending_coh := Mux(is_hit, pending_coh_on_hit, Mux(tag_match, coh, pending_coh_on_miss)) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index 2ae5653d..f7eb5992 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -139,7 +139,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientStatesWithWritePermission = Vec(clientValid) def clientStatesWithDirtyData = Vec(clientValid) - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive @@ -230,7 +230,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) def clientStatesWithDirtyData = Vec(clientExclusiveDirty) - def isValid (meta: ClientMetadata) = meta.state != clientInvalid + def isValid (meta: ClientMetadata) = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) @@ -332,7 +332,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientStatesWithWritePermission = Vec(clientExclusiveDirty) def clientStatesWithDirtyData = Vec(clientExclusiveDirty) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) @@ -385,7 +385,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -450,7 +450,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) def clientStatesWithDirtyData = Vec(clientExclusiveDirty) - def isValid(meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) @@ -505,7 +505,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) @@ -566,7 +566,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) def clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) - def isValid (meta: ClientMetadata): Bool = meta.state != clientInvalid + def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = Mux(isWriteIntent(cmd), @@ -594,7 +594,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d releaseInvalidateAckMigratory, releaseInvalidateAck), probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(meta.state != clientInvalid, + probeDowngrade -> Mux(meta.state =/= clientInvalid, releaseDowngradeAckHasCopy, releaseDowngradeAck), probeCopy -> releaseCopyAck)) Mux(dirty, with_data, without_data) @@ -646,7 +646,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def requiresProbes(a: HasAcquireType, meta: ManagerMetadata) = Mux(dir.none(meta.sharers), Bool(false), Mux(dir.one(meta.sharers), Bool(true), //TODO: for now we assume it's Exclusive - Mux(a.isBuiltInType(), a.hasData(), a.a_type != acquireShared))) + Mux(a.isBuiltInType(), a.hasData(), a.a_type =/= acquireShared))) def requiresProbes(cmd: UInt, meta: ManagerMetadata) = !dir.none(meta.sharers) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index f392fffd..9b72459b 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -49,7 +49,7 @@ class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { val io = new Bundle { val host = new HostIO(w) - val cpu = Vec(new HtifIO, nCores).flip + val cpu = Vec(nCores, new HtifIO).flip val mem = new ClientUncachedTileLinkIO val scr = new SmiIO(scrDataBits, scrAddrBits) } @@ -99,7 +99,7 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, - Mux(cmd === cmd_readcr || cmd === cmd_writecr, size != UInt(1), + Mux(cmd === cmd_readcr || cmd === cmd_writecr, size =/= UInt(1), Bool(true))) val tx_count = Reg(init=UInt(0, rx_count_w)) @@ -110,7 +110,7 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt tx_count := tx_count + UInt(1) } - val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd != cmd_writemem && next_cmd != cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) + val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd =/= cmd_writemem && next_cmd =/= cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) @@ -147,7 +147,7 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt rx_count := UInt(0) tx_count := UInt(0) } - state := Mux(cmd === cmd_readmem && pos != UInt(0), state_mem_rreq, state_rx) + state := Mux(cmd === cmd_readmem && pos =/= UInt(0), state_mem_rreq, state_rx) } val n = dataBits/short_request_bits @@ -177,7 +177,7 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val cpu = io.cpu(i) val me = csr_coreid === UInt(i) - cpu.csr.req.valid := state === state_csr_req && me && csr_addr != UInt(csr_RESET) + cpu.csr.req.valid := state === state_csr_req && me && csr_addr =/= UInt(csr_RESET) cpu.csr.req.bits.rw := cmd === cmd_writecr cpu.csr.req.bits.addr := csr_addr cpu.csr.req.bits.data := csr_wdata diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 8efad8bf..2819745c 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -20,7 +20,7 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { /** Metadata equality */ def ===(rhs: ClientMetadata): Bool = this.state === rhs.state - def !=(rhs: ClientMetadata): Bool = !this.===(rhs) + def =/=(rhs: ClientMetadata): Bool = !this.===(rhs) /** Is the block's data present in this cache */ def isValid(dummy: Int = 0): Bool = co.isValid(this) @@ -168,7 +168,7 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { /** Metadata equality */ def ===(rhs: ManagerMetadata): Bool = //this.state === rhs.state && TODO: Fix 0-width wires in Chisel this.sharers === rhs.sharers - def !=(rhs: ManagerMetadata): Bool = !this.===(rhs) + def =/=(rhs: ManagerMetadata): Bool = !this.===(rhs) /** Converts the directory info into an N-hot sharer bitvector (i.e. full representation) */ def full(dummy: Int = 0): UInt = co.dir.full(this.sharers) @@ -319,7 +319,7 @@ class HierarchicalMetadata(implicit p: Parameters) extends CoherenceMetadata()(p val outer: ClientMetadata = new ClientMetadata()(p.alterPartial({case TLId => p(OuterTLId)})) def ===(rhs: HierarchicalMetadata): Bool = this.inner === rhs.inner && this.outer === rhs.outer - def !=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) + def =/=(rhs: HierarchicalMetadata): Bool = !this.===(rhs) } /** Factories for HierarchicalMetadata, including on reset */ diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index eee9aa6e..43b379c2 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -19,8 +19,8 @@ class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { } class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { - val in = Vec(Decoupled(new PhysicalNetworkIO(n,dType)), n).flip - val out = Vec(Decoupled(new PhysicalNetworkIO(n,dType)), n) + val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip + val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) } abstract class PhysicalNetwork extends Module diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index b716793d..ddc35ded 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -5,7 +5,7 @@ import junctions.{SmiIO, MMIOBase} import cde.Parameters class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { - val rdata = Vec(Bits(INPUT, scrDataBits), nSCR) + val rdata = Vec(nSCR, Bits(INPUT, scrDataBits)) val wen = Bool(OUTPUT) val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, scrDataBits) @@ -17,7 +17,7 @@ class SCRFile(implicit p: Parameters) extends HtifModule()(p) { val scr = new SCRIO } - val scr_rdata = Wire(Vec(Bits(width=scrDataBits), io.scr.rdata.size)) + val scr_rdata = Wire(Vec(io.scr.rdata.size, Bits(width=scrDataBits))) for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) scr_rdata(0) := UInt(nCores) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 41d3d5ee..f5ac25de 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1058,7 +1058,7 @@ trait TileLinkArbiterLike extends HasTileLinkParameters { abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec(new UncachedTileLinkIO, arbN).flip + val in = Vec(arbN, new UncachedTileLinkIO).flip val out = new UncachedTileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -1070,7 +1070,7 @@ abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Paramete abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike { val io = new Bundle { - val in = Vec(new TileLinkIO, arbN).flip + val in = Vec(arbN, new TileLinkIO).flip val out = new TileLinkIO } hookupClientSource(io.in.map(_.acquire), io.out.acquire) @@ -1114,7 +1114,7 @@ class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends /** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { - val in = Vec(new ClientUncachedTileLinkIO, arbN).flip + val in = Vec(arbN, new ClientUncachedTileLinkIO).flip val out = new ClientUncachedTileLinkIO } hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) @@ -1124,7 +1124,7 @@ class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) /** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { val io = new Bundle { - val in = Vec(new ClientTileLinkIO, arbN).flip + val in = Vec(arbN, new ClientTileLinkIO).flip val out = new ClientTileLinkIO } hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 04a38c71..3d38f6b2 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -62,7 +62,7 @@ trait HasCoherenceAgentWiringHelpers { trait HasInnerTLIO extends HasCoherenceAgentParameters { val inner = new ManagerTileLinkIO()(p.alterPartial({case TLId => p(InnerTLId)})) - val incoherent = Vec(Bool(), inner.tlNCachingClients).asInput + val incoherent = Vec(inner.tlNCachingClients, Bool()).asInput def iacq(dummy: Int = 0) = inner.acquire.bits def iprb(dummy: Int = 0) = inner.probe.bits def irel(dummy: Int = 0) = inner.release.bits diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index cc5036bf..c0727c64 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -75,7 +75,7 @@ class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) ex val rbits = Reg{io.in.bits} val active = Reg(init=Bool(false)) - val shifter = Vec(Bits(width = narrowWidth), n) + val shifter = Vec(n, Bits(width = narrowWidth)) (0 until n).foreach { i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } @@ -138,8 +138,8 @@ class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) } } - val roq_data = Reg(Vec(dType.cloneType, size)) - val roq_tags = Reg(Vec(UInt(width = tagWidth), size)) + val roq_data = Reg(Vec(size, dType.cloneType)) + val roq_tags = Reg(Vec(size, UInt(width = tagWidth))) val roq_free = Reg(init = Vec.fill(size)(Bool(true))) val roq_enq_addr = PriorityEncoder(roq_free) From 2946bc928ed8171dbd2346583776d6455cde3762 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 16 Jan 2016 19:01:24 -0800 Subject: [PATCH 509/688] Avoid muxing between bundles of different size --- uncore/src/main/scala/converters.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 8806e5f2..22e9532a 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -649,7 +649,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) smallget_roq.io.enq.bits.data := readshift smallget_roq.io.enq.bits.tag := iacq.client_xact_id - io.out.acquire.bits := MuxBundle(iacq, Seq( + io.out.acquire.bits := MuxBundle(Wire(io.out.acquire.bits, init=iacq), Seq( (sending_put, put_block_acquire), (shrink, get_block_acquire), (smallput, put_acquire), @@ -709,7 +709,8 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - io.in.grant.bits := MuxBundle(ognt, Seq( + + io.in.grant.bits := MuxBundle(Wire(io.in.grant.bits, init=ognt), Seq( sending_get -> get_block_grant, smallget_grant -> get_grant)) From 0dc8cd5b111e2084669a134c4a3ebaa12785c71f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Jan 2016 15:36:22 -0800 Subject: [PATCH 510/688] move ReorderQueue and DecoupledHelper to junctions --- uncore/src/main/scala/util.scala | 54 -------------------------------- 1 file changed, 54 deletions(-) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index c0727c64..121ea68f 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -116,58 +116,4 @@ object FlowThroughSerializer { in.ready := fs.io.in.ready fs.io.out } -} - -class ReorderQueueWrite[T <: Data](dType: T, tagWidth: Int) extends Bundle { - val data = dType.cloneType - val tag = UInt(width = tagWidth) - - override def cloneType = - new ReorderQueueWrite(dType, tagWidth).asInstanceOf[this.type] -} - -class ReorderQueue[T <: Data](dType: T, tagWidth: Int, size: Int) - extends Module { - val io = new Bundle { - val enq = Decoupled(new ReorderQueueWrite(dType, tagWidth)).flip - val deq = new Bundle { - val valid = Bool(INPUT) - val tag = UInt(INPUT, tagWidth) - val data = dType.cloneType.asOutput - val matches = Bool(OUTPUT) - } - } - - val roq_data = Reg(Vec(size, dType.cloneType)) - val roq_tags = Reg(Vec(size, UInt(width = tagWidth))) - val roq_free = Reg(init = Vec.fill(size)(Bool(true))) - - val roq_enq_addr = PriorityEncoder(roq_free) - val roq_matches = roq_tags.zip(roq_free) - .map { case (tag, free) => tag === io.deq.tag && !free } - val roq_deq_addr = PriorityEncoder(roq_matches) - - io.enq.ready := roq_free.reduce(_ || _) - io.deq.data := roq_data(roq_deq_addr) - io.deq.matches := roq_matches.reduce(_ || _) - - when (io.enq.valid && io.enq.ready) { - roq_data(roq_enq_addr) := io.enq.bits.data - roq_tags(roq_enq_addr) := io.enq.bits.tag - roq_free(roq_enq_addr) := Bool(false) - } - - when (io.deq.valid) { - roq_free(roq_deq_addr) := Bool(true) - } -} - -object DecoupledHelper { - def apply(rvs: Bool*) = new DecoupledHelper(rvs) -} - -class DecoupledHelper(val rvs: Seq[Bool]) { - def fire(exclude: Bool, includes: Bool*) = { - (rvs.filter(_ ne exclude) ++ includes).reduce(_ && _) - } } From ba94010928d37ab3ab8ca46b3f4560f926a6e9c3 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 14 Jan 2016 11:37:35 -0800 Subject: [PATCH 511/688] DMA requests should go through MMIO --- uncore/src/main/scala/dma.scala | 269 +++++++++++++++++++++++--------- 1 file changed, 191 insertions(+), 78 deletions(-) diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala index 46794ad2..69a3fb26 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/dma.scala @@ -6,25 +6,29 @@ import junctions._ import junctions.NastiConstants._ case object NDmaTransactors extends Field[Int] +case object NDmaXacts extends Field[Int] case object NDmaClients extends Field[Int] -case object NDmaXactsPerClient extends Field[Int] trait HasDmaParameters { implicit val p: Parameters val nDmaTransactors = p(NDmaTransactors) + val nDmaXacts = p(NDmaXacts) val nDmaClients = p(NDmaClients) - val nDmaXactsPerClient = p(NDmaXactsPerClient) - val dmaClientXactIdBits = log2Up(nDmaClients * nDmaXactsPerClient) + val dmaXactIdBits = log2Up(nDmaXacts) + val dmaClientIdBits = log2Up(nDmaClients) val addrBits = p(PAddrBits) val dmaStatusBits = 2 val dmaWordSizeBits = 2 + val csrDataBits = 64 + val csrDataBytes = csrDataBits / 8 } abstract class DmaModule(implicit val p: Parameters) extends Module with HasDmaParameters abstract class DmaBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) with HasDmaParameters class DmaRequest(implicit p: Parameters) extends DmaBundle()(p) { - val client_xact_id = UInt(width = dmaClientXactIdBits) + val xact_id = UInt(width = dmaXactIdBits) + val client_id = UInt(width = dmaClientIdBits) val cmd = UInt(width = DmaRequest.DMA_CMD_SZ) val source = UInt(width = addrBits) val dest = UInt(width = addrBits) @@ -33,7 +37,8 @@ class DmaRequest(implicit p: Parameters) extends DmaBundle()(p) { } class DmaResponse(implicit p: Parameters) extends DmaBundle()(p) { - val client_xact_id = UInt(width = dmaClientXactIdBits) + val xact_id = UInt(width = dmaXactIdBits) + val client_id = UInt(width = dmaClientIdBits) val status = UInt(width = dmaStatusBits) } @@ -46,14 +51,16 @@ object DmaRequest { val DMA_CMD_SIN = UInt("b100") val DMA_CMD_SOUT = UInt("b101") - def apply(client_xact_id: UInt = UInt(0), + def apply(xact_id: UInt = UInt(0), + client_id: UInt, cmd: UInt, source: UInt, dest: UInt, length: UInt, size: UInt = UInt(0))(implicit p: Parameters): DmaRequest = { val req = Wire(new DmaRequest) - req.client_xact_id := client_xact_id + req.xact_id := xact_id + req.client_id := client_id req.cmd := cmd req.source := source req.dest := dest @@ -71,25 +78,158 @@ class DmaIO(implicit p: Parameters) extends DmaBundle()(p) { class DmaTrackerIO(implicit p: Parameters) extends DmaBundle()(p) { val dma = (new DmaIO).flip - val inner = new ClientUncachedTileLinkIO - val outer = new NastiIO + val mem = new ClientUncachedTileLinkIO + val mmio = new NastiIO } -class DmaEngine(implicit p: Parameters) extends DmaModule()(p) { - val io = new DmaTrackerIO +class DmaManager(outstandingCSR: Int)(implicit p: Parameters) extends DmaModule()(p) + with HasNastiParameters with HasAddrMapParameters { + val io = new Bundle { + val ctrl = (new NastiIO).flip + val mmio = new NastiIO + val dma = new DmaIO + } + + private val wordBits = 1 << log2Up(addrBits) + private val wordBytes = wordBits / 8 + private val wordOff = log2Up(wordBytes) + private val wordMSB = wordOff + 2 + + val s_idle :: s_wdata :: s_dma_req :: s_wresp :: Nil = Enum(Bits(), 4) + val state = Reg(init = s_idle) + + val nCtrlWords = (addrBits * 4) / nastiXDataBits + val ctrl_regs = Reg(Vec(nCtrlWords, UInt(width = nastiXDataBits))) + val ctrl_idx = Reg(UInt(width = log2Up(nCtrlWords))) + val ctrl_done = Reg(Bool()) + val ctrl_blob = ctrl_regs.toBits + val ctrl_id = Reg(UInt(width = nastiXIdBits)) + + val sizeOffset = 3 * addrBits + val cmdOffset = sizeOffset + dmaWordSizeBits + + val dma_req = new DmaRequest().fromBits(ctrl_blob) + val dma_busy = Reg(init = UInt(0, nDmaXacts)) + val dma_xact_id = PriorityEncoder(~dma_busy) + + when (io.ctrl.aw.fire()) { + ctrl_id := io.ctrl.aw.bits.id + ctrl_idx := UInt(0) + ctrl_done := Bool(false) + state := s_wdata + } + + when (io.ctrl.w.fire()) { + when (!ctrl_done) { + ctrl_regs(ctrl_idx) := io.ctrl.w.bits.data + ctrl_idx := ctrl_idx + UInt(1) + } + when (ctrl_idx === UInt(nCtrlWords - 1)) { ctrl_done := Bool(true) } + when (io.ctrl.w.bits.last) { state := s_dma_req } + } + + dma_busy := (dma_busy | + Mux(io.dma.req.fire(), UIntToOH(dma_xact_id), UInt(0))) & + ~Mux(io.dma.resp.fire(), UIntToOH(io.dma.resp.bits.xact_id), UInt(0)) + + when (io.dma.req.fire()) { state := s_wresp } + when (io.ctrl.b.fire()) { state := s_idle } + + io.ctrl.ar.ready := Bool(false) + io.ctrl.aw.ready := (state === s_idle) + io.ctrl.w.ready := (state === s_wdata) + + io.ctrl.r.valid := Bool(false) + io.ctrl.b.valid := (state === s_wresp) + io.ctrl.b.bits := NastiWriteResponseChannel(id = ctrl_id) + + io.dma.req.valid := (state === s_dma_req) && !dma_busy.andR + io.dma.req.bits := dma_req + io.dma.req.bits.xact_id := dma_xact_id + + val resp_waddr_pending = Reg(init = Bool(false)) + val resp_wdata_pending = Reg(init = Bool(false)) + val resp_wresp_pending = Reg(init = Bool(false)) + val resp_pending = resp_waddr_pending || resp_wdata_pending || resp_wresp_pending + + val resp_client_id = Reg(UInt(width = dmaClientIdBits)) + val resp_status = Reg(UInt(width = dmaStatusBits)) + + io.dma.resp.ready := !resp_pending + + when (io.dma.resp.fire()) { + resp_client_id := io.dma.resp.bits.client_id + resp_status := io.dma.resp.bits.status + resp_waddr_pending := Bool(true) + resp_wdata_pending := Bool(true) + resp_wresp_pending := Bool(true) + } + + val addrTable = Vec.tabulate(nDmaClients) { i => + UInt(addrMap(s"conf:csr$i").start + outstandingCSR * csrDataBytes) + } + + io.mmio.ar.valid := Bool(false) + io.mmio.aw.valid := resp_waddr_pending + io.mmio.aw.bits := NastiWriteAddressChannel( + id = UInt(0), + addr = addrTable(resp_client_id), + size = UInt(log2Up(csrDataBytes))) + io.mmio.w.valid := resp_wdata_pending + io.mmio.w.bits := NastiWriteDataChannel(data = resp_status) + io.mmio.b.ready := resp_wresp_pending + io.mmio.r.ready := Bool(false) + + when (io.mmio.aw.fire()) { resp_waddr_pending := Bool(false) } + when (io.mmio.w.fire()) { resp_wdata_pending := Bool(false) } + when (io.mmio.b.fire()) { resp_wresp_pending := Bool(false) } +} + +class DmaEngine(outstandingCSR: Int)(implicit p: Parameters) extends DmaModule()(p) { + val io = new Bundle { + val ctrl = (new NastiIO).flip + val mem = new ClientUncachedTileLinkIO + val mmio = new NastiIO + } + + val manager = Module(new DmaManager(outstandingCSR)) + val trackers = Module(new DmaTrackerFile) + + manager.io.ctrl <> io.ctrl + trackers.io.dma <> manager.io.dma + + val innerIOs = trackers.io.mem + val outerIOs = trackers.io.mmio :+ manager.io.mmio + + val innerArb = Module(new ClientUncachedTileLinkIOArbiter(innerIOs.size)) + innerArb.io.in <> innerIOs + io.mem <> innerArb.io.out + + val outerArb = Module(new NastiArbiter(outerIOs.size)) + outerArb.io.master <> outerIOs + io.mmio <> outerArb.io.slave + + assert(!io.mmio.b.valid || io.mmio.b.bits.resp === UInt(0), + "DmaEngine: NASTI write response error") + + assert(!io.mmio.r.valid || io.mmio.r.bits.resp === UInt(0), + "DmaEngine: NASTI read response error") +} + +class DmaTrackerFile(implicit p: Parameters) extends DmaModule()(p) { + val io = new Bundle { + val dma = (new DmaIO).flip + val mem = Vec(nDmaTransactors, new ClientUncachedTileLinkIO) + val mmio = Vec(nDmaTransactors, new NastiIO) + } val trackers = List.fill(nDmaTransactors) { Module(new DmaTracker) } val reqReadys = Vec(trackers.map(_.io.dma.req.ready)).toBits + io.mem <> trackers.map(_.io.mem) + io.mmio <> trackers.map(_.io.mmio) + if (nDmaTransactors > 1) { - val inner_arb = Module(new ClientUncachedTileLinkIOArbiter(nDmaTransactors)) - inner_arb.io.in <> trackers.map(_.io.inner) - io.inner <> inner_arb.io.out - - val outer_arb = Module(new NastiArbiter(nDmaTransactors)) - outer_arb.io.master <> trackers.map(_.io.outer) - io.outer <> outer_arb.io.slave - val resp_arb = Module(new RRArbiter(new DmaResponse, nDmaTransactors)) resp_arb.io.in <> trackers.map(_.io.dma.resp) io.dma.resp <> resp_arb.io.out @@ -101,8 +241,6 @@ class DmaEngine(implicit p: Parameters) extends DmaModule()(p) { } io.dma.req.ready := reqReadys.orR } else { - io.inner <> trackers.head.io.inner - io.outer <> trackers.head.io.outer io.dma <> trackers.head.io.dma } } @@ -146,13 +284,13 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) val stream_byte_idx = stream_idx(tlByteAddrBits - 1, 0) val stream_bitshift = Cat(stream_byte_idx, UInt(0, 3)) val stream_in_beat = - (((io.outer.r.bits.data & stream_mask) << stream_bitshift)) | + (((io.mmio.r.bits.data & stream_mask) << stream_bitshift)) | (data_buffer(stream_beat_idx) & ~(stream_mask << stream_bitshift)) val stream_out_word = data_buffer(stream_beat_idx) >> stream_bitshift val stream_out_last = bytes_left === stream_word_bytes - val acq = io.inner.acquire.bits - val gnt = io.inner.grant.bits + val acq = io.mem.acquire.bits + val gnt = io.mem.grant.bits val (s_idle :: s_get :: s_put :: s_prefetch :: s_stream_read_req :: s_stream_read_resp :: @@ -161,14 +299,14 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) val state = Reg(init = s_idle) val (put_beat, put_done) = Counter( - io.inner.acquire.fire() && acq.hasData(), tlDataBeats) + io.mem.acquire.fire() && acq.hasData(), tlDataBeats) val put_mask = Vec.tabulate(tlDataBytes) { i => val byte_index = Cat(put_beat, UInt(i, tlByteAddrBits)) byte_index >= offset && byte_index < bytes_left }.toBits - val prefetch_sent = io.inner.acquire.fire() && io.inner.acquire.bits.isPrefetch() + val prefetch_sent = io.mem.acquire.fire() && io.mem.acquire.bits.isPrefetch() val prefetch_busy = Reg(init = UInt(0, tlMaxClientXacts)) val (prefetch_id, _) = Counter(prefetch_sent, tlMaxClientXacts) @@ -223,48 +361,51 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) PutPrefetch(client_xact_id = prefetch_id, addr_block = dst_block), GetPrefetch(client_xact_id = prefetch_id, addr_block = dst_block)) - val resp_id = Reg(UInt(width = dmaClientXactIdBits)) + val resp_xact_id = Reg(UInt(width = dmaXactIdBits)) + val resp_client_id = Reg(UInt(width = dmaClientIdBits)) - io.inner.acquire.valid := (state === s_get) || + io.mem.acquire.valid := (state === s_get) || (state === s_put && get_done) || (state === s_prefetch && !prefetch_busy(prefetch_id)) - io.inner.acquire.bits := MuxBundle( + io.mem.acquire.bits := MuxBundle( state, prefetch_acquire, Seq( s_get -> get_acquire, s_put -> put_acquire)) - io.inner.grant.ready := Bool(true) + io.mem.grant.ready := Bool(true) io.dma.req.ready := state === s_idle io.dma.resp.valid := state === s_resp - io.dma.resp.bits.client_xact_id := resp_id + io.dma.resp.bits.xact_id := resp_xact_id + io.dma.resp.bits.client_id := resp_client_id io.dma.resp.bits.status := UInt(0) - io.outer.ar.valid := (state === s_stream_read_req) - io.outer.ar.bits := NastiReadAddressChannel( + io.mmio.ar.valid := (state === s_stream_read_req) + io.mmio.ar.bits := NastiReadAddressChannel( id = UInt(0), addr = stream_addr, size = stream_size, len = stream_len, burst = BURST_FIXED) - io.outer.r.ready := (state === s_stream_read_resp) + io.mmio.r.ready := (state === s_stream_read_resp) - io.outer.aw.valid := (state === s_stream_write_req) - io.outer.aw.bits := NastiWriteAddressChannel( + io.mmio.aw.valid := (state === s_stream_write_req) + io.mmio.aw.bits := NastiWriteAddressChannel( id = UInt(0), addr = stream_addr, size = stream_size, len = stream_len, burst = BURST_FIXED) - io.outer.w.valid := (state === s_stream_write_data) && get_done - io.outer.w.bits := NastiWriteDataChannel( + io.mmio.w.valid := (state === s_stream_write_data) && get_done + io.mmio.w.bits := NastiWriteDataChannel( data = stream_out_word, last = stream_out_last) - io.outer.b.ready := (state === s_stream_write_resp) + io.mmio.b.ready := (state === s_stream_write_resp) when (io.dma.req.fire()) { val src_off = io.dma.req.bits.source(blockOffset - 1, 0) val dst_off = io.dma.req.bits.dest(blockOffset - 1, 0) val direction = src_off < dst_off - resp_id := io.dma.req.bits.client_xact_id + resp_xact_id := io.dma.req.bits.xact_id + resp_client_id := io.dma.req.bits.client_id src_block := io.dma.req.bits.source(addrBits - 1, blockOffset) dst_block := io.dma.req.bits.dest(addrBits - 1, blockOffset) alignment := Mux(direction, dst_off - src_off, src_off - dst_off) @@ -300,29 +441,31 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) } } - when (io.outer.ar.fire()) { state := s_stream_read_resp } + when (io.mmio.ar.fire()) { state := s_stream_read_resp } - when (io.outer.r.fire()) { + when (io.mmio.r.fire()) { data_buffer(stream_beat_idx) := stream_in_beat stream_idx := stream_idx + stream_word_bytes val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes - when (block_finished || io.outer.r.bits.last) { state := s_put } + when (block_finished || io.mmio.r.bits.last) { state := s_put } } - when (io.outer.aw.fire()) { state := s_get } + when (io.mmio.aw.fire()) { state := s_get } - when (io.outer.w.fire()) { + when (io.mmio.w.fire()) { stream_idx := stream_idx + stream_word_bytes bytes_left := bytes_left - stream_word_bytes val block_finished = stream_idx === UInt(blockBytes) - stream_word_bytes when (stream_out_last) { - state := s_resp + state := s_stream_write_resp } .elsewhen (block_finished) { state := s_get } } - when (state === s_get && io.inner.acquire.ready) { + when (io.mmio.b.fire()) { state := s_resp } + + when (state === s_get && io.mem.acquire.ready) { get_inflight := get_inflight | FillInterleaved(tlDataBeats, UIntToOH(get_half)) src_block := src_block + UInt(1) when (streaming) { @@ -348,7 +491,7 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) } } - when (io.inner.grant.fire()) { + when (io.mem.grant.fire()) { when (gnt.g_type === Grant.prefetchAckType) { prefetch_busy := prefetch_busy & ~UIntToOH(gnt.client_xact_id) } .elsewhen (gnt.hasData()) { @@ -385,33 +528,3 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) when (io.dma.resp.fire()) { state := s_idle } } - -class DmaArbiter(arbN: Int)(implicit p: Parameters) extends DmaModule()(p) { - val io = new Bundle { - val in = Vec(arbN, new DmaIO).flip - val out = new DmaIO - } - - if (arbN > 1) { - val idBits = log2Up(arbN) - val req_arb = Module(new RRArbiter(new DmaRequest, arbN)) - val out_resp_client_id = io.out.resp.bits.client_xact_id(idBits - 1, 0) - - for (i <- 0 until arbN) { - req_arb.io.in(i) <> io.in(i).req - req_arb.io.in(i).bits.client_xact_id := Cat( - io.in(i).req.bits.client_xact_id, - UInt(i, idBits)) - - io.in(i).resp.valid := io.out.resp.valid && out_resp_client_id === UInt(i) - io.in(i).resp.bits := io.out.resp.bits - } - - val respReadys = Vec(io.in.map(_.resp.ready)) - - io.out.req <> req_arb.io.out - io.out.resp.ready := respReadys(out_resp_client_id) - } else { - io.out <> io.in.head - } -} From c1fe188c8171690cb6b127fbbb383ada4aedaec2 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 14 Jan 2016 22:01:14 -0800 Subject: [PATCH 512/688] some fixes to RTC --- uncore/src/main/scala/rtc.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 93c8807c..830c4351 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -6,10 +6,10 @@ import cde.{Parameters, Field} case object RTCPeriod extends Field[Int] -class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule { +class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule + with HasAddrMapParameters { val io = new NastiIO - val addrMap = new AddrHashMap(p(GlobalAddrMap)) val addrTable = Vec.tabulate(nCores) { i => UInt(addrMap(s"conf:csr$i").start + csr_MTIME * scrDataBytes) } @@ -60,5 +60,8 @@ class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule { io.r.ready := Bool(false) assert(!rtc_tick || send_acked.reduce(_ && _), - s"Not all clocks were updated for rtc tick") + "Not all clocks were updated for rtc tick") + + assert(!io.b.valid || io.b.bits.resp === UInt(0), + "RTC received NASTI error response") } From adaec18bec11af88570a18de37467747361bb20c Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 14 Jan 2016 22:01:42 -0800 Subject: [PATCH 513/688] add TL manager for MMIO requests --- uncore/src/main/scala/converters.scala | 68 ++++++++++++++++++++++++++ uncore/src/main/scala/uncore.scala | 1 + 2 files changed, 69 insertions(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 22e9532a..509aa2b4 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -729,3 +729,71 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) } } else { io.out <> io.in } } + +class MMIOTileLinkManagerData(implicit p: Parameters) + extends TLBundle()(p) + with HasClientId + with HasClientTransactionId + +class MMIOTileLinkManager(implicit p: Parameters) + extends CoherenceAgentModule()(p) { + val io = new ManagerTLIO + + // MMIO requests should never need probe or release + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + + val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData() + val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0) + val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1) + + // Acquire and Grant are basically passthru, + // except client_id and client_xact_id need to be converted. + // Associate the inner client_id and client_xact_id + // with the outer client_xact_id. + val xact_pending = Reg(init = UInt(0, maxManagerXacts)) + val xact_id_sel = PriorityEncoder(~xact_pending) + val xact_id_reg = RegEnable(xact_id_sel, multibeat_start) + val xact_multibeat = Reg(init = Bool(false)) + val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel) + val xact_free = !xact_pending.andR + val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData)) + + io.inner.acquire.ready := io.outer.acquire.ready && xact_free + io.outer.acquire.valid := io.inner.acquire.valid && xact_free + io.outer.acquire.bits := io.inner.acquire.bits + io.outer.acquire.bits.client_xact_id := outer_xact_id + + def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool = + !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1) + + def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt = + Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0)) + + def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt = + ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(), + UIntToOH(in.bits.manager_xact_id), UInt(0)) + + def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt = + ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0)) + + xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) & + clearPendingBitOnFin(io.inner.finish) & + clearPendingBitOnGnt(io.inner.grant) + + when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) { + xact_buffer(outer_xact_id) := io.iacq() + } + + when (multibeat_start) { xact_multibeat := Bool(true) } + when (multibeat_end) { xact_multibeat := Bool(false) } + + val gnt_xact = xact_buffer(io.ognt().client_xact_id) + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + io.inner.grant.bits := io.outer.grant.bits + io.inner.grant.bits.client_id := gnt_xact.client_id + io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id + io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id + io.inner.finish.ready := xact_pending(io.inner.finish.bits.manager_xact_id) +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 3d38f6b2..34405aae 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -32,6 +32,7 @@ trait HasCoherenceAgentParameters { val innerWriteMaskBits = innerTLParams.writeMaskBits val innerBeatAddrBits = log2Up(innerDataBeats) val innerByteAddrBits = log2Up(innerDataBits/8) + val maxManagerXacts = innerTLParams.maxManagerXacts require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement } From 66e9cc8c820e00b5f64a61ad2cc314e64de2b20f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 28 Jan 2016 14:06:38 -0800 Subject: [PATCH 514/688] make sure CSR width is parameterizable --- uncore/src/main/scala/dma.scala | 10 ++++++---- uncore/src/main/scala/htif.scala | 16 +++++++++------- uncore/src/main/scala/rtc.scala | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala index 69a3fb26..16fa01b0 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/dma.scala @@ -19,8 +19,6 @@ trait HasDmaParameters { val addrBits = p(PAddrBits) val dmaStatusBits = 2 val dmaWordSizeBits = 2 - val csrDataBits = 64 - val csrDataBytes = csrDataBits / 8 } abstract class DmaModule(implicit val p: Parameters) extends Module with HasDmaParameters @@ -82,8 +80,12 @@ class DmaTrackerIO(implicit p: Parameters) extends DmaBundle()(p) { val mmio = new NastiIO } -class DmaManager(outstandingCSR: Int)(implicit p: Parameters) extends DmaModule()(p) - with HasNastiParameters with HasAddrMapParameters { +class DmaManager(outstandingCSR: Int)(implicit p: Parameters) + extends DmaModule()(p) + with HasNastiParameters + with HasAddrMapParameters + with HasHtifParameters { + val io = new Bundle { val ctrl = (new NastiIO).flip val mmio = new NastiIO diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 9b72459b..f20d1954 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -9,20 +9,22 @@ import cde.{Parameters, Field} case object HtifKey extends Field[HtifParameters] -case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, nSCR: Int = 64) +case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, csrDataBits: Int, nSCR: Int = 64) trait HasHtifParameters { implicit val p: Parameters - val external = p(HtifKey) + val htifExternal = p(HtifKey) val dataBits = p(TLKey(p(TLId))).dataBitsPerBeat val dataBeats = p(TLKey(p(TLId))).dataBeats - val w = external.width - val nSCR = external.nSCR + val w = htifExternal.width + val nSCR = htifExternal.nSCR val scrAddrBits = log2Up(nSCR) val scrDataBits = 64 val scrDataBytes = scrDataBits / 8 - val offsetBits = external.offsetBits - val nCores = external.nCores + val csrDataBits = htifExternal.csrDataBits + val csrDataBytes = csrDataBits / 8 + val offsetBits = htifExternal.offsetBits + val nCores = htifExternal.nCores } abstract class HtifModule(implicit val p: Parameters) extends Module with HasHtifParameters @@ -40,7 +42,7 @@ class HostIO(w: Int) extends Bundle { class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val csr = new SmiIO(scrDataBits, 12).flip + val csr = new SmiIO(csrDataBits, 12).flip val debug_stats_csr = Bool(OUTPUT) // wired directly to stats register // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 830c4351..6c3f83a4 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -11,7 +11,7 @@ class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule val io = new NastiIO val addrTable = Vec.tabulate(nCores) { i => - UInt(addrMap(s"conf:csr$i").start + csr_MTIME * scrDataBytes) + UInt(addrMap(s"conf:csr$i").start + csr_MTIME * csrDataBytes) } val rtc = Reg(init=UInt(0, scrDataBits)) @@ -50,7 +50,7 @@ class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule io.aw.bits := NastiWriteAddressChannel( id = coreId, addr = addrTable(coreId), - size = UInt(log2Up(scrDataBytes))) + size = UInt(log2Up(csrDataBytes))) io.w.valid := sending_data io.w.bits := NastiWriteDataChannel(data = rtc) From 2825b2d6455db8ba1724c1bae14b043aed47f7b0 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 10 Feb 2016 11:06:41 -0800 Subject: [PATCH 515/688] make sure TL to NASTI converter handles MT_WU --- uncore/src/main/scala/converters.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 509aa2b4..19dac127 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -393,6 +393,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) MT_H -> UInt(1), MT_HU -> UInt(1), MT_W -> UInt(2), + MT_WU -> UInt(2), MT_D -> UInt(3), MT_Q -> UInt(log2Up(tlDataBytes)))) From 53ad8387cc5a17231603ee7e453ee319865c11a1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 10 Feb 2016 11:06:52 -0800 Subject: [PATCH 516/688] add NASTI to TL converter --- uncore/src/main/scala/converters.scala | 327 +++++++++++++++++++++++++ 1 file changed, 327 insertions(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 19dac127..25953952 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -515,6 +515,332 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") } +class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val nasti = (new NastiIO).flip + val tl = new ClientUncachedTileLinkIO + } + + val (s_idle :: s_put :: Nil) = Enum(Bits(), 2) + val state = Reg(init = s_idle) + + private val blockOffset = tlByteAddrBits + tlBeatAddrBits + + val aw_req = Reg(new NastiWriteAddressChannel) + + def is_singlebeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(0) + + def is_multibeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(tlDataBeats - 1) && chan.size === UInt(log2Up(tlDataBytes)) + + def nasti_addr_block(chan: NastiAddressChannel): UInt = + chan.addr(nastiXAddrBits - 1, blockOffset) + + def nasti_addr_beat(chan: NastiAddressChannel): UInt = + chan.addr(blockOffset - 1, tlByteAddrBits) + + def nasti_addr_byte(chan: NastiAddressChannel): UInt = + chan.addr(tlByteAddrBits - 1, 0) + + def nasti_operand_size(chan: NastiAddressChannel): UInt = + MuxLookup(chan.size, MT_Q, Seq( + UInt(0) -> MT_BU, + UInt(1) -> MT_HU, + UInt(2) -> MT_WU, + UInt(3) -> MT_D)) + + def size_mask(size: UInt): UInt = + (UInt(1) << (UInt(1) << size)) - UInt(1) + + def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { + val base = w.strb & size_mask(aw.size) + val addr_byte = nasti_addr_byte(aw) + base << addr_byte + } + + def nasti_wdata(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { + val base = w.data & FillInterleaved(8, size_mask(aw.size)) + val addr_byte = nasti_addr_byte(aw) + base << Cat(addr_byte, UInt(0, 3)) + } + + def tl_last(gnt: GrantMetadata): Bool = + !gnt.hasMultibeatData() || gnt.addr_beat === UInt(tlDataBeats - 1) + + def tl_b_grant(gnt: GrantMetadata): Bool = + gnt.g_type === Grant.putAckType + + assert(!io.nasti.ar.valid || + is_singlebeat(io.nasti.ar.bits) || is_multibeat(io.nasti.ar.bits), + "NASTI read transaction cannot convert to TileLInk") + + assert(!io.nasti.aw.valid || + is_singlebeat(io.nasti.aw.bits) || is_multibeat(io.nasti.aw.bits), + "NASTI write transaction cannot convert to TileLInk") + + val put_count = Reg(init = UInt(0, tlBeatAddrBits)) + + when (io.nasti.aw.fire()) { + aw_req := io.nasti.aw.bits + state := s_put + } + + when (io.nasti.w.fire()) { + put_count := put_count + UInt(1) + when (io.nasti.w.bits.last) { + put_count := UInt(0) + state := s_idle + } + } + + val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), + GetBlock( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits)), + Get( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits), + addr_beat = nasti_addr_beat(io.nasti.ar.bits), + addr_byte = nasti_addr_byte(io.nasti.ar.bits), + operand_size = nasti_operand_size(io.nasti.ar.bits), + alloc = Bool(false))) + + val put_acquire = Mux(is_multibeat(aw_req), + PutBlock( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = put_count, + data = io.nasti.w.bits.data, + wmask = io.nasti.w.bits.strb), + Put( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = nasti_addr_beat(aw_req), + data = nasti_wdata(aw_req, io.nasti.w.bits), + wmask = nasti_wmask(aw_req, io.nasti.w.bits))) + + io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) + io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || + (state === s_put && io.nasti.w.valid) + io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) + io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) + io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) + + val acq = io.tl.acquire.bits + val nXacts = tlMaxClientXacts * tlMaxClientsPerPort + val get_align = Reg(Vec(nXacts, UInt(width = tlByteAddrBits))) + val is_narrow_get = acq.a_type === Acquire.getType + + when (io.tl.acquire.fire() && is_narrow_get) { + get_align(acq.client_xact_id) := acq.addr_byte() + } + + def tl_data(gnt: Grant): UInt = + Mux(gnt.g_type === Grant.getDataBeatType, + gnt.data >> Cat(get_align(gnt.client_xact_id), UInt(0, 3)), + gnt.data) + + io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) + io.nasti.b.bits := NastiWriteResponseChannel( + id = io.tl.grant.bits.client_xact_id) + + io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) + io.nasti.r.bits := NastiReadDataChannel( + id = io.tl.grant.bits.client_xact_id, + data = tl_data(io.tl.grant.bits), + last = tl_last(io.tl.grant.bits)) + + io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), + io.nasti.b.ready, io.nasti.r.ready) +} + +class TileLinkIOWidener(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val paddrBits = p(PAddrBits) + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val innerMaxXacts = innerParams.maxClientXacts * innerParams.maxClientsPerPort + val innerXactIdBits = log2Up(innerMaxXacts) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerClientIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + val outerManagerIdBits = log2Up(outerParams.maxManagerXacts) + val outerBlockAddrBits = paddrBits - outerBlockOffset + + require(outerDataBeats <= innerDataBeats) + require(outerDataBits >= innerDataBits) + require(outerDataBits % innerDataBits == 0) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = innerDataBeats / outerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + val ignt = io.in.grant.bits + + val shrink = iacq.a_type === Acquire.putBlockType + val stretch = ognt.g_type === Grant.getDataBlockType + val smallget = iacq.a_type === Acquire.getType + val smallput = iacq.a_type === Acquire.putType + val smallgnt = ognt.g_type === Grant.getDataBeatType + + val sending_put = Reg(init = Bool(false)) + val collecting = Reg(init = Bool(false)) + val put_block = Reg(UInt(width = outerBlockAddrBits)) + val put_id = Reg(UInt(width = outerClientIdBits)) + val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) + val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) + val put_allocate = Reg(Bool()) + val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) + val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) + + val in_addr = iacq.full_addr() + val out_addr_block = in_addr(paddrBits - 1, outerBlockOffset) + val out_addr_beat = in_addr(outerBlockOffset - 1, outerByteAddrBits) + val out_addr_byte = in_addr(outerByteAddrBits - 1, 0) + + val switch_addr = in_addr(outerByteAddrBits - 1, innerByteAddrBits) + val smallget_switch = Reg(Vec(innerMaxXacts, switch_addr)) + + def align_data(addr: UInt, data: UInt): UInt = + data << Cat(addr, UInt(0, log2Up(innerDataBits))) + + def align_wmask(addr: UInt, wmask: UInt): UInt = + wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + addr_byte = out_addr_byte, + operand_size = iacq.op_size(), + alloc = iacq.allocate()) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + alloc = iacq.allocate()) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + data = align_data(switch_addr, iacq.data), + wmask = align_wmask(switch_addr, iacq.wmask()), + alloc = iacq.allocate()) + + val put_block_acquire = PutBlock( + client_xact_id = put_id, + addr_block = put_block, + addr_beat = put_beat, + data = put_data.toBits, + wmask = put_wmask.toBits) + + io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) + io.out.acquire.bits := MuxBundle(get_block_acquire, Seq( + sending_put -> put_block_acquire, + smallget -> get_acquire, + smallput -> put_acquire)) + io.in.acquire.ready := !sending_put && (shrink || io.out.acquire.ready) + + when (io.in.acquire.fire() && shrink) { + when (!collecting) { + put_block := out_addr_block + put_id := iacq.client_xact_id + put_allocate := iacq.allocate() + collecting := Bool(true) + } + put_data(recv_idx) := iacq.data + put_wmask(recv_idx) := iacq.wmask() + } + + when (io.in.acquire.fire() && smallget) { + smallget_switch(iacq.client_xact_id) := switch_addr + } + + when (recv_done) { sending_put := Bool(true) } + when (sending_put && io.out.acquire.ready) { sending_put := Bool(false) } + when (put_done) { collecting := Bool(false) } + + val returning_data = Reg(init = Bool(false)) + val (send_idx, send_done) = Counter( + io.in.grant.ready && returning_data, factor) + + val gnt_beat = Reg(UInt(width = outerBeatAddrBits)) + val gnt_client_id = Reg(UInt(width = outerClientIdBits)) + val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) + val gnt_data = Reg(UInt(width = outerDataBits)) + val gnt_data_vec = Vec.tabulate(factor) { i => + gnt_data(innerDataBits * (i + 1) - 1, innerDataBits * i) + } + + when (io.out.grant.fire() && stretch) { + gnt_data := ognt.data + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + gnt_beat := ognt.addr_beat + returning_data := Bool(true) + } + + when (send_done) { returning_data := Bool(false) } + + def select_data(data: UInt, sel: UInt): UInt = { + val data_vec = Vec.tabulate(factor) { i => + data(innerDataBits * (i + 1) - 1, innerDataBits * i) + } + data_vec(sel) + } + + val gnt_switch = smallget_switch(ognt.client_xact_id) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = Cat(gnt_beat, send_idx), + data = gnt_data_vec(send_idx)) + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = Cat(ognt.addr_beat, gnt_switch), + data = select_data(ognt.data, gnt_switch)) + + val default_grant = Grant( + is_builtin_type = Bool(true), + g_type = ognt.g_type, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) + io.in.grant.bits := MuxBundle(default_grant, Seq( + returning_data -> get_block_grant, + smallgnt -> get_grant)) + io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) +} + class TileLinkIONarrower(innerTLId: String, outerTLId: String) (implicit p: Parameters) extends TLModule()(p) { @@ -594,6 +920,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) MT_H -> Bool(outerDataBits >= 16), MT_HU -> Bool(outerDataBits >= 16), MT_W -> Bool(outerDataBits >= 32), + MT_WU -> Bool(outerDataBits >= 32), MT_D -> Bool(outerDataBits >= 64), MT_Q -> Bool(false))) From 1ac9f59b310b361bcecb9dc51b915a49ed7b56b3 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Wed, 17 Feb 2016 14:06:34 -0800 Subject: [PATCH 517/688] Allow SCR files to be enumerated in C headers Right now there's no way to ensure that SCR addresses don't conflict within RocketChip. Since upstream only has one of them this isn't a big deal, but we want to add a whole bunch more to control all the IP on Hurricane. This patch adds some Scala code to allocate registers inside the SCR file, ensure they don't conflict, to provide names for SCRs, attach registers to the SCR file, and generate a C header file that contains the addresses of every SCR on a chip. With this patch we'll be able to get rid of that constant in the testbench. This also allows us to kill one of the Raven diffs, which is does pretty much the same thing (just in a second SCR file, and hacked in). --- uncore/src/main/scala/scr.scala | 65 ++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index ddc35ded..08949ca9 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -3,25 +3,71 @@ package uncore import Chisel._ import junctions.{SmiIO, MMIOBase} import cde.Parameters +import scala.collection.mutable.HashMap +import scala.collection.mutable.ArrayBuffer -class SCRIO(implicit p: Parameters) extends HtifBundle()(p) { +/** Stores a map between SCR file names and address in the SCR file, which can + * later be dumped to a header file for the test bench. */ +class SCRFileMap(prefix: String, maxAddress: Int) { + private val addr2name = HashMap.empty[Int, String] + private val name2addr = HashMap.empty[String, Int] + + def allocate(address: Int, name: String): Int = { + Predef.assert(!addr2name.contains(address), "address already allocated") + Predef.assert(!name2addr.contains(name), "name already allocated") + Predef.assert(address < maxAddress, "address too large") + addr2name += (address -> name) + name2addr += (name -> address) + println(prefix + ": " + address + " -> " + name) + address + } + + def allocate(name: String): Int = { + val addr = (0 until maxAddress).filter{ addr => !addr2name.contains(addr) }(0) + allocate(addr, name) + } + + def as_c_header(): String = { + addr2name.map{ case(address, name) => + "#define " + prefix + "__" + name + " " + address + }.mkString("\n") + "\n" + } +} + +class SCRIO(map: SCRFileMap)(implicit p: Parameters) extends HtifBundle()(p) { val rdata = Vec(nSCR, Bits(INPUT, scrDataBits)) val wen = Bool(OUTPUT) val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, scrDataBits) + + def attach(reg: Data, name: String): Data = { + val addr = map.allocate(name) + when (wen && (waddr === UInt(addr))) { + reg := wdata + } + rdata(addr) := reg + reg + } + + def allocate(address: Int, name: String): Unit = { + map.allocate(address, name) + } } -class SCRFile(implicit p: Parameters) extends HtifModule()(p) { +class SCRFile(prefix: String)(implicit p: Parameters) extends HtifModule()(p) { + val map = new SCRFileMap(prefix, 64) + AllSCRFiles += map + val io = new Bundle { val smi = new SmiIO(scrDataBits, scrAddrBits).flip - val scr = new SCRIO + val scr = new SCRIO(map) } val scr_rdata = Wire(Vec(io.scr.rdata.size, Bits(width=scrDataBits))) for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nCores) - scr_rdata(1) := UInt(p(MMIOBase) >> 20) + scr_rdata(0) := UInt(nCores); map.allocate(0, "N_CORES") + scr_rdata(1) := UInt(p(MMIOBase) >> 20); map.allocate(1, "MMIO_BASE") val read_addr = Reg(init = UInt(0, scrAddrBits)) val resp_valid = Reg(init = Bool(false)) @@ -40,3 +86,12 @@ class SCRFile(implicit p: Parameters) extends HtifModule()(p) { } when (io.smi.resp.fire()) { resp_valid := Bool(false) } } + +/** Every elaborated SCR file ends up in this global arry so it can be printed + * out later. */ +object AllSCRFiles { + private var maps = ArrayBuffer.empty[SCRFileMap] + + def +=(map: SCRFileMap): Unit = { maps += map } + def foreach( f: (SCRFileMap => Unit) ): Unit = { maps.foreach{ m => f(m) } } +} From f97bd70df54fb2e8542324d0c6dea16ca638fe81 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 17 Feb 2016 21:29:29 -0800 Subject: [PATCH 518/688] add NastiIO to HostIO converter --- uncore/src/main/scala/htif.scala | 100 +++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index f20d1954..1391d71c 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -5,6 +5,7 @@ package uncore import Chisel._ import Chisel.ImplicitConversions._ import junctions._ +import junctions.NastiConstants._ import cde.{Parameters, Field} case object HtifKey extends Field[HtifParameters] @@ -225,3 +226,102 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt io.host.out.valid := state === state_tx io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) } + +class NastiIOHostIOConverter(htifW: Int)(implicit val p: Parameters) + extends Module with HasNastiParameters { + val io = new Bundle { + val nasti = (new NastiIO).flip + val host = new HostIO(htifW).flip + val reset = Bool(OUTPUT) + } + + val raddr = io.nasti.ar.bits.addr(6, 2) + val waddr = io.nasti.aw.bits.addr(6, 2) + + + val DCOUNT_ADDR = 0x00 + val RFIFO_ADDR = 0x01 + + val WFIFO_ADDR = 0x00 + val RESET_ADDR = 0x1f + + val FIFO_DEPTH = 32 + + val fifo_ren = Reg(init = Bool(false)) + val fifo_wen = Reg(init = Bool(false)) + + val fifo_rd_len = Reg(UInt(width = nastiXLenBits)) + val fifo_rd_id = Reg(UInt(width = nastiXIdBits)) + val fifo_wr_id = Reg(UInt(width = nastiXIdBits)) + val fifo_wr_ack = Reg(init = Bool(false)) + + val rd_count = Reg(init = Bool(false)) + val wr_reset = Reg(init = Bool(false)) + + when (io.nasti.ar.fire()) { + fifo_rd_len := io.nasti.ar.bits.len + fifo_rd_id := io.nasti.ar.bits.id + when (raddr === UInt(RFIFO_ADDR)) { + fifo_ren := Bool(true) + } .elsewhen (raddr === UInt(DCOUNT_ADDR)) { + rd_count := Bool(true) + } + } + + when (io.nasti.r.fire()) { + when (io.nasti.r.bits.last) { + fifo_ren := Bool(false) + rd_count := Bool(false) + } .otherwise { fifo_rd_len := fifo_rd_len - UInt(1) } + } + + when (io.nasti.aw.fire()) { + fifo_wr_id := io.nasti.aw.bits.id + when (waddr === UInt(WFIFO_ADDR)) { + fifo_wen := Bool(true) + } .elsewhen (waddr === UInt(RESET_ADDR)) { + wr_reset := Bool(true) + } + } + + when (io.nasti.w.fire() && io.nasti.w.bits.last) { + fifo_wen := Bool(false) + wr_reset := Bool(false) + fifo_wr_ack := Bool(true) + } + + when (io.nasti.b.fire()) { fifo_wr_ack := Bool(false) } + + io.nasti.ar.ready := !fifo_ren + io.nasti.aw.ready := !fifo_wen && !fifo_wr_ack + io.nasti.b.valid := fifo_wr_ack + io.nasti.b.bits := NastiWriteResponseChannel(id = fifo_wr_id) + + io.reset := io.nasti.w.valid && wr_reset + + val hn_fifo = Module(new MultiWidthFifo(htifW, nastiXDataBits, FIFO_DEPTH)) + hn_fifo.io.in <> io.host.out + hn_fifo.io.out.ready := fifo_ren && io.nasti.r.ready + io.nasti.r.valid := (fifo_ren && hn_fifo.io.out.valid) || rd_count + io.nasti.r.bits := NastiReadDataChannel( + id = fifo_rd_id, + data = Mux(fifo_ren, hn_fifo.io.out.bits, hn_fifo.io.count), + last = (fifo_rd_len === UInt(0))) + + val nh_fifo = Module(new MultiWidthFifo(nastiXDataBits, htifW, FIFO_DEPTH)) + io.host.in <> nh_fifo.io.out + nh_fifo.io.in.valid := fifo_wen && io.nasti.w.valid + nh_fifo.io.in.bits := io.nasti.w.bits.data + io.nasti.w.ready := (fifo_wen && nh_fifo.io.in.ready) || wr_reset + + assert(!io.nasti.w.valid || io.nasti.w.bits.strb.andR, + "Nasti to HostIO converter cannot take partial writes") + assert(!io.nasti.ar.valid || + io.nasti.ar.bits.len === UInt(0) || + io.nasti.ar.bits.burst === BURST_FIXED, + "Nasti to HostIO converter can only take fixed bursts") + assert(!io.nasti.aw.valid || + io.nasti.aw.bits.len === UInt(0) || + io.nasti.aw.bits.burst === BURST_FIXED, + "Nasti to HostIO converter can only take fixed bursts") +} From 929d8e31f702985cf5d116a6ca88213dad9d008c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 10 Feb 2016 11:12:43 -0800 Subject: [PATCH 519/688] refactor ready/valid logic for routing release messages in the l2 --- uncore/src/main/scala/broadcast.scala | 115 +++++++++++--------------- uncore/src/main/scala/cache.scala | 39 +++------ uncore/src/main/scala/uncore.scala | 14 +++- 3 files changed, 71 insertions(+), 97 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 4b6627aa..c3eb7038 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -83,19 +83,15 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) // Handle releases, which might be voluntary and might have data val trackerReleaseIOs = trackerList.map(_.io.inner.release) val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits - val releaseMatches = Vec(trackerList.map(_.io.has_release_match)).toBits - val release_idx = PriorityEncoder(releaseMatches) - io.inner.release.ready := releaseReadys(release_idx) + io.inner.release.ready := releaseReadys.orR trackerReleaseIOs.zipWithIndex.foreach { case(tracker, i) => - tracker.valid := io.inner.release.valid && (release_idx === UInt(i)) + tracker.valid := io.inner.release.valid tracker.bits := io.inner.release.bits tracker.bits.data := DataQueueLocation(rel_data_cnt, (if(i < nReleaseTransactors) inVolWBQueue else inClientReleaseQueue)).toBits } - assert(!(io.inner.release.valid && !releaseMatches.orR), - "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -129,85 +125,68 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { val io = new ManagerXactTrackerIO + pinAllReadyValidLow(io) } class BroadcastVoluntaryReleaseTracker(trackerId: Int) (implicit p: Parameters) extends BroadcastXactTracker()(p) { - val s_idle :: s_outer :: s_grant :: s_ack :: Nil = Enum(UInt(), 4) + val s_idle :: s_busy :: Nil = Enum(UInt(), 2) val state = Reg(init=s_idle) val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId }))) val coh = ManagerMetadata.onReset - val collect_irel_data = Reg(init=Bool(false)) - val irel_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) + val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats)) + val pending_ignt = Reg(init=Bool(false)) - io.has_acquire_conflict := Bool(false) - io.has_release_match := io.irel().isVoluntary() - io.has_acquire_match := Bool(false) + val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt) - io.outer.acquire.valid := Bool(false) - io.outer.grant.ready := Bool(false) - io.inner.acquire.ready := Bool(false) - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.grant.valid := Bool(false) - io.inner.finish.ready := Bool(false) - - io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + // Accept a voluntary Release (and any further beats of data) + pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR + when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } + // Write the voluntarily written back data to outer memory using an Acquire.PutBlock //TODO: Use io.outer.release instead? + pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) | + addPendingBitWhenBeatHasData(io.inner.release) + val curr_write_beat = PriorityEncoder(pending_writes) + io.outer.acquire.valid := state === s_busy && pending_writes.orR io.outer.acquire.bits := PutBlock( client_xact_id = UInt(trackerId), addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = xact.data_buffer(oacq_data_cnt)) + addr_beat = curr_write_beat, + data = xact.data_buffer(curr_write_beat)) (p.alterPartial({ case TLId => outerTLId })) - when(collect_irel_data) { - io.inner.release.ready := Bool(true) - when(io.inner.release.valid) { - xact.data_buffer(io.irel().addr_beat) := io.irel().data - irel_data_valid := irel_data_valid.bitSet(io.irel().addr_beat, Bool(true)) - } - when(irel_data_done) { collect_irel_data := Bool(false) } - } + // Send an acknowledgement + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid + io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + when(io.inner.grant.fire()) { pending_ignt := Bool(false) } + io.outer.grant.ready := state === s_busy && io.inner.grant.ready - switch (state) { - is(s_idle) { - io.inner.release.ready := Bool(true) - when( io.inner.release.valid ) { - xact := io.irel() - xact.data_buffer(UInt(0)) := io.irel().data - collect_irel_data := io.irel().hasMultibeatData() - irel_data_valid := io.irel().hasData() << io.irel().addr_beat - state := Mux(io.irel().hasData(), s_outer, - Mux(io.irel().requiresAck(), s_ack, s_idle)) - } - } - is(s_outer) { - io.outer.acquire.valid := !collect_irel_data || irel_data_valid(oacq_data_cnt) - when(oacq_data_done) { - state := s_grant // converted irel to oacq, so expect grant TODO: Mux(xact.requiresAck(), s_grant, s_idle) ? - } - } - is(s_grant) { // Forward the Grant.voluntaryAck - io.outer.grant.ready := io.inner.grant.ready - io.inner.grant.valid := io.outer.grant.valid - when(io.inner.grant.fire()) { - state := Mux(io.ignt().requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { - // TODO: This state is unnecessary if no client will ever issue the - // pending Acquire that caused this writeback until it receives the - // Grant.voluntaryAck for this writeback - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } + // State machine updates and transaction handler metadata intialization + when(state === s_idle && io.inner.release.fire()) { + xact := io.irel() + when(io.irel().hasMultibeatData()) { + pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) + }. otherwise { + pending_irels := UInt(0) } + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + pending_ignt := io.irel().requiresAck() + state := s_busy } + when(state === s_busy && all_pending_done) { state := s_idle } + + // These IOs are used for routing in the parent + io.has_acquire_match := Bool(false) + io.has_acquire_conflict := Bool(false) + + // Checks for illegal behavior + assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } class BroadcastAcquireTracker(trackerId: Int) @@ -247,9 +226,6 @@ class BroadcastAcquireTracker(trackerId: Int) !collect_iacq_data io.has_acquire_match := xact.conflicts(io.iacq()) && collect_iacq_data - io.has_release_match := xact.conflicts(io.irel()) && - !io.irel().isVoluntary() && - (state === s_probe) val outerParams = p.alterPartial({ case TLId => outerTLId }) @@ -357,8 +333,9 @@ class BroadcastAcquireTracker(trackerId: Int) } // Handle releases, which may have data to be written back - io.inner.release.ready := !io.irel().hasData() || io.outer.acquire.ready - when(io.inner.release.valid) { + val matches = xact.conflicts(io.irel()) && !io.irel().isVoluntary() + io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches + when(io.inner.release.valid && matches) { when(io.irel().hasData()) { io.outer.acquire.valid := Bool(true) when(io.outer.acquire.ready) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3192cb66..e01187eb 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -405,15 +405,14 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) // Wire releases from clients val releaseReadys = Vec(trackerAndWbIOs.map(_.inner.release.ready)).toBits - val releaseMatches = Vec(trackerAndWbIOs.map(_.has_release_match)).toBits - io.inner.release.ready := (releaseMatches & releaseReadys).orR + io.inner.release.ready := releaseReadys.orR trackerAndWbIOs foreach { tracker => tracker.inner.release.bits := io.inner.release.bits - tracker.inner.release.valid := io.inner.release.valid && tracker.has_release_match + tracker.inner.release.valid := io.inner.release.valid } - assert(PopCount(releaseMatches) <= UInt(nReleaseTransactors), + assert(PopCount(releaseReadys) <= UInt(nReleaseTransactors), "At most a single tracker should match for any given Release") - assert(!(io.inner.release.valid && !releaseMatches.orR), + assert(!(io.inner.release.valid && !releaseReadys.orR), "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients @@ -491,19 +490,6 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) val isPartial = a.wmask() =/= Acquire.fullWriteMask addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) } - - def pinAllReadyValidLow[T <: Data](b: Bundle) { - b.elements.foreach { - _._2 match { - case d: DecoupledIO[_] => - if(d.ready.dir == OUTPUT) d.ready := Bool(false) - else if(d.valid.dir == OUTPUT) d.valid := Bool(false) - case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) - case b: Bundle => pinAllReadyValidLow(b) - case _ => - } - } - } } class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { @@ -528,7 +514,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends // Accept a voluntary Release (and any further beats of data) pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) - io.inner.release.ready := state === s_idle || pending_irels.orR + io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } // Begin a transaction by getting the current block metadata @@ -566,7 +552,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends xact_old_meta.coh.outer) // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.valid) { + when(state === s_idle && io.inner.release.fire()) { xact := io.irel() when(io.irel().hasMultibeatData()) { pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) @@ -587,7 +573,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends when(state === s_meta_write && io.meta.write.ready) { state := s_idle } // These IOs are used for routing in the parent - io.has_release_match := io.irel().isVoluntary() io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) @@ -816,7 +801,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data - io.inner.release.ready := state === s_inner_probe + io.inner.release.ready := state === s_inner_probe && + io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() val pending_coh_on_irel = HierarchicalMetadata( pending_coh.inner.onRelease(io.irel()), // Drop sharer Mux(io.irel().hasData(), // Dirty writeback @@ -1025,9 +1012,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra // These IOs are used for routing in the parent val in_same_set = xact_addr_idx === io.iacq().addr_block(idxMSB,idxLSB) - io.has_release_match := io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() && - io.inner.release.ready io.has_acquire_match := iacq_can_merge || iacq_same_xact io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match //TODO: relax from in_same_set to xact.conflicts(io.iacq())? @@ -1083,7 +1067,9 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac // and/or write back dirty data val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) - io.inner.release.ready := state === s_inner_probe || state === s_busy + io.inner.release.ready := (state === s_inner_probe || state === s_busy) && + io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() when(io.inner.release.fire()) { xact.coh.inner := inner_coh_on_irel data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data @@ -1150,7 +1136,6 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac when(state === s_wb_resp ) { state := s_idle } // These IOs are used for routing in the parent - io.has_release_match := io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && io.inner.release.ready io.has_acquire_match := Bool(false) io.has_acquire_conflict := Bool(false) } diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 34405aae..c90adeeb 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -117,7 +117,6 @@ abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends Cohere trait HasTrackerConflictIO extends Bundle { val has_acquire_conflict = Bool(OUTPUT) val has_acquire_match = Bool(OUTPUT) - val has_release_match = Bool(OUTPUT) } class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) @@ -154,4 +153,17 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[_] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } } From 8873222e4274d59ebaac3bb1aa176a84b06956e7 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 23 Feb 2016 16:03:51 -0800 Subject: [PATCH 520/688] fix cache release assertion --- uncore/src/main/scala/cache.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e01187eb..edc9b015 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -412,7 +412,8 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) } assert(PopCount(releaseReadys) <= UInt(nReleaseTransactors), "At most a single tracker should match for any given Release") - assert(!(io.inner.release.valid && !releaseReadys.orR), + assert(!io.inner.release.valid || + io.inner.release.bits.isVoluntary() || releaseReadys.orR, "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients From 19420cd5df7bd6e7b1cbd00d54934e3db184f3ba Mon Sep 17 00:00:00 2001 From: John Wright Date: Mon, 22 Feb 2016 20:15:57 -0800 Subject: [PATCH 521/688] add utility overloads of SCRIO.attach, pass base address so that generated c header is correct, and print debug messages/header in hex instead of decimal --- uncore/src/main/scala/scr.scala | 53 +++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 08949ca9..74205fb5 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -8,7 +8,7 @@ import scala.collection.mutable.ArrayBuffer /** Stores a map between SCR file names and address in the SCR file, which can * later be dumped to a header file for the test bench. */ -class SCRFileMap(prefix: String, maxAddress: Int) { +class SCRFileMap(prefix: String, maxAddress: Int, baseAddress: BigInt) { private val addr2name = HashMap.empty[Int, String] private val name2addr = HashMap.empty[String, Int] @@ -18,7 +18,7 @@ class SCRFileMap(prefix: String, maxAddress: Int) { Predef.assert(address < maxAddress, "address too large") addr2name += (address -> name) name2addr += (name -> address) - println(prefix + ": " + address + " -> " + name) + println(prefix + ": %x -> ".format(baseAddress + address) + name) address } @@ -29,7 +29,7 @@ class SCRFileMap(prefix: String, maxAddress: Int) { def as_c_header(): String = { addr2name.map{ case(address, name) => - "#define " + prefix + "__" + name + " " + address + "#define " + prefix + "__" + name + " 0x%x".format(baseAddress + address) }.mkString("\n") + "\n" } } @@ -40,22 +40,57 @@ class SCRIO(map: SCRFileMap)(implicit p: Parameters) extends HtifBundle()(p) { val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, scrDataBits) - def attach(reg: Data, name: String): Data = { + def attach(regs: Seq[Data]): Seq[Data] = { + regs.zipWithIndex.map{ case(reg, i) => attach(reg) } + } + + def attach(regs: Seq[Data], name_base: String): Seq[Data] = { + regs.zipWithIndex.map{ case(reg, i) => attach(reg, name_base + "__" + i) } + } + + def attach(data: Data): Data = attach(data, data.name, false, false) + def attach(data: Data, name: String): Data = attach(data, name, false, false) + def attach(data: Data, addReg: Boolean): Data = attach(data, data.name, false, false) + def attach(data: Data, addReg: Boolean, readOnly: Boolean): Data = attach(data, data.name, readOnly, false) + def attach(data: Data, name: String, addReg: Boolean): Data = attach(data, name, addReg, false) + + def attach(data: Data, name: String, addReg: Boolean, readOnly: Boolean): Data = { val addr = map.allocate(name) - when (wen && (waddr === UInt(addr))) { - reg := wdata + val reg = if(addReg) { Reg(init = Bits(0, width=data.getWidth)) } else { data } + if (!readOnly) { + when (wen && (waddr === UInt(addr))) { + reg := wdata(data.getWidth-1,0) + } + } + require(data.getWidth <= scrDataBits, "SCR Width must be <= %d for %s".format(scrDataBits,name)) + if (data.getWidth < scrDataBits) { + rdata(addr) := Cat(UInt(0, width=(scrDataBits-data.getWidth)),reg) + } else { + rdata(addr) := reg } - rdata(addr) := reg reg } + + def attach(bundle: Bundle): Array[Data] = attach(bundle, "") + + def attach(bundle: Bundle, prefix: String): Array[Data] = { + bundle.flatten.map { x => + if (x._2.dir == OUTPUT) { + attach(x._2, prefix + x._1, false, true) + } else { + attach(x._2, prefix + x._1, true) + } + } + } + def allocate(address: Int, name: String): Unit = { map.allocate(address, name) } } -class SCRFile(prefix: String)(implicit p: Parameters) extends HtifModule()(p) { - val map = new SCRFileMap(prefix, 64) +class SCRFile(prefix: String, baseAddress: BigInt)(implicit p: Parameters) extends HtifModule()(p) { + val map = new SCRFileMap(prefix, 64, baseAddress) AllSCRFiles += map val io = new Bundle { From ebffd69b8ea90b690514dc48d40041c005682b46 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 25 Feb 2016 21:48:32 -0800 Subject: [PATCH 522/688] Provide both __OFFSET and __PADDR for SCR entries This was recently changed to write out physical addresses for SCR file entries, but to bring up the chip we need SCR offsets so we can write the uncore SCR file over HTIF. This changes the map generator to generate both. Without this change things happened to work anyway because the high bits were getting dropped by the SCR file. --- uncore/src/main/scala/scr.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 74205fb5..ecc9317f 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -29,7 +29,8 @@ class SCRFileMap(prefix: String, maxAddress: Int, baseAddress: BigInt) { def as_c_header(): String = { addr2name.map{ case(address, name) => - "#define " + prefix + "__" + name + " 0x%x".format(baseAddress + address) + "#define " + prefix + "__" + name + "__PADDR 0x%x".format(baseAddress + address) + "#define " + prefix + "__" + name + "__OFFSET 0x%x".format(address) }.mkString("\n") + "\n" } } From 6095e7361e37a13032fd973bd0762020c1b671d3 Mon Sep 17 00:00:00 2001 From: John Wright Date: Thu, 25 Feb 2016 10:39:43 -0800 Subject: [PATCH 523/688] Move N_CORES and MMIO_BASE to SCRFile instance in RocketChip --- uncore/src/main/scala/scr.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index ecc9317f..72b4c63d 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -102,8 +102,6 @@ class SCRFile(prefix: String, baseAddress: BigInt)(implicit p: Parameters) exten val scr_rdata = Wire(Vec(io.scr.rdata.size, Bits(width=scrDataBits))) for (i <- 0 until scr_rdata.size) scr_rdata(i) := io.scr.rdata(i) - scr_rdata(0) := UInt(nCores); map.allocate(0, "N_CORES") - scr_rdata(1) := UInt(p(MMIOBase) >> 20); map.allocate(1, "MMIO_BASE") val read_addr = Reg(init = UInt(0, scrAddrBits)) val resp_valid = Reg(init = Bool(false)) From 6d984273b7441b189705a8d2d52df4afb22a7f72 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 29 Feb 2016 15:22:24 -0800 Subject: [PATCH 524/688] finally fix all release assertions ... hopefully --- uncore/src/main/scala/cache.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index edc9b015..da383ee8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -410,10 +410,9 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) tracker.inner.release.bits := io.inner.release.bits tracker.inner.release.valid := io.inner.release.valid } - assert(PopCount(releaseReadys) <= UInt(nReleaseTransactors), + assert(!io.inner.release.valid || PopCount(releaseReadys) <= UInt(1), "At most a single tracker should match for any given Release") - assert(!io.inner.release.valid || - io.inner.release.bits.isVoluntary() || releaseReadys.orR, + assert(!io.inner.release.valid || io.irel().isVoluntary() || releaseReadys.orR, "Non-voluntary release should always have a Tracker waiting for it.") // Wire probe requests and grant reply to clients, finish acks from clients From ab30983aa92057e3b634921b5b582e8524697f76 Mon Sep 17 00:00:00 2001 From: Albert Magyar Date: Mon, 29 Feb 2016 14:48:49 -0800 Subject: [PATCH 525/688] Add support for per-way cache metadata Exposes new parameter field SplitMetadata to determine whether the metadata array in a particular cache is stored in a single SeqMem or with one SeqMem per way. Closes #14 --- uncore/src/main/scala/cache.scala | 36 ++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index da383ee8..5fe47a82 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -20,6 +20,7 @@ case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] case object CacheIdBits extends Field[Int] case object CacheId extends Field[Int] +case object SplitMetadata extends Field[Boolean] trait HasCacheParameters { implicit val p: Parameters @@ -36,6 +37,7 @@ trait HasCacheParameters { val rowBytes = rowBits/8 val rowOffBits = log2Up(rowBytes) val code = p(ECCCode).getOrElse(new IdentityCode) + val hasSplitMetadata = p(SplitMetadata) } abstract class CacheModule(implicit val p: Parameters) extends Module @@ -128,10 +130,10 @@ abstract class Metadata(implicit p: Parameters) extends CacheBundle()(p) { class MetaReadReq(implicit p: Parameters) extends CacheBundle()(p) { val idx = Bits(width = idxBits) + val way_en = Bits(width = nWays) } class MetaWriteReq[T <: Metadata](gen: T)(implicit p: Parameters) extends MetaReadReq()(p) { - val way_en = Bits(width = nWays) val data = gen.cloneType override def cloneType = new MetaWriteReq(gen)(p).asInstanceOf[this.type] } @@ -148,16 +150,31 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits val wmask = Mux(rst, SInt(-1), io.write.bits.way_en.toSInt).toBools + val rmask = Mux(rst, SInt(-1), io.read.bits.way_en.toSInt).toBools when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth - val tag_arr = SeqMem(nSets, Vec(nWays, UInt(width = metabits))) - when (rst || io.write.valid) { - tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) + + if (hasSplitMetadata) { + val tag_arrs = List.fill(nWays){ SeqMem(nSets, UInt(width = metabits)) } + val tag_readout = Wire(Vec(nWays,rstVal.cloneType)) + val tags_vec = Wire(Vec.fill(nWays)(UInt(width = metabits))) + (0 until nWays).foreach { (i) => + when (rst || (io.write.valid && wmask(i))) { + tag_arrs(i).write(waddr, wdata) + } + tags_vec(i) := tag_arrs(i).read(io.read.bits.idx, io.read.valid && rmask(i)) + } + io.resp := io.resp.fromBits(tags_vec.toBits) + } else { + val tag_arr = SeqMem(nSets, Vec(nWays, UInt(width = metabits))) + when (rst || io.write.valid) { + tag_arr.write(waddr, Vec.fill(nWays)(wdata), wmask) + } + val tags = tag_arr.read(io.read.bits.idx, io.read.valid).toBits + io.resp := io.resp.fromBits(tags) } - val tags = tag_arr.read(io.read.bits.idx, io.read.valid).toBits - io.resp := io.resp.fromBits(tags) io.read.ready := !rst && !io.write.valid // so really this could be a 6T RAM io.write.ready := !rst } @@ -268,13 +285,16 @@ class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val meta = Module(new MetadataArray(onReset _)) meta.io.read <> io.read meta.io.write <> io.write - + val way_en_1h = (Vec.fill(nWays){Bool(true)}).toBits + val s1_way_en_1h = RegEnable(way_en_1h, io.read.valid) + meta.io.read.bits.way_en := way_en_1h + val s1_tag = RegEnable(io.read.bits.tag, io.read.valid) val s1_id = RegEnable(io.read.bits.id, io.read.valid) def wayMap[T <: Data](f: Int => T) = Vec((0 until nWays).map(f)) val s1_clk_en = Reg(next = io.read.fire()) val s1_tag_eq_way = wayMap((w: Int) => meta.io.resp(w).tag === s1_tag) - val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid()).toBits + val s1_tag_match_way = wayMap((w: Int) => s1_tag_eq_way(w) && meta.io.resp(w).coh.outer.isValid() && s1_way_en_1h(w).toBool).toBits val s1_idx = RegEnable(io.read.bits.idx, io.read.valid) // deal with stalls? val s2_tag_match_way = RegEnable(s1_tag_match_way, s1_clk_en) val s2_tag_match = s2_tag_match_way.orR From 4acdc6748504aec863d8d4467f40990fdc9f5cc3 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Fri, 26 Feb 2016 18:36:47 -0800 Subject: [PATCH 526/688] Add an assertion in the NastiIOTileLink converter This uses an reorder queue but doesn't check to ensure that the data it fetches from the queue is actually in the queue before using it. It seems that during correct operation this never breaks, but I'm trying to get the backup memory port working again and this assertion fails with it enabled (without the assertion the core just gets a bogus data beat dies). Closes #16 --- uncore/src/main/scala/converters.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 25953952..e626a4a7 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -500,6 +500,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), data = r_aligned_data) + assert(!gnt_arb.io.in(0).valid || roq.io.deq.matches, "NASTI tag error") gnt_arb.io.in(1).valid := io.nasti.b.valid io.nasti.b.ready := gnt_arb.io.in(1).ready From 57370bdf49aca8817fac8f68cef22d65f9ccc045 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Tue, 1 Mar 2016 15:45:35 -0800 Subject: [PATCH 527/688] first and last on HasTileLinkData --- uncore/src/main/scala/cache.scala | 4 +--- uncore/src/main/scala/converters.scala | 14 ++++---------- uncore/src/main/scala/tilelink.scala | 2 ++ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 5fe47a82..338f8f88 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -789,9 +789,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } // Enqueue some metadata information that we'll use to make coherence updates with later - ignt_q.io.enq.valid := Mux(io.iacq().hasMultibeatData(), - io.inner.acquire.fire() && io.iacq().addr_beat === UInt(0), - io.inner.acquire.fire()) + ignt_q.io.enq.valid := io.inner.acquire.fire() && io.iacq().first() ignt_q.io.enq.bits := io.iacq() // Track whether any beats are missing from a PutBlock diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index e626a4a7..4900a253 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -275,12 +275,6 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val out = new ClientUncachedTileLinkIO } - def needsRoqEnq(channel: HasTileLinkData): Bool = - !channel.hasMultibeatData() || channel.addr_beat === UInt(0) - - def needsRoqDeq(channel: HasTileLinkData): Bool = - !channel.hasMultibeatData() || channel.addr_beat === UInt(tlDataBeats - 1) - val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, Some((acq: Acquire) => acq.hasMultibeatData()))) @@ -294,8 +288,8 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val irel = io.in.release.bits val ognt = io.out.grant.bits - val acq_roq_enq = needsRoqEnq(iacq) - val rel_roq_enq = needsRoqEnq(irel) + val acq_roq_enq = iacq.first() + val rel_roq_enq = irel.first() val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready @@ -342,10 +336,10 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { io.out.acquire <> acqArb.io.out - acqRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + acqRoq.io.deq.valid := io.out.grant.fire() && ognt.last() acqRoq.io.deq.tag := ognt.client_xact_id - relRoq.io.deq.valid := io.out.grant.fire() && needsRoqDeq(ognt) + relRoq.io.deq.valid := io.out.grant.fire() && ognt.last() relRoq.io.deq.tag := ognt.client_xact_id val gnt_builtin = acqRoq.io.deq.data diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index f5ac25de..1969f259 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -132,6 +132,8 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool + def first(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(0), Bool(true)) + def last(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(tlDataBeats-1), Bool(true)) } /** An entire cache block of data */ From 7eef3393f11ef5c87e1532fa2171e1436c962a31 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Wed, 2 Mar 2016 10:59:18 -0800 Subject: [PATCH 528/688] fix bug resulting in different g_types on tail beats in L2CacheBank.io.inner.grant --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 338f8f88..fda44abc 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -935,7 +935,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra Mux(ognt_data_done, pending_coh_on_ognt.outer, pending_coh.outer)) - updatePendingCohWhen(io.inner.grant.fire(), pending_coh_on_ignt) + updatePendingCohWhen(io.inner.grant.fire() && io.ignt().last(), pending_coh_on_ignt) // We must wait for as many Finishes as we sent Grants io.inner.finish.ready := state === s_busy From 36f2e6504c7f0c0e57abb0bd5c02c1afb19baa7a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 3 Mar 2016 16:56:53 -0800 Subject: [PATCH 529/688] Fix width of NastiROM rows, preventing out-of-range extraction --- uncore/src/main/scala/NastiROM.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/NastiROM.scala b/uncore/src/main/scala/NastiROM.scala index ba1cef66..2c68340f 100644 --- a/uncore/src/main/scala/NastiROM.scala +++ b/uncore/src/main/scala/NastiROM.scala @@ -20,7 +20,7 @@ class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { val rows = (contents.size + byteWidth - 1)/byteWidth + 1 val rom = Vec.tabulate(rows) { i => val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) - UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) } val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data From 93773a449664d1ba79853cf946a257e94c4e1d0c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Sun, 6 Mar 2016 23:12:16 -0800 Subject: [PATCH 530/688] Refactor L2 transaction trackers to each be capable of processing Voluntary Writebacks. To elide several races between reading and writing the metadata array for different types of transactions, all L2XactTrackers can now sink Voluntary Releases (writebacks from the L1 in the current implementation). These writebacks are merged with the ongoing transaction, and the merging tracker supplies an acknowledgment of the writeback in addition to its ongoing activities. This change involved another refactoring of the control logic for allocating new trackers and routing incoming Acquires and Releases. BroadcastHub uses the new routing logic, but still processes all voluntary releases through the VoluntaryReleaseTracker (not a problem because there are no metadata update races). Closes #18 Closes #20 --- uncore/src/main/scala/broadcast.scala | 73 +++---- uncore/src/main/scala/cache.scala | 278 +++++++++++++++++-------- uncore/src/main/scala/converters.scala | 8 +- uncore/src/main/scala/metadata.scala | 5 +- uncore/src/main/scala/tilelink.scala | 32 ++- uncore/src/main/scala/uncore.scala | 70 ++++++- 6 files changed, 319 insertions(+), 147 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index c3eb7038..ee338e89 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -52,26 +52,22 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = io.inner.acquire.fire() && io.iacq().hasData() + val sdq_enq = trackerList.map(_.io.alloc.iacq).reduce(_||_) && + io.inner.acquire.fire() && + io.iacq().hasData() when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) - val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits - val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits - val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits - val acquire_idx = Mux(acquireMatches.orR, - PriorityEncoder(acquireMatches), - PriorityEncoder(acquireReadys)) - - val block_acquires = acquireConflicts.orR || !sdq_rdy - io.inner.acquire.ready := acquireReadys.orR && !block_acquires - trackerAcquireIOs.zipWithIndex.foreach { - case(tracker, i) => - tracker.bits := io.inner.acquire.bits - tracker.bits.data := DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) + val sdqLoc = List.fill(nTransactors) { + DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits } + doInputRoutingWithAllocation( + io.inner.acquire, + trackerList.map(_.io.inner.acquire), + trackerList.map(_.io.matches.iacq), + trackerList.map(_.io.alloc.iacq), + Some(sdqLoc), + Some(sdq_rdy)) // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() @@ -81,17 +77,16 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } // Handle releases, which might be voluntary and might have data - val trackerReleaseIOs = trackerList.map(_.io.inner.release) - val releaseReadys = Vec(trackerReleaseIOs.map(_.ready)).toBits - io.inner.release.ready := releaseReadys.orR - trackerReleaseIOs.zipWithIndex.foreach { - case(tracker, i) => - tracker.valid := io.inner.release.valid - tracker.bits := io.inner.release.bits - tracker.bits.data := DataQueueLocation(rel_data_cnt, - (if(i < nReleaseTransactors) inVolWBQueue - else inClientReleaseQueue)).toBits - } + val vwbqLoc = (0 until nTransactors).map(i => + (DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits)) + doInputRoutingWithAllocation( + io.inner.release, + trackerList.map(_.io.inner.release), + trackerList.map(_.io.matches.irel), + trackerList.map(_.io.alloc.irel), + Some(vwbqLoc)) // Wire probe requests and grant reply to clients, finish acks from clients // Note that we bypass the Grant data subbundles @@ -162,12 +157,12 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) // Send an acknowledgement io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid - io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.makeGrant(xact) when(io.inner.grant.fire()) { pending_ignt := Bool(false) } io.outer.grant.ready := state === s_busy && io.inner.grant.ready // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.fire()) { + when(state === s_idle && io.inner.release.valid && io.alloc.irel) { xact := io.irel() when(io.irel().hasMultibeatData()) { pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) @@ -181,8 +176,9 @@ class BroadcastVoluntaryReleaseTracker(trackerId: Int) when(state === s_busy && all_pending_done) { state := s_idle } // These IOs are used for routing in the parent - io.has_acquire_match := Bool(false) - io.has_acquire_conflict := Bool(false) + io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) + io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && io.irel().isVoluntary() + io.matches.oprb := Bool(false) // Checks for illegal behavior assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), @@ -221,11 +217,10 @@ class BroadcastAcquireTracker(trackerId: Int) val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() val subblock_type = xact.isSubBlockType() - io.has_acquire_conflict := xact.conflicts(io.iacq()) && - (state =/= s_idle) && - !collect_iacq_data - io.has_acquire_match := xact.conflicts(io.iacq()) && - collect_iacq_data + // These IOs are used for routing in the parent + io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) + io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && !io.irel().isVoluntary() + io.matches.oprb := Bool(false) val outerParams = p.alterPartial({ case TLId => outerTLId }) @@ -286,7 +281,7 @@ class BroadcastAcquireTracker(trackerId: Int) io.iacq().client_xact_id =/= xact.client_xact_id), "AcquireTracker accepted data beat from different client transaction than initial request.") - assert(!(state === s_idle && io.inner.acquire.fire() && + assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq && io.iacq().hasMultibeatData() && io.iacq().addr_beat =/= UInt(0)), "AcquireTracker initialized with a tail data beat.") @@ -309,7 +304,7 @@ class BroadcastAcquireTracker(trackerId: Int) switch (state) { is(s_idle) { io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { + when(io.inner.acquire.valid && io.alloc.iacq) { xact := io.iacq() xact.data_buffer(UInt(0)) := io.iacq().data xact.wmask_buffer(UInt(0)) := io.iacq().wmask() @@ -333,7 +328,7 @@ class BroadcastAcquireTracker(trackerId: Int) } // Handle releases, which may have data to be written back - val matches = xact.conflicts(io.irel()) && !io.irel().isVoluntary() + val matches = io.matches.irel io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches when(io.inner.release.valid && matches) { when(io.irel().hasData()) { diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index fda44abc..d3c704c0 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -186,6 +186,8 @@ trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgen val idxLSB = cacheIdBits val idxMSB = idxLSB + idxBits - 1 val tagLSB = idxLSB + idxBits + def inSameSet(addr1: UInt, addr2: UInt): Bool = addr1(idxMSB,idxLSB) === addr2(idxMSB,idxLSB) + def haveSameTag(addr1: UInt, addr2: UInt): Bool = addr1 >> UInt(tagLSB) === addr2 >> UInt(tagLSB) //val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats @@ -393,8 +395,11 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) val io = new TSHRFileIO // Create TSHRs for outstanding transactions - val trackerList = (0 until nReleaseTransactors).map(id => Module(new L2VoluntaryReleaseTracker(id))) ++ - (nReleaseTransactors until nTransactors).map(id => Module(new L2AcquireTracker(id))) + val trackerList = + (0 until nReleaseTransactors).map(id => + Module(new L2VoluntaryReleaseTracker(id))) ++ + (nReleaseTransactors until nTransactors).map(id => + Module(new L2AcquireTracker(id))) // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors)) @@ -406,38 +411,33 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) (trackerList.map(_.io.incoherent) :+ wb.io.incoherent) foreach { _ := io.incoherent } // Handle acquire transaction initiation - val trackerAcquireIOs = trackerList.map(_.io.inner.acquire) - val acquireConflicts = Vec(trackerList.map(_.io.has_acquire_conflict)).toBits - val acquireMatches = Vec(trackerList.map(_.io.has_acquire_match)).toBits - val acquireReadys = Vec(trackerAcquireIOs.map(_.ready)).toBits - val acquire_idx = Mux(acquireMatches.orR, - OHToUInt(acquireMatches), - PriorityEncoder(acquireReadys)) - val block_acquires = acquireConflicts.orR - io.inner.acquire.ready := acquireReadys.orR && !block_acquires - trackerAcquireIOs.zipWithIndex.foreach { - case(tracker, i) => - tracker.bits := io.inner.acquire.bits - tracker.valid := io.inner.acquire.valid && !block_acquires && (acquire_idx === UInt(i)) - } - assert(PopCount(acquireMatches) <= UInt(1), - "At most a single tracker should match for any given Acquire") + val irel_vs_iacq_conflict = + io.inner.acquire.valid && + io.inner.release.valid && + inSameSet(io.inner.acquire.bits.addr_block, io.inner.release.bits.addr_block) + doInputRoutingWithAllocation( + io.inner.acquire, + trackerList.map(_.io.inner.acquire), + trackerList.map(_.io.matches.iacq), + trackerList.map(_.io.alloc.iacq), + allocOverride = !irel_vs_iacq_conflict) + + assert(PopCount(trackerList.map(_.io.alloc.iacq)) <= UInt(1), + "At most a single tracker should now be allocated for any given Acquire") // Wire releases from clients - val releaseReadys = Vec(trackerAndWbIOs.map(_.inner.release.ready)).toBits - io.inner.release.ready := releaseReadys.orR - trackerAndWbIOs foreach { tracker => - tracker.inner.release.bits := io.inner.release.bits - tracker.inner.release.valid := io.inner.release.valid - } - assert(!io.inner.release.valid || PopCount(releaseReadys) <= UInt(1), - "At most a single tracker should match for any given Release") - assert(!io.inner.release.valid || io.irel().isVoluntary() || releaseReadys.orR, - "Non-voluntary release should always have a Tracker waiting for it.") + doInputRoutingWithAllocation( + io.inner.release, + trackerAndWbIOs.map(_.inner.release), + trackerAndWbIOs.map(_.matches.irel), + trackerAndWbIOs.map(_.alloc.irel)) + + assert(PopCount(trackerAndWbIOs.map(_.alloc.irel)) <= UInt(1), + "At most a single tracker should now be allocated for any given Release") // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe) :+ wb.io.inner.probe) - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant) :+ wb.io.inner.grant) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) // Create an arbiter for the one memory port @@ -532,6 +532,11 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends !(pending_writes.orR || pending_ignt) + // These IOs are used for routing in the parent + io.matches.iacq := (state =/= s_idle) && inSameSet(io.iacq().addr_block, xact.addr_block) + io.matches.irel := (state =/= s_idle) && io.irel().conflicts(xact) + io.matches.oprb := (state =/= s_idle) && io.oprb().conflicts(xact) + // Accept a voluntary Release (and any further beats of data) pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR @@ -557,7 +562,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends // Send an acknowledgement io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels - io.inner.grant.bits := coh.inner.makeGrant(xact, UInt(trackerId)) + io.inner.grant.bits := coh.inner.makeGrant(xact) when(io.inner.grant.fire()) { pending_ignt := Bool(false) } // End a transaction by updating the block metadata @@ -572,7 +577,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends xact_old_meta.coh.outer) // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.fire()) { + when(state === s_idle && io.inner.release.valid && io.alloc.irel) { xact := io.irel() when(io.irel().hasMultibeatData()) { pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) @@ -592,10 +597,6 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends when(state === s_busy && all_pending_done) { state := s_meta_write } when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - // These IOs are used for routing in the parent - io.has_acquire_match := Bool(false) - io.has_acquire_conflict := Bool(false) - // Checks for illegal behavior assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") @@ -624,6 +625,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val xact_op_code = Reg{ UInt() } val xact_addr_byte = Reg{ UInt() } val xact_op_size = Reg{ UInt() } + val xact_vol_irel_r_type = Reg{ io.irel().r_type } + val xact_vol_irel_src = Reg{ io.irel().client_id } + val xact_vol_irel_client_xact_id = Reg{ io.irel().client_xact_id } // Miss queue holds transaction metadata used to make grants val ignt_q = Module(new Queue( @@ -634,12 +638,27 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val xact = ignt_q.io.deq.bits val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) val xact_addr_tag = xact_addr_block >> UInt(tagLSB) - + val xact_vol_irel = Release( + src = xact_vol_irel_src, + voluntary = Bool(true), + r_type = xact_vol_irel_r_type, + client_xact_id = xact_vol_irel_client_xact_id, + addr_block = xact_addr_block) + (p.alterPartial({ case TLId => p(InnerTLId) })) + // Counters and scoreboard tracking progress made on processing this transaction val pending_irels = connectTwoWayBeatCounter( max = io.inner.tlNCachingClients, up = io.inner.probe, - down = io.inner.release)._1 + down = io.inner.release, + trackDown = (r: Release) => !r.isVoluntary())._1 + + val pending_vol_ignt = connectTwoWayBeatCounter( + max = 1, + up = io.inner.release, + down = io.inner.grant, + trackUp = (r: Release) => r.isVoluntary(), + trackDown = (g: Grant) => g.isVoluntary())._1 val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = connectTwoWayBeatCounter( @@ -656,7 +675,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra max = nSecondaryMisses, up = io.inner.grant, down = io.inner.finish, - track = (g: Grant) => g.requiresAck())._1 + trackUp = (g: Grant) => g.requiresAck())._1 val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) @@ -674,6 +693,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_puts.orR || pending_ognt || ignt_q.io.count > UInt(0) || + pending_vol_ignt || //pending_meta_write || // Has own state: s_meta_write pending_ifins) @@ -726,7 +746,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), amoalu.io.out << (xact_amo_shift_bytes << 3), new_data) - wmask_buffer(beat) := ~UInt(0, wmask_buffer.head.getWidth) when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } } @@ -764,6 +783,15 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra xact_allocate } + // These IOs are used for routing in the parent + val iacq_in_same_set = inSameSet(xact_addr_idx, io.iacq().addr_block) + val irel_in_same_set = inSameSet(xact_addr_idx,io.irel().addr_block) + val before_wb_alloc = Vec(s_meta_read, s_meta_resp, s_wb_req).contains(state) + io.matches.iacq := (state =/= s_idle) && iacq_in_same_set + io.matches.irel := (state =/= s_idle) && + Mux(before_wb_alloc, irel_in_same_set, io.irel().conflicts(xact_addr_block)) + io.matches.oprb := Bool(false) //TODO + // Actual transaction processing logic begins here: // // First, take care of accpeting new acquires or secondary misses @@ -774,14 +802,19 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra !io.outer.grant.fire() && !io.data.resp.valid && ignt_q.io.enq.ready && ignt_q.io.deq.valid - val iacq_same_xact = xact.client_xact_id === io.iacq().client_xact_id && - xact.hasMultibeatData() && ignt_q.io.deq.valid && - pending_puts(io.iacq().addr_beat) - io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + val iacq_same_xact = xact.client_xact_id === io.iacq().client_xact_id && + xact.hasMultibeatData() && + ignt_q.io.deq.valid && // i.e. state =/= s_idle + pending_puts(io.iacq().addr_beat) + + val iacq_accepted = io.inner.acquire.fire() && + (io.alloc.iacq || iacq_can_merge || iacq_same_xact) + + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact // Handling of primary and secondary misses' data and write mask merging - when(io.inner.acquire.fire() && io.iacq().hasData()) { + when(iacq_accepted && io.iacq().hasData()) { val beat = io.iacq().addr_beat val full = FillInterleaved(8, io.iacq().wmask()) data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) @@ -789,7 +822,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } // Enqueue some metadata information that we'll use to make coherence updates with later - ignt_q.io.enq.valid := io.inner.acquire.fire() && io.iacq().first() + ignt_q.io.enq.valid := iacq_accepted && io.iacq().first() ignt_q.io.enq.bits := io.iacq() // Track whether any beats are missing from a PutBlock @@ -818,17 +851,35 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact, xact_addr_block) // Handle incoming releases from clients, which may reduce sharer counts - // and/or write back dirty data - io.inner.release.ready := state === s_inner_probe && - io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() + // and/or write back dirty data, and may be unexpected voluntary releases + val irel_can_merge = io.irel().conflicts(xact_addr_block) && + io.irel().isVoluntary() && + !Vec(s_idle, s_meta_read, s_meta_resp, s_meta_write).contains(state) && + !all_pending_done && + !io.outer.grant.fire() && + !io.inner.grant.fire() && + !pending_vol_ignt + + val irel_same_xact = io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() && + state === s_inner_probe + + val irel_accepted = io.inner.release.fire() && + (io.alloc.irel || irel_can_merge || irel_same_xact) + + io.inner.release.ready := irel_can_merge || irel_same_xact val pending_coh_on_irel = HierarchicalMetadata( pending_coh.inner.onRelease(io.irel()), // Drop sharer - Mux(io.irel().hasData(), // Dirty writeback + Mux(io.irel().hasData(), // Dirty writeback pending_coh.outer.onHit(M_XWR), pending_coh.outer)) updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) + when(io.inner.release.fire() && irel_can_merge) { + xact_vol_irel_r_type := io.irel().r_type + xact_vol_irel_src := io.irel().client_id + xact_vol_irel_client_xact_id := io.irel().client_xact_id + } // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // @@ -902,7 +953,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact_addr_idx io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := wmask_buffer(curr_write_beat) + io.data.write.bits.wmask := SInt(-1) // Always writes a full beat io.data.write.bits.data := data_buffer(curr_write_beat) // soon as the data is released, granted, put, or read from the cache @@ -912,22 +963,28 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra addPendingBitInternal(io.data.resp) // We can issue a grant for a pending write once all data is // received and committed to the data array or outer memory - val ignt_ack_ready = !(state === s_idle || state === s_meta_read || - pending_puts.orR || pending_writes.orR || pending_ognt) + val ignt_ack_ready = !(state === s_idle || + state === s_meta_read || + pending_puts.orR || + pending_writes.orR || + pending_ognt) - ignt_q.io.deq.ready := ignt_data_done - io.inner.grant.valid := state === s_busy && - ignt_q.io.deq.valid && - Mux(io.ignt().hasData(), - ignt_data_ready(ignt_data_idx), - ignt_ack_ready) + ignt_q.io.deq.ready := !pending_vol_ignt && ignt_data_done + io.inner.grant.valid := pending_vol_ignt || + (state === s_busy && + ignt_q.io.deq.valid && + Mux(io.ignt().hasData(), + ignt_data_ready(ignt_data_idx), + ignt_ack_ready)) // Make the Grant message using the data stored in the secondary miss queue - io.inner.grant.bits := pending_coh.inner.makeGrant( - sec = ignt_q.io.deq.bits, - manager_xact_id = UInt(trackerId), - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) + val grant_from_acquire = pending_coh.inner.makeGrant( + sec = ignt_q.io.deq.bits, + manager_xact_id = UInt(trackerId), + data = Mux(xact.is(Acquire.putAtomicType), + amo_result, + data_buffer(ignt_data_idx))) + val grant_from_release = pending_coh.inner.makeGrant(xact_vol_irel) + io.inner.grant.bits := Mux(pending_vol_ignt, grant_from_release, grant_from_acquire) io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter val pending_coh_on_ignt = HierarchicalMetadata( @@ -949,7 +1006,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.meta.write.bits.data.coh := pending_coh // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.acquire.valid) { + when(state === s_idle && io.inner.acquire.valid && io.alloc.iacq) { xact_addr_block := io.iacq().addr_block xact_allocate := io.iacq().allocate() xact_amo_shift_bytes := io.iacq().amo_shift_bytes() @@ -1027,12 +1084,6 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra state := Mux(pending_meta_write, s_meta_write, s_idle) } when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - - // These IOs are used for routing in the parent - val in_same_set = xact_addr_idx === io.iacq().addr_block(idxMSB,idxLSB) - io.has_acquire_match := iacq_can_merge || iacq_same_xact - io.has_acquire_conflict := in_same_set && (state =/= s_idle) && !io.has_acquire_match - //TODO: relax from in_same_set to xact.conflicts(io.iacq())? } class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { @@ -1061,17 +1112,47 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac val xact = Reg(new L2WritebackReq) val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val xact_addr_block = Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) + val xact_vol_irel_r_type = Reg{ io.irel().r_type } + val xact_vol_irel_src = Reg{ io.irel().client_id } + val xact_vol_irel_client_xact_id = Reg{ io.irel().client_xact_id } + + val xact_addr_block = Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) + val xact_vol_irel = Release( + src = xact_vol_irel_src, + voluntary = Bool(true), + r_type = xact_vol_irel_r_type, + client_xact_id = xact_vol_irel_client_xact_id, + addr_block = xact_addr_block) + + val pending_irels = connectTwoWayBeatCounter( + max = io.inner.tlNCachingClients, + up = io.inner.probe, + down = io.inner.release, + trackDown = (r: Release) => !r.isVoluntary())._1 + + val pending_vol_ignt = connectTwoWayBeatCounter( + max = 1, + up = io.inner.release, + down = io.inner.grant, + trackUp = (r: Release) => r.isVoluntary(), + trackDown = (g: Grant) => g.isVoluntary())._1 - val pending_irels = - connectTwoWayBeatCounter(max = io.inner.tlNCachingClients, up = io.inner.probe, down = io.inner.release)._1 val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = - connectTwoWayBeatCounter(max = 1, up = io.outer.release, down = io.outer.grant) + connectTwoWayBeatCounter( + max = 1, + up = io.outer.release, + down = io.outer.grant) + val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + // These IOs are used for routing in the parent + io.matches.iacq := (state =/= s_idle) && io.iacq().conflicts(xact_addr_block) + io.matches.irel := (state =/= s_idle) && io.irel().conflicts(xact_addr_block) + io.matches.oprb := (state =/= s_idle) && io.oprb().conflicts(xact_addr_block) + // Start the writeback sub-transaction io.wb.req.ready := state === s_idle @@ -1083,17 +1164,32 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data - val inner_coh_on_irel = xact.coh.inner.onRelease(io.irel()) - val outer_coh_on_irel = xact.coh.outer.onHit(M_XWR) - io.inner.release.ready := (state === s_inner_probe || state === s_busy) && - io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() + val irel_can_merge = io.irel().conflicts(xact_addr_block) && + io.irel().isVoluntary() && + state === s_inner_probe && + !pending_vol_ignt + + val irel_same_xact = io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() && + state === s_inner_probe + + val irel_accepted = io.inner.release.fire() && + (io.alloc.irel || irel_can_merge || irel_same_xact) + + io.inner.release.ready := irel_can_merge || irel_same_xact + val pending_coh_on_irel = HierarchicalMetadata( + xact.coh.inner.onRelease(io.irel()), // Drop sharer + Mux(io.irel().hasData(), // Dirty writeback + xact.coh.outer.onHit(M_XWR), + xact.coh.outer)) when(io.inner.release.fire()) { - xact.coh.inner := inner_coh_on_irel - data_buffer(io.inner.release.bits.addr_beat) := io.inner.release.bits.data - } - when(io.inner.release.valid && io.irel().conflicts(xact_addr_block) && io.irel().hasData()) { - xact.coh.outer := outer_coh_on_irel // must writeback dirty data supplied by any matching release, even voluntary ones + xact.coh := pending_coh_on_irel + when(io.irel().hasData()) { data_buffer(io.irel().addr_beat) := io.irel().data } + when(irel_can_merge) { + xact_vol_irel_r_type := io.irel().r_type + xact_vol_irel_src := io.irel().client_id + xact_vol_irel_client_xact_id := io.irel().client_xact_id + } } // If a release didn't write back data, have to read it from data array @@ -1126,6 +1222,10 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac addr_beat = orel_data_idx, data = data_buffer(orel_data_idx)) + // Ack a voluntary release if we got one + io.inner.grant.valid := pending_vol_ignt + io.inner.grant.bits := xact.coh.inner.makeGrant(xact_vol_irel) + // Wait for an acknowledgement io.outer.grant.ready := state === s_outer_grant @@ -1144,7 +1244,7 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac pending_orel_data := UInt(0) state := Mux(needs_inner_probes, s_inner_probe, s_busy) } - when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { + when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels || pending_vol_ignt)) { state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) } when(state === s_busy && orel_data_done) { @@ -1152,8 +1252,4 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac } when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } when(state === s_wb_resp ) { state := s_idle } - - // These IOs are used for routing in the parent - io.has_acquire_match := Bool(false) - io.has_acquire_conflict := Bool(false) } diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 4900a253..5b510360 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -199,6 +199,7 @@ object ManagerTileLinkHeaderCreator { */ trait HasDataBeatCounters { type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasId = TileLinkChannel with HasClientId /** Returns the current count on this channel and when a message is done * @param inc increment the counter (usually .valid or .fire()) @@ -259,11 +260,12 @@ trait HasDataBeatCounters { up: DecoupledIO[T], down: DecoupledIO[S], beat: UInt = UInt(0), - track: T => Bool = (t: T) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { + trackUp: T => Bool = (t: T) => Bool(true), + trackDown: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) - val do_inc = up_done && track(up.bits) - val do_dec = down_done + val do_inc = up_done && trackUp(up.bits) + val do_dec = down_done && trackDown(down.bits) val cnt = TwoWayCounter(do_inc, do_dec, max) (cnt > UInt(0), up_idx, up_done, down_idx, down_done) } diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index 2819745c..ede9e397 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -218,15 +218,14 @@ class ManagerMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { /** Construct an appropriate [[uncore.GrantToDst]] to acknowledge an [[uncore.Release]] * * @param rel Release message being acknowledged by this Grant - * @param manager_xact_id manager's transaction id */ - def makeGrant(rel: ReleaseMetadata with HasClientId, manager_xact_id: UInt): GrantToDst = + def makeGrant(rel: ReleaseMetadata with HasClientId): GrantToDst = Grant( dst = rel.client_id, is_builtin_type = Bool(true), g_type = Grant.voluntaryAckType, client_xact_id = rel.client_xact_id, - manager_xact_id = manager_xact_id)(p) + manager_xact_id = UInt(0))(p) /** Construct an appropriate [[uncore.GrantToDst]] to respond to an [[uncore.Acquire]] * diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 1969f259..b8f1512c 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -227,7 +227,11 @@ trait HasProbeType extends HasTileLinkParameters { def hasMultibeatData(dummy: Int = 0) = Bool(false) } -trait HasReleaseType extends HasTileLinkParameters { +trait MightBeVoluntary { + def isVoluntary(dummy: Int = 0): Bool +} + +trait HasReleaseType extends HasTileLinkParameters with MightBeVoluntary { val voluntary = Bool() val r_type = UInt(width = tlCoh.releaseTypeWidth) @@ -239,7 +243,7 @@ trait HasReleaseType extends HasTileLinkParameters { def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } -trait HasGrantType extends HasTileLinkParameters { +trait HasGrantType extends HasTileLinkParameters with MightBeVoluntary { val is_builtin_type = Bool() val g_type = UInt(width = tlGrantTypeBits) @@ -720,8 +724,8 @@ object Release { r_type: UInt, client_xact_id: UInt, addr_block: UInt, - addr_beat: UInt = UInt(0), - data: UInt = UInt(0)) + addr_beat: UInt, + data: UInt) (implicit p: Parameters): Release = { val rel = Wire(new Release) rel.r_type := r_type @@ -732,6 +736,26 @@ object Release { rel.voluntary := voluntary rel } + + def apply( + src: UInt, + voluntary: Bool, + r_type: UInt, + client_xact_id: UInt, + addr_block: UInt, + addr_beat: UInt = UInt(0), + data: UInt = UInt(0)) + (implicit p: Parameters): ReleaseFromSrc = { + val rel = Wire(new ReleaseFromSrc) + rel.client_id := src + rel.voluntary := voluntary + rel.r_type := r_type + rel.client_xact_id := client_xact_id + rel.addr_block := addr_block + rel.addr_beat := addr_beat + rel.data := data + rel + } } /** The Grant channel is used to refill data or grant permissions requested of the diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index c90adeeb..6444f54f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -59,6 +59,38 @@ trait HasCoherenceAgentWiringHelpers { outs.zipWithIndex.map { case (o,i) => o.valid := in.valid && idx === UInt(i) } in.ready := Vec(outs.map(_.ready)).read(idx) } + + /** Broadcasts valid messages on this channel to all trackers, + * but includes logic to allocate a new tracker in the case where + * no previously allocated tracker matches the new req's addr. + * + * When a match is reported, if ready is high the new transaction + * is merged; when ready is low the transaction is being blocked. + * When no match is reported, any high readys are presumed to be + * from trackers that are available for allocation, and one is + * assigned via alloc based on priority; f no readys are high then + * all trackers are busy with other transactions. + */ + def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData]( + in: DecoupledIO[T], + outs: Seq[DecoupledIO[T]], + matches: Seq[Bool], + allocs: Seq[Bool], + dataOverrides: Option[Seq[UInt]] = None, + allocOverride: Option[Bool] = None) { + val ready_bits = Vec(outs.map(_.ready)).toBits + val alloc_bits = PriorityEncoderOH(ready_bits) + val match_bits = Vec(matches).toBits + val no_matches = !match_bits.orR + val do_alloc = allocOverride.getOrElse(Bool(true)) + in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && do_alloc + outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => + out.valid := in.valid + out.bits := in.bits + dataOverrides foreach { d => out.bits.data := d(i) } + a := alloc_bits(i) & no_matches & do_alloc + } + } } trait HasInnerTLIO extends HasCoherenceAgentParameters { @@ -114,24 +146,39 @@ abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends Cohere def incoherent = io.incoherent } -trait HasTrackerConflictIO extends Bundle { - val has_acquire_conflict = Bool(OUTPUT) - val has_acquire_match = Bool(OUTPUT) +trait HasTrackerAllocationIO extends Bundle { + val matches = new Bundle { + val iacq = Bool(OUTPUT) + val irel = Bool(OUTPUT) + val oprb = Bool(OUTPUT) + } + val alloc = new Bundle { + val iacq = Bool(INPUT) + val irel = Bool(INPUT) + val oprb = Bool(INPUT) + } } class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) - with HasTrackerConflictIO + with HasTrackerAllocationIO class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p) - with HasTrackerConflictIO + with HasTrackerAllocationIO abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p) with HasDataBeatCounters { def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt = + Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id) + + def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt = + ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id) + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) @@ -151,8 +198,17 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - def dropPendingBitAtDest(in: DecoupledIO[ProbeToDst]): UInt = - ~Fill(in.bits.tlNCachingClients, in.fire()) | ~UIntToOH(in.bits.client_id) + def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt = + dropPendingBitWhenId(in.fire(), in.bits) + + def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = + dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) + + def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt = + addPendingBitWhenId(in.fire(), in.bits) + + def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = + addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) def pinAllReadyValidLow[T <: Data](b: Bundle) { b.elements.foreach { From c28d115b306e6a78870342896177ce82a85a2476 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 10 Mar 2016 15:50:44 -0800 Subject: [PATCH 531/688] Chisel3 compatibility fix --- uncore/src/main/scala/util.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 121ea68f..12316458 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -75,7 +75,7 @@ class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) ex val rbits = Reg{io.in.bits} val active = Reg(init=Bool(false)) - val shifter = Vec(n, Bits(width = narrowWidth)) + val shifter = Wire(Vec(n, Bits(width = narrowWidth))) (0 until n).foreach { i => shifter(i) := rbits.data((i+1)*narrowWidth-1,i*narrowWidth) } From 2eafc4c8f3d494bc7c9e177c7f0fe71599bda9ab Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 10 Mar 2016 17:19:42 -0800 Subject: [PATCH 532/688] Extend AMOALU to support RV32 --- uncore/src/main/scala/amoalu.scala | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala index 6f083e9f..d0eaecd7 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/amoalu.scala @@ -53,7 +53,7 @@ class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool, maxSize: Int) { class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends CacheModule()(p) { val operandBits = p(AmoAluOperandBits) - require(operandBits == 64) + require(operandBits == 32 || operandBits == 64) val io = new Bundle { val addr = Bits(INPUT, blockOffBits) val cmd = Bits(INPUT, M_SZ) @@ -74,16 +74,24 @@ class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends Cach val word = io.typ === MT_W || io.typ === MT_WU || // Logic minimization: io.typ === MT_B || io.typ === MT_BU - val mask = ~UInt(0,64) ^ (io.addr(2) << 31) - val adder_out = (io.lhs & mask).toUInt + (rhs & mask) + val adder_out = + if (operandBits == 32) io.lhs + rhs + else { + val mask = ~UInt(0,64) ^ (io.addr(2) << 31) + (io.lhs & mask).toUInt + (rhs & mask) + } - val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) - val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) - val lt_lo = io.lhs(31,0) < rhs(31,0) - val lt_hi = io.lhs(63,32) < rhs(63,32) - val eq_hi = io.lhs(63,32) === rhs(63,32) - val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) - val less = Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + val less = + if (operandBits == 32) Mux(io.lhs(31) === rhs(31), io.lhs < rhs, Mux(sgned, io.lhs(31), io.rhs(31))) + else { + val cmp_lhs = Mux(word && !io.addr(2), io.lhs(31), io.lhs(63)) + val cmp_rhs = Mux(word && !io.addr(2), rhs(31), rhs(63)) + val lt_lo = io.lhs(31,0) < rhs(31,0) + val lt_hi = io.lhs(63,32) < rhs(63,32) + val eq_hi = io.lhs(63,32) === rhs(63,32) + val lt = Mux(word, Mux(io.addr(2), lt_hi, lt_lo), lt_hi || eq_hi && lt_lo) + Mux(cmp_lhs === cmp_rhs, lt, Mux(sgned, cmp_lhs, cmp_rhs)) + } val out = Mux(io.cmd === M_XA_ADD, adder_out, Mux(io.cmd === M_XA_AND, io.lhs & rhs, From 8c7e29eacd5a05dec569decf2a614e4f3bd458b2 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Sun, 6 Mar 2016 16:32:14 -0800 Subject: [PATCH 533/688] Avoid generating 0-width UInts Chisel3 requires a 1-bit width to represent UInt(0). --- uncore/src/main/scala/cache.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d3c704c0..e555b003 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1116,7 +1116,10 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac val xact_vol_irel_src = Reg{ io.irel().client_id } val xact_vol_irel_client_xact_id = Reg{ io.irel().client_xact_id } - val xact_addr_block = Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) + val xact_addr_block = if (cacheIdBits == 0) + Cat(xact.tag, xact.idx) + else + Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) val xact_vol_irel = Release( src = xact_vol_irel_src, voluntary = Bool(true), From e2185d40f6c239a3ba8752fecefbe8bde0e86408 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Sun, 6 Mar 2016 16:32:59 -0800 Subject: [PATCH 534/688] Avoid right-shift by larger that the bit width FIRRTL bails out on this. There's an outstanding bug, this is just a workaround. See https://github.com/ucb-bar/firrtl/issues/69 --- uncore/src/main/scala/tilelink.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index b8f1512c..c4850c62 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1110,8 +1110,15 @@ abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) exte trait AppendsArbiterId extends TileLinkArbiterLike { def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerSourcedWithId) = - in.client_xact_id >> log2Up(arbN) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = { + /* This shouldn't be necessary, but Chisel3 doesn't emit correct Verilog + * when right shifting by too many bits. See + * https://github.com/ucb-bar/firrtl/issues/69 */ + if (in.client_xact_id.getWidth > log2Up(arbN)) + in.client_xact_id >> log2Up(arbN) + else + UInt(0) + } def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt } From 67e711844a30c465b9cb421a6cfa0b3c92c8e6f2 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 10 Mar 2016 17:35:22 -0800 Subject: [PATCH 535/688] index extraction bug --- uncore/src/main/scala/cache.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e555b003..d03ef445 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -784,8 +784,8 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } // These IOs are used for routing in the parent - val iacq_in_same_set = inSameSet(xact_addr_idx, io.iacq().addr_block) - val irel_in_same_set = inSameSet(xact_addr_idx,io.irel().addr_block) + val iacq_in_same_set = inSameSet(xact_addr_block, io.iacq().addr_block) + val irel_in_same_set = inSameSet(xact_addr_block, io.irel().addr_block) val before_wb_alloc = Vec(s_meta_read, s_meta_resp, s_wb_req).contains(state) io.matches.iacq := (state =/= s_idle) && iacq_in_same_set io.matches.irel := (state =/= s_idle) && From 8a47c3f346c82a13bfa685639bd52b6b07137414 Mon Sep 17 00:00:00 2001 From: Eric Love Date: Fri, 11 Mar 2016 16:54:56 -0800 Subject: [PATCH 536/688] Make sure there's enough xact id bits --- uncore/src/main/scala/rtc.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 6c3f83a4..f22121a9 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -51,6 +51,7 @@ class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule id = coreId, addr = addrTable(coreId), size = UInt(log2Up(csrDataBytes))) + require(p(MIFMasterTagBits) >= log2Up(nCores)) io.w.valid := sending_data io.w.bits := NastiWriteDataChannel(data = rtc) From 49d82864bf4c4a9c9dcb28f7c6ddee797ab068bf Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 17 Mar 2016 12:31:18 -0700 Subject: [PATCH 537/688] Fix StoreDataQueue allocation bug in BroadcastHub Closes #27 --- uncore/src/main/scala/broadcast.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index ee338e89..d2a6fe65 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -52,9 +52,11 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) val sdq_val = Reg(init=Bits(0, sdqDepth)) val sdq_alloc_id = PriorityEncoder(~sdq_val) val sdq_rdy = !sdq_val.andR - val sdq_enq = trackerList.map(_.io.alloc.iacq).reduce(_||_) && - io.inner.acquire.fire() && - io.iacq().hasData() + val sdq_enq = trackerList.map( t => + (t.io.alloc.iacq || t.io.matches.iacq) && + t.io.inner.acquire.fire() && + t.io.iacq().hasData() + ).reduce(_||_) when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation From 5f3d3a0b2d0c59ad83c530a44ef30b490f4f60cc Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 17 Mar 2016 16:42:40 -0700 Subject: [PATCH 538/688] Bugfix for probe flags in L2BroadcastHub Closes #25 --- uncore/src/main/scala/broadcast.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index d2a6fe65..d47e6e36 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -202,7 +202,9 @@ class BroadcastAcquireTracker(trackerId: Int) val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) val curr_p_id = PriorityEncoder(pending_probes) - val mask_self = coh.full().bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe()) + val mask_self = SInt(-1, width = io.inner.tlNCachingClients) + .toUInt + .bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe()) val mask_incoherent = mask_self & ~io.incoherent.toBits val collect_iacq_data = Reg(init=Bool(false)) From c13b8d243db0dc0b9cefa8bc8f3395491e32481e Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Thu, 17 Mar 2016 18:32:35 -0700 Subject: [PATCH 539/688] BroadcastHub race on allocating VolWBs vs Acquires --- uncore/src/main/scala/broadcast.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index d47e6e36..520c68af 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -60,6 +60,10 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } // Handle acquire transaction initiation + val irel_vs_iacq_conflict = + io.inner.acquire.valid && + io.inner.release.valid && + io.irel().conflicts(io.iacq()) val sdqLoc = List.fill(nTransactors) { DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits } @@ -69,7 +73,7 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) trackerList.map(_.io.matches.iacq), trackerList.map(_.io.alloc.iacq), Some(sdqLoc), - Some(sdq_rdy)) + Some(sdq_rdy && !irel_vs_iacq_conflict)) // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() From 1344d09cef950814afcfca7304bb3bc75c1f1ecd Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Sat, 5 Mar 2016 17:20:54 -0800 Subject: [PATCH 540/688] Fix the SCR file for Chisel 3 --- uncore/src/main/scala/scr.scala | 39 ++++----------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 72b4c63d..9f1ed7d1 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -41,50 +41,19 @@ class SCRIO(map: SCRFileMap)(implicit p: Parameters) extends HtifBundle()(p) { val waddr = UInt(OUTPUT, log2Up(nSCR)) val wdata = Bits(OUTPUT, scrDataBits) - def attach(regs: Seq[Data]): Seq[Data] = { - regs.zipWithIndex.map{ case(reg, i) => attach(reg) } - } - def attach(regs: Seq[Data], name_base: String): Seq[Data] = { regs.zipWithIndex.map{ case(reg, i) => attach(reg, name_base + "__" + i) } } - def attach(data: Data): Data = attach(data, data.name, false, false) - def attach(data: Data, name: String): Data = attach(data, name, false, false) - def attach(data: Data, addReg: Boolean): Data = attach(data, data.name, false, false) - def attach(data: Data, addReg: Boolean, readOnly: Boolean): Data = attach(data, data.name, readOnly, false) - def attach(data: Data, name: String, addReg: Boolean): Data = attach(data, name, addReg, false) - - def attach(data: Data, name: String, addReg: Boolean, readOnly: Boolean): Data = { + def attach(reg: Data, name: String): Data = { val addr = map.allocate(name) - val reg = if(addReg) { Reg(init = Bits(0, width=data.getWidth)) } else { data } - if (!readOnly) { - when (wen && (waddr === UInt(addr))) { - reg := wdata(data.getWidth-1,0) - } - } - require(data.getWidth <= scrDataBits, "SCR Width must be <= %d for %s".format(scrDataBits,name)) - if (data.getWidth < scrDataBits) { - rdata(addr) := Cat(UInt(0, width=(scrDataBits-data.getWidth)),reg) - } else { - rdata(addr) := reg + when (wen && (waddr === UInt(addr))) { + reg := wdata } + rdata(addr) := reg reg } - - def attach(bundle: Bundle): Array[Data] = attach(bundle, "") - - def attach(bundle: Bundle, prefix: String): Array[Data] = { - bundle.flatten.map { x => - if (x._2.dir == OUTPUT) { - attach(x._2, prefix + x._1, false, true) - } else { - attach(x._2, prefix + x._1, true) - } - } - } - def allocate(address: Int, name: String): Unit = { map.allocate(address, name) } From aa22f175c3844598023015e1d94935b8dbabcb23 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Mon, 14 Mar 2016 14:12:21 -0700 Subject: [PATCH 541/688] Add cloneType methods for Chisel3 --- uncore/src/main/scala/htif.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 1391d71c..42f6547d 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -38,6 +38,8 @@ class HostIO(w: Int) extends Bundle { val in = Decoupled(Bits(width = w)).flip val out = Decoupled(Bits(width = w)) val debug_stats_csr = Bool(OUTPUT) + + override def cloneType = new HostIO(w).asInstanceOf[this.type] } class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { @@ -235,6 +237,8 @@ class NastiIOHostIOConverter(htifW: Int)(implicit val p: Parameters) val reset = Bool(OUTPUT) } + def cloneType = new NastiIOHostIOConverter(htifW).asInstanceOf[this.type] + val raddr = io.nasti.ar.bits.addr(6, 2) val waddr = io.nasti.aw.bits.addr(6, 2) From c9e1b729725f6f10abff66ddb017897119b397d0 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Mon, 14 Mar 2016 16:53:55 -0700 Subject: [PATCH 542/688] Don't assign SInt(-1) to a UInt --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d03ef445..6d7c1241 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -953,7 +953,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra io.data.write.bits.way_en := xact_way_en io.data.write.bits.addr_idx := xact_addr_idx io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := SInt(-1) // Always writes a full beat + io.data.write.bits.wmask := ~UInt(0, io.data.write.bits.wmask.getWidth) io.data.write.bits.data := data_buffer(curr_write_beat) // soon as the data is released, granted, put, or read from the cache From 00b3908d92bb84c798643f7602a2e71f65c0f032 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 21 Mar 2016 22:59:55 -0700 Subject: [PATCH 543/688] git rid of reorder queue in narrower --- uncore/src/main/scala/converters.scala | 34 +++++--------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 5b510360..954e5ebb 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -884,9 +884,6 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val acq_addr_beat = Reg(iacq.addr_beat) val oacq_ctr = Counter(factor) - // this part of the address shifts from the inner byte address - // to the outer beat address - val readshift = iacq.full_addr()(innerByteAddrBits - 1, outerByteAddrBits) val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) @@ -958,32 +955,15 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val sending_put = Reg(init = Bool(false)) - val pass_valid = io.in.acquire.valid && !stretch && !smallget - val smallget_valid = smallget && io.in.acquire.valid - - val smallget_roq = Module(new ReorderQueue( - readshift, outerIdBits, outerMaxClients)) - - val smallget_helper = DecoupledHelper( - smallget_valid, - smallget_roq.io.enq.ready, - io.out.acquire.ready) - - smallget_roq.io.enq.valid := smallget_helper.fire( - smallget_roq.io.enq.ready, !sending_put) - smallget_roq.io.enq.bits.data := readshift - smallget_roq.io.enq.bits.tag := iacq.client_xact_id + val pass_valid = io.in.acquire.valid && !stretch io.out.acquire.bits := MuxBundle(Wire(io.out.acquire.bits, init=iacq), Seq( (sending_put, put_block_acquire), (shrink, get_block_acquire), (smallput, put_acquire), (smallget, get_acquire))) - io.out.acquire.valid := sending_put || pass_valid || - smallget_helper.fire(io.out.acquire.ready) - io.in.acquire.ready := !sending_put && (stretch || - (!smallget && io.out.acquire.ready) || - smallget_helper.fire(smallget_valid)) + io.out.acquire.valid := sending_put || pass_valid + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) when (io.in.acquire.fire() && stretch) { acq_data_buffer := iacq.data @@ -1018,11 +998,9 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) data = gnt_data_buffer.toBits)(innerConfig) val smallget_grant = ognt.g_type === Grant.getDataBeatType - val get_grant_shift = Cat(smallget_roq.io.deq.data, - UInt(0, outerByteAddrBits + 3)) - - smallget_roq.io.deq.valid := io.out.grant.fire() && smallget_grant - smallget_roq.io.deq.tag := ognt.client_xact_id + val get_grant_align = io.out.grant.bits.addr_beat( + innerByteAddrBits - outerByteAddrBits - 1, 0) + val get_grant_shift = Cat(get_grant_align, UInt(0, outerByteAddrBits + 3)) val get_grant = Grant( is_builtin_type = Bool(true), From 11bd15432a55e3c00a6c9b1b10bd348fa8f3105c Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 24 Mar 2016 18:09:37 -0700 Subject: [PATCH 544/688] fix bug in RTC --- uncore/src/main/scala/rtc.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index f22121a9..2b10daef 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -39,7 +39,7 @@ class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule coreId := addr_send_cnt } else { when (io.aw.fire()) { sending_addr := Bool(false) } - when (io.w.fire()) { sending_addr := Bool(false) } + when (io.w.fire()) { sending_data := Bool(false) } coreId := UInt(0) } From 7f8f138d6a3bad78abe1a6f0ac219cfe8d2b7967 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 24 Mar 2016 19:48:52 -0700 Subject: [PATCH 545/688] fix addPendingBitWhenPartialWritemask --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 6d7c1241..9fc29cc6 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -508,7 +508,7 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { val a = in.bits val isPartial = a.wmask() =/= Acquire.fullWriteMask - addPendingBitWhenBeat(in.fire() && isPartial && Bool(ignoresWriteMask), a) + addPendingBitWhenBeat(in.fire() && (isPartial || Bool(ignoresWriteMask)), a) } } From 8e7f18084ba86fa784a393b935a0ea9378bf0470 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 28 Mar 2016 12:23:16 -0700 Subject: [PATCH 546/688] switch RTC to use TileLink instead of AXI --- uncore/src/main/scala/rtc.scala | 54 +++++++++++++++------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 2b10daef..0942e2a4 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -7,62 +7,58 @@ import cde.{Parameters, Field} case object RTCPeriod extends Field[Int] class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule + with HasTileLinkParameters with HasAddrMapParameters { - val io = new NastiIO + val io = new ClientUncachedTileLinkIO val addrTable = Vec.tabulate(nCores) { i => - UInt(addrMap(s"conf:csr$i").start + csr_MTIME * csrDataBytes) + UInt(addrMap(s"conf:csr$i").start + csr_MTIME * csrDataBytes, p(PAddrBits)) } - val rtc = Reg(init=UInt(0, scrDataBits)) + val rtc = Reg(init=UInt(0, csrDataBits)) val rtc_tick = Counter(p(RTCPeriod)).inc() - val sending_addr = Reg(init = Bool(false)) - val sending_data = Reg(init = Bool(false)) + val sending = Reg(init = Bool(false)) val send_acked = Reg(init = Vec.fill(nCores)(Bool(true))) val coreId = Wire(UInt(width = log2Up(nCores))) when (rtc_tick) { rtc := rtc + UInt(1) send_acked := Vec.fill(nCores)(Bool(false)) - sending_addr := Bool(true) - sending_data := Bool(true) + sending := Bool(true) } if (nCores > 1) { - val (addr_send_cnt, addr_send_done) = Counter(io.aw.fire(), nCores) - val (_, data_send_done) = Counter(io.w.fire(), nCores) + val (send_cnt, send_done) = Counter(io.acquire.fire(), nCores) - when (addr_send_done) { sending_addr := Bool(false) } - when (data_send_done) { sending_data := Bool(false) } + when (send_done) { sending := Bool(false) } - coreId := addr_send_cnt + coreId := send_cnt } else { - when (io.aw.fire()) { sending_addr := Bool(false) } - when (io.w.fire()) { sending_data := Bool(false) } + when (io.acquire.fire()) { sending := Bool(false) } coreId := UInt(0) } - when (io.b.fire()) { send_acked(io.b.bits.id) := Bool(true) } + when (io.grant.fire()) { send_acked(io.grant.bits.client_xact_id) := Bool(true) } - io.aw.valid := sending_addr - io.aw.bits := NastiWriteAddressChannel( - id = coreId, - addr = addrTable(coreId), - size = UInt(log2Up(csrDataBytes))) - require(p(MIFMasterTagBits) >= log2Up(nCores)) + val addr_full = addrTable(coreId) + val addr_block = addr_full(p(PAddrBits) - 1, p(CacheBlockOffsetBits)) + val addr_beat = addr_full(p(CacheBlockOffsetBits) - 1, tlByteAddrBits) + val addr_byte = addr_full(tlByteAddrBits - 1, 0) + val wmask = Fill(csrDataBytes, UInt(1, 1)) << addr_byte - io.w.valid := sending_data - io.w.bits := NastiWriteDataChannel(data = rtc) + io.acquire.valid := sending + io.acquire.bits := Put( + client_xact_id = coreId, + addr_block = addr_block, + addr_beat = addr_beat, + wmask = wmask, + data = Fill(tlDataBytes / csrDataBytes, rtc)) + io.grant.ready := Bool(true) - io.b.ready := Bool(true) - io.ar.valid := Bool(false) - io.r.ready := Bool(false) + require(tlClientXactIdBits >= log2Up(nCores)) assert(!rtc_tick || send_acked.reduce(_ && _), "Not all clocks were updated for rtc tick") - - assert(!io.b.valid || io.b.bits.resp === UInt(0), - "RTC received NASTI error response") } From 3d990bdbef020b1d7fc3f6ea9e86a70ed6a25445 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 30 Mar 2016 19:15:22 -0700 Subject: [PATCH 547/688] workaround for Chisel3 name-aliasing issue --- uncore/src/main/scala/cache.scala | 36 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 9fc29cc6..02406f57 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -625,9 +625,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val xact_op_code = Reg{ UInt() } val xact_addr_byte = Reg{ UInt() } val xact_op_size = Reg{ UInt() } - val xact_vol_irel_r_type = Reg{ io.irel().r_type } - val xact_vol_irel_src = Reg{ io.irel().client_id } - val xact_vol_irel_client_xact_id = Reg{ io.irel().client_xact_id } + val xact_vol_ir_r_type = Reg{ io.irel().r_type } + val xact_vol_ir_src = Reg{ io.irel().client_id } + val xact_vol_ir_client_xact_id = Reg{ io.irel().client_xact_id } // Miss queue holds transaction metadata used to make grants val ignt_q = Module(new Queue( @@ -639,10 +639,10 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) val xact_addr_tag = xact_addr_block >> UInt(tagLSB) val xact_vol_irel = Release( - src = xact_vol_irel_src, + src = xact_vol_ir_src, voluntary = Bool(true), - r_type = xact_vol_irel_r_type, - client_xact_id = xact_vol_irel_client_xact_id, + r_type = xact_vol_ir_r_type, + client_xact_id = xact_vol_ir_client_xact_id, addr_block = xact_addr_block) (p.alterPartial({ case TLId => p(InnerTLId) })) @@ -876,9 +876,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) mergeDataInner(io.inner.release) when(io.inner.release.fire() && irel_can_merge) { - xact_vol_irel_r_type := io.irel().r_type - xact_vol_irel_src := io.irel().client_id - xact_vol_irel_client_xact_id := io.irel().client_xact_id + xact_vol_ir_r_type := io.irel().r_type + xact_vol_ir_src := io.irel().client_id + xact_vol_ir_client_xact_id := io.irel().client_xact_id } // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: @@ -1112,19 +1112,19 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac val xact = Reg(new L2WritebackReq) val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val xact_vol_irel_r_type = Reg{ io.irel().r_type } - val xact_vol_irel_src = Reg{ io.irel().client_id } - val xact_vol_irel_client_xact_id = Reg{ io.irel().client_xact_id } + val xact_vol_ir_r_type = Reg{ io.irel().r_type } + val xact_vol_ir_src = Reg{ io.irel().client_id } + val xact_vol_ir_client_xact_id = Reg{ io.irel().client_xact_id } val xact_addr_block = if (cacheIdBits == 0) Cat(xact.tag, xact.idx) else Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) val xact_vol_irel = Release( - src = xact_vol_irel_src, + src = xact_vol_ir_src, voluntary = Bool(true), - r_type = xact_vol_irel_r_type, - client_xact_id = xact_vol_irel_client_xact_id, + r_type = xact_vol_ir_r_type, + client_xact_id = xact_vol_ir_client_xact_id, addr_block = xact_addr_block) val pending_irels = connectTwoWayBeatCounter( @@ -1189,9 +1189,9 @@ class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTrac xact.coh := pending_coh_on_irel when(io.irel().hasData()) { data_buffer(io.irel().addr_beat) := io.irel().data } when(irel_can_merge) { - xact_vol_irel_r_type := io.irel().r_type - xact_vol_irel_src := io.irel().client_id - xact_vol_irel_client_xact_id := io.irel().client_xact_id + xact_vol_ir_r_type := io.irel().r_type + xact_vol_ir_src := io.irel().client_id + xact_vol_ir_client_xact_id := io.irel().client_xact_id } } From d78066db5c27c861c5881e346f3e7911583eded8 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 30 Mar 2016 22:11:19 -0700 Subject: [PATCH 548/688] chisel3 fix for split metadata --- uncore/src/main/scala/cache.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 02406f57..dd219469 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -158,7 +158,7 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext if (hasSplitMetadata) { val tag_arrs = List.fill(nWays){ SeqMem(nSets, UInt(width = metabits)) } val tag_readout = Wire(Vec(nWays,rstVal.cloneType)) - val tags_vec = Wire(Vec.fill(nWays)(UInt(width = metabits))) + val tags_vec = Wire(Vec(nWays, UInt(width = metabits))) (0 until nWays).foreach { (i) => when (rst || (io.write.valid && wmask(i))) { tag_arrs(i).write(waddr, wdata) From cf363b1fe48d9a2073e0be08a47342dd305ce804 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 31 Mar 2016 11:38:54 -0700 Subject: [PATCH 549/688] add TileLink interconnect generator --- uncore/src/main/scala/interconnect.scala | 128 +++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 uncore/src/main/scala/interconnect.scala diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala new file mode 100644 index 00000000..799bb724 --- /dev/null +++ b/uncore/src/main/scala/interconnect.scala @@ -0,0 +1,128 @@ +package uncore + +import Chisel._ +import junctions._ +import scala.collection.mutable.ArraySeq +import cde.{Parameters, Field} + +class ClientUncachedTileLinkIORouter( + nOuter: Int, routeSel: UInt => UInt)(implicit p: Parameters) + extends TLModule { + + val io = new Bundle { + val in = (new ClientUncachedTileLinkIO).flip + val out = Vec(nOuter, new ClientUncachedTileLinkIO) + } + + val acq_route = routeSel(io.in.acquire.bits.full_addr()) + + io.in.acquire.ready := Bool(false) + + io.out.zipWithIndex.foreach { case (out, i) => + out.acquire.valid := io.in.acquire.valid && acq_route(i) + out.acquire.bits := io.in.acquire.bits + when (acq_route(i)) { io.in.acquire.ready := out.acquire.ready } + } + + val gnt_arb = Module(new LockingRRArbiter( + new Grant, nOuter, tlDataBeats, Some((gnt: Grant) => gnt.hasMultibeatData()))) + gnt_arb.io.in <> io.out.map(_.grant) + io.in.grant <> gnt_arb.io.out +} + +class TileLinkInterconnectIO(val nInner: Int, val nOuter: Int) + (implicit p: Parameters) extends Bundle { + val in = Vec(nInner, new ClientUncachedTileLinkIO).flip + val out = Vec(nOuter, new ClientUncachedTileLinkIO) +} + +class ClientUncachedTileLinkIOCrossbar( + nInner: Int, nOuter: Int, routeSel: UInt => UInt) + (implicit p: Parameters) extends TLModule { + + val io = new TileLinkInterconnectIO(nInner, nOuter) + + if (nInner == 1) { + val router = Module(new ClientUncachedTileLinkIORouter(nOuter, routeSel)) + router.io.in <> io.in.head + io.out <> router.io.out + } else { + val routers = List.fill(nInner) { + Module(new ClientUncachedTileLinkIORouter(nOuter, routeSel)) } + val arbiters = List.fill(nOuter) { + Module(new ClientUncachedTileLinkIOArbiter(nInner)) } + + for (i <- 0 until nInner) { + routers(i).io.in <> io.in(i) + } + + for (i <- 0 until nOuter) { + arbiters(i).io.in <> routers.map(r => r.io.out(i)) + io.out(i) <> arbiters(i).io.out + } + } +} + +abstract class TileLinkInterconnect(implicit p: Parameters) extends TLModule()(p) { + val nInner: Int + val nOuter: Int + + lazy val io = new TileLinkInterconnectIO(nInner, nOuter) +} + +class TileLinkRecursiveInterconnect( + val nInner: Int, val nOuter: Int, + addrmap: AddrMap, base: BigInt) + (implicit p: Parameters) extends TileLinkInterconnect()(p) { + var lastEnd = base + var outInd = 0 + val levelSize = addrmap.size + val realAddrMap = new ArraySeq[(BigInt, BigInt)](addrmap.size) + + addrmap.zipWithIndex.foreach { case (AddrMapEntry(name, startOpt, region), i) => + val start = startOpt.getOrElse(lastEnd) + val size = region.size + + require(bigIntPow2(size), + s"Region $name size $size is not a power of 2") + require(start % size == 0, + f"Region $name start address 0x$start%x not divisible by 0x$size%x" ) + require(start >= lastEnd, + f"Region $name start address 0x$start%x before previous region end") + + realAddrMap(i) = (start, size) + lastEnd = start + size + } + + val routeSel = (addr: UInt) => { + Cat(realAddrMap.map { case (start, size) => + addr >= UInt(start) && addr < UInt(start + size) + }.reverse) + } + + val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, levelSize, routeSel)) + xbar.io.in <> io.in + + addrmap.zip(realAddrMap).zip(xbar.io.out).zipWithIndex.foreach { + case (((entry, (start, size)), xbarOut), i) => { + entry.region match { + case MemSize(_, _) => + io.out(outInd) <> xbarOut + outInd += 1 + case MemSubmap(_, submap) => + if (submap.isEmpty) { + xbarOut.acquire.ready := Bool(false) + xbarOut.grant.valid := Bool(false) + } else { + val subSlaves = submap.countSlaves + val outputs = io.out.drop(outInd).take(subSlaves) + val ic = Module(new TileLinkRecursiveInterconnect(1, subSlaves, submap, start)) + ic.io.in.head <> xbarOut + for ((o, s) <- outputs zip ic.io.out) + o <> s + outInd += subSlaves + } + } + } + } +} From 3083bbca21589820798b79e36a21d839d84a7194 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 31 Mar 2016 18:15:51 -0700 Subject: [PATCH 550/688] fix TileLink arbiters and add memory interconnect and memory selector --- uncore/src/main/scala/interconnect.scala | 98 ++++++++++++++++++++++++ uncore/src/main/scala/tilelink.scala | 16 ++-- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 799bb724..9cb39485 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -126,3 +126,101 @@ class TileLinkRecursiveInterconnect( } } } + +class TileLinkMemoryInterconnect( + nBanksPerChannel: Int, nChannels: Int) + (implicit p: Parameters) extends TileLinkInterconnect()(p) { + + val nBanks = nBanksPerChannel * nChannels + val nInner = nBanks + val nOuter = nChannels + + def connectChannel(outer: ClientUncachedTileLinkIO, inner: ClientUncachedTileLinkIO) { + outer <> inner + outer.acquire.bits.addr_block := inner.acquire.bits.addr_block >> UInt(log2Ceil(nChannels)) + } + + for (i <- 0 until nChannels) { + /* Bank assignments to channels are strided so that consecutive banks + * map to different channels. That way, consecutive cache lines also + * map to different channels */ + val banks = (i until nBanks by nChannels).map(j => io.in(j)) + + val channelArb = Module(new ClientUncachedTileLinkIOArbiter(nBanksPerChannel)) + channelArb.io.in <> banks + connectChannel(io.out(i), channelArb.io.out) + } +} + +/** Allows users to switch between various memory configurations. Note that + * this is a dangerous operation: not only does switching the select input to + * this module violate TileLink, it also causes the memory of the machine to + * become garbled. It's expected that select only changes at boot time, as + * part of the memory controller configuration. */ +class TileLinkMemorySelectorIO(val nBanks: Int, val maxMemChannels: Int, nConfigs: Int) + (implicit p: Parameters) + extends TileLinkInterconnectIO(nBanks, maxMemChannels) { + val select = UInt(INPUT, width = log2Up(nConfigs)) + override def cloneType = + new TileLinkMemorySelectorIO(nBanks, maxMemChannels, nConfigs).asInstanceOf[this.type] +} + +class TileLinkMemorySelector(nBanks: Int, maxMemChannels: Int, configs: Seq[Int]) + (implicit p: Parameters) + extends TileLinkInterconnect()(p) { + val nInner = nBanks + val nOuter = maxMemChannels + val nConfigs = configs.size + + override lazy val io = new TileLinkMemorySelectorIO(nBanks, maxMemChannels, nConfigs) + + def muxOnSelect[T <: Data](up: DecoupledIO[T], dn: DecoupledIO[T], active: Bool): Unit = { + when (active) { dn.bits := up.bits } + when (active) { up.ready := dn.ready } + when (active) { dn.valid := up.valid } + } + + def muxOnSelect(up: ClientUncachedTileLinkIO, dn: ClientUncachedTileLinkIO, active: Bool): Unit = { + muxOnSelect(up.acquire, dn.acquire, active) + muxOnSelect(dn.grant, up.grant, active) + } + + def muxOnSelect(up: Vec[ClientUncachedTileLinkIO], dn: Vec[ClientUncachedTileLinkIO], active: Bool) : Unit = { + for (i <- 0 until up.size) + muxOnSelect(up(i), dn(i), active) + } + + /* Disconnects a vector of TileLink ports, which involves setting them to + * invalid. Due to Chisel reasons, we need to also set the bits to 0 (since + * there can't be any unconnected inputs). */ + def disconnectOuter(outer: Vec[ClientUncachedTileLinkIO]) = { + outer.foreach{ m => + m.acquire.valid := Bool(false) + m.acquire.bits := m.acquire.bits.fromBits(UInt(0)) + m.grant.ready := Bool(false) + } + } + + def disconnectInner(inner: Vec[ClientUncachedTileLinkIO]) = { + inner.foreach { m => + m.grant.valid := Bool(false) + m.grant.bits := m.grant.bits.fromBits(UInt(0)) + m.acquire.ready := Bool(false) + } + } + + /* Provides default wires on all our outputs. */ + disconnectOuter(io.out) + disconnectInner(io.in) + + /* Constructs interconnects for each of the layouts suggested by the + * configuration and switches between them based on the select input. */ + configs.zipWithIndex.foreach{ case (nChannels, select) => + val nBanksPerChannel = nBanks / nChannels + val ic = Module(new TileLinkMemoryInterconnect(nBanksPerChannel, nChannels)) + disconnectInner(ic.io.out) + disconnectOuter(ic.io.in) + muxOnSelect(io.in, ic.io.in, io.select === UInt(select)) + muxOnSelect(ic.io.out, io.out, io.select === UInt(select)) + } +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index c4850c62..607d855b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -1150,8 +1150,10 @@ class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) val in = Vec(arbN, new ClientUncachedTileLinkIO).flip val out = new ClientUncachedTileLinkIO } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } } /** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ @@ -1160,8 +1162,10 @@ class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends val in = Vec(arbN, new ClientTileLinkIO).flip val out = new ClientTileLinkIO } - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } } From 8957b5e97359f0fb8b5ee3bde319d15776080896 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 1 Apr 2016 13:27:50 -0700 Subject: [PATCH 551/688] Improve simulation speed of BasicCrossbar --- uncore/src/main/scala/network.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 43b379c2..64eff1d3 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -28,20 +28,18 @@ abstract class PhysicalNetwork extends Module class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { val io = new BasicCrossbarIO(n, dType) - val rdyVecs = Seq.fill(n){Seq.fill(n)(Wire(Bool()))} + io.in.foreach { _.ready := Bool(false) } - io.out.zip(rdyVecs).zipWithIndex.map{ case ((out, rdys), i) => { + io.out.zipWithIndex.map{ case (out, i) => { val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) - (rrarb.io.in, io.in, rdys).zipped.map{ case (arb, in, rdy) => { - arb.valid := in.valid && (in.bits.header.dst === UInt(i)) + (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { + val destined = in.bits.header.dst === UInt(i) + arb.valid := in.valid && destined arb.bits := in.bits - rdy := arb.ready && (in.bits.header.dst === UInt(i)) + when (arb.ready && destined) { in.ready := Bool(true) } }} out <> rrarb.io.out }} - for(i <- 0 until n) { - io.in(i).ready := rdyVecs.map(r => r(i)).reduceLeft(_||_) - } } abstract class LogicalNetwork extends Module From 82bdf3afcbe4d1652cb4da34c8ec5c2838c9cedb Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 1 Apr 2016 15:29:52 -0700 Subject: [PATCH 552/688] Fix LRSC starvation bug by punching Finish messages out to caching clients via a new TileLinkNetworkPort. --- uncore/src/main/scala/converters.scala | 68 +++++++-- uncore/src/main/scala/interconnect.scala | 185 +++++++++++++++++++++++ uncore/src/main/scala/tilelink.scala | 20 ++- 3 files changed, 256 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 954e5ebb..ef35ecc8 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -77,12 +77,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e } val q = Module(new FinishQueue(outstanding)) q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits.fin := g.makeFinish() - q.io.enq.bits.dst := io.grant.bits.header.src + q.io.enq.bits := g.makeFinish() + q.io.enq.bits.client_id := io.grant.bits.header.src io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.dst - io.finish.bits.payload := q.io.deq.bits.fin + io.finish.bits.header.dst := q.io.deq.bits.client_id + io.finish.bits.payload := q.io.deq.bits io.finish.valid := q.io.deq.valid q.io.deq.ready := io.finish.ready @@ -93,19 +93,15 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e } } -class FinishQueueEntry(implicit p: Parameters) extends TLBundle()(p) { - val fin = new Finish - val dst = UInt(width = log2Up(p(LNEndpoints))) -} - -class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishQueueEntry()(p), entries) +class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries) /** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] * * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, * calculating header.dst and filling in header.src. * Strips headers from [[uncore.Probe Probes]]. - * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response, + * setting up the headers for each. * * @param clientId network port id of this agent * @param addrConvert how a physical address maps to a destination manager port id @@ -117,24 +113,64 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) val network = new TileLinkIO } + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) + val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant) + + io.network.acquire <> acq_with_header + io.network.release <> rel_with_header + io.network.finish <> fin_with_header + io.client.probe <> prb_without_header + io.client.grant.bits.client_id := io.network.grant.bits.header.src + io.client.grant <> gnt_without_header +} + +/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientUncachedTileLinkIO().flip + val network = new TileLinkIO + } + val finisher = Module(new FinishUnit(clientId)) finisher.io.grant <> io.network.grant io.network.finish <> finisher.io.finish val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) - val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) val gnt_without_header = finisher.io.refill io.network.acquire.bits := acq_with_header.bits io.network.acquire.valid := acq_with_header.valid && finisher.io.ready acq_with_header.ready := io.network.acquire.ready && finisher.io.ready - io.network.release <> rel_with_header - io.client.probe <> prb_without_header io.client.grant <> gnt_without_header + io.network.probe.ready := Bool(false) + io.network.release.valid := Bool(false) } object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasClientId]( + in: DecoupledIO[T], + clientId: Int) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := in.bits.client_id + out.valid := in.valid + in.ready := out.ready + out + } def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( in: DecoupledIO[T], clientId: Int, @@ -1098,5 +1134,5 @@ class MMIOTileLinkManager(implicit p: Parameters) io.inner.grant.bits.client_id := gnt_xact.client_id io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id - io.inner.finish.ready := xact_pending(io.inner.finish.bits.manager_xact_id) + io.inner.finish.ready := Bool(true) } diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 9cb39485..c9ad3f1e 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -5,6 +5,191 @@ import junctions._ import scala.collection.mutable.ArraySeq import cde.{Parameters, Field} + +/** PortedTileLinkNetworks combine a TileLink protocol with a particular physical + * network implementation. + * + * Specifically, they provide mappings between ClientTileLinkIO/ + * ManagerTileLinkIO channels and LogicalNetwork ports (i.e. generic + * TileLinkIO with networking headers). Channels coming into the network have + * appropriate networking headers appended and outgoing channels have their + * headers stripped. + * + * @constructor base class constructor for Ported TileLink NoC + * @param addrToManagerId a mapping from a physical address to the network + * id of a coherence manager + * @param sharerToClientId a mapping from the id of a particular coherent + * client (as determined by e.g. the directory) and the network id + * of that client + * @param clientDepths the depths of the queue that should be used to buffer + * each channel on the client side of the network + * @param managerDepths the depths of the queue that should be used to buffer + * each channel on the manager side of the network + */ +abstract class PortedTileLinkNetwork( + addrToManagerId: UInt => UInt, + sharerToClientId: UInt => UInt, + clientDepths: TileLinkDepths, + managerDepths: TileLinkDepths) + (implicit p: Parameters) extends TLModule()(p) { + val nClients = tlNClients + val nManagers = tlNManagers + val io = new Bundle { + val clients_cached = Vec(tlNCachingClients, new ClientTileLinkIO).flip + val clients_uncached = Vec(tlNCachelessClients, new ClientUncachedTileLinkIO).flip + val managers = Vec(nManagers, new ManagerTileLinkIO).flip + } + + val clients = (io.clients_cached ++ io.clients_uncached).zipWithIndex.map { + case (io, idx) => { + val qs = Module(new TileLinkEnqueuer(clientDepths)) + io match { + case c: ClientTileLinkIO => { + val port = Module(new ClientTileLinkNetworkPort(idx, addrToManagerId)) + port.io.client <> c + qs.io.client <> port.io.network + qs.io.manager + } + case u: ClientUncachedTileLinkIO => { + val port = Module(new ClientUncachedTileLinkNetworkPort(idx, addrToManagerId)) + port.io.client <> u + qs.io.client <> port.io.network + qs.io.manager + } + } + } + } + + val managers = io.managers.zipWithIndex.map { + case (m, i) => { + val port = Module(new ManagerTileLinkNetworkPort(i, sharerToClientId)) + val qs = Module(new TileLinkEnqueuer(managerDepths)) + port.io.manager <> m + port.io.network <> qs.io.manager + qs.io.client + } + } +} + +/** A simple arbiter for each channel that also deals with header-based routing. + * Assumes a single manager agent. */ +class PortedTileLinkArbiter( + sharerToClientId: UInt => UInt = (u: UInt) => u, + clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), + managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0)) + (implicit p: Parameters) + extends PortedTileLinkNetwork(u => UInt(0), sharerToClientId, clientDepths, managerDepths)(p) + with TileLinkArbiterLike + with PassesId { + val arbN = nClients + require(nManagers == 1) + if(arbN > 1) { + hookupClientSource(clients.map(_.acquire), managers.head.acquire) + hookupClientSource(clients.map(_.release), managers.head.release) + hookupFinish(clients.map(_.finish), managers.head.finish) + hookupManagerSourceWithHeader(clients.map(_.probe), managers.head.probe) + hookupManagerSourceWithHeader(clients.map(_.grant), managers.head.grant) + } else { + managers.head <> clients.head + } +} + +/** Provides a separate physical crossbar for each channel. Assumes multiple manager + * agents. Managers are assigned to higher physical network port ids than + * clients, and translations between logical network id and physical crossbar + * port id are done automatically. + */ +class PortedTileLinkCrossbar( + addrToManagerId: UInt => UInt = u => UInt(0), + sharerToClientId: UInt => UInt = u => u, + clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), + managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0)) + (implicit p: Parameters) + extends PortedTileLinkNetwork(addrToManagerId, sharerToClientId, clientDepths, managerDepths)(p) { + val n = p(LNEndpoints) + val count = tlDataBeats + // Actually instantiate the particular networks required for TileLink + val acqNet = Module(new BasicCrossbar(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData()))) + val relNet = Module(new BasicCrossbar(n, new Release, count, Some((r: PhysicalNetworkIO[Release]) => r.payload.hasMultibeatData()))) + val prbNet = Module(new BasicCrossbar(n, new Probe)) + val gntNet = Module(new BasicCrossbar(n, new Grant, count, Some((g: PhysicalNetworkIO[Grant]) => g.payload.hasMultibeatData()))) + val ackNet = Module(new BasicCrossbar(n, new Finish)) + + // Aliases for the various network IO bundle types + type PNIO[T <: Data] = DecoupledIO[PhysicalNetworkIO[T]] + type LNIO[T <: Data] = DecoupledIO[LogicalNetworkIO[T]] + type FromCrossbar[T <: Data] = PNIO[T] => LNIO[T] + type ToCrossbar[T <: Data] = LNIO[T] => PNIO[T] + + // Shims for converting between logical network IOs and physical network IOs + def crossbarToManagerShim[T <: Data](in: PNIO[T]): LNIO[T] = { + val out = DefaultFromPhysicalShim(in) + out.bits.header.src := in.bits.header.src - UInt(nManagers) + out + } + def crossbarToClientShim[T <: Data](in: PNIO[T]): LNIO[T] = { + val out = DefaultFromPhysicalShim(in) + out.bits.header.dst := in.bits.header.dst - UInt(nManagers) + out + } + def managerToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { + val out = DefaultToPhysicalShim(n, in) + out.bits.header.dst := in.bits.header.dst + UInt(nManagers) + out + } + def clientToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { + val out = DefaultToPhysicalShim(n, in) + out.bits.header.src := in.bits.header.src + UInt(nManagers) + out + } + + // Make an individual connection between virtual and physical ports using + // a particular shim. Also pin the unused Decoupled control signal low. + def doDecoupledInputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: ToCrossbar[T]) = { + val s = shim(log_io) + phys_in.valid := s.valid + phys_in.bits := s.bits + s.ready := phys_in.ready + phys_out.ready := Bool(false) + } + + def doDecoupledOutputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: FromCrossbar[T]) = { + val s = shim(phys_out) + log_io.valid := s.valid + log_io.bits := s.bits + s.ready := log_io.ready + phys_in.valid := Bool(false) + } + + //Hookup all instances of a particular subbundle of TileLink + def doDecoupledHookups[T <: Data](physIO: BasicCrossbarIO[T], getLogIO: TileLinkIO => LNIO[T]) = { + physIO.in.head.bits.payload match { + case c: ClientToManagerChannel => { + managers.zipWithIndex.map { case (i, id) => + doDecoupledOutputHookup(physIO.in(id), physIO.out(id), getLogIO(i), crossbarToManagerShim[T]) + } + clients.zipWithIndex.map { case (i, id) => + doDecoupledInputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), clientToCrossbarShim[T]) + } + } + case m: ManagerToClientChannel => { + managers.zipWithIndex.map { case (i, id) => + doDecoupledInputHookup(physIO.in(id), physIO.out(id), getLogIO(i), managerToCrossbarShim[T]) + } + clients.zipWithIndex.map { case (i, id) => + doDecoupledOutputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), crossbarToClientShim[T]) + } + } + } + } + + doDecoupledHookups(acqNet.io, (tl: TileLinkIO) => tl.acquire) + doDecoupledHookups(relNet.io, (tl: TileLinkIO) => tl.release) + doDecoupledHookups(prbNet.io, (tl: TileLinkIO) => tl.probe) + doDecoupledHookups(gntNet.io, (tl: TileLinkIO) => tl.grant) + doDecoupledHookups(ackNet.io, (tl: TileLinkIO) => tl.finish) +} + class ClientUncachedTileLinkIORouter( nOuter: Int, routeSel: UInt => UInt)(implicit p: Parameters) extends TLModule { diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 607d855b..d24765b3 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -785,6 +785,17 @@ class Grant(implicit p: Parameters) extends GrantMetadata class GrantToDst(implicit p: Parameters) extends Grant with HasClientId +/** [[uncore.Grant]] with an extra field stating its destination */ +class GrantFromSrc(implicit p: Parameters) extends Grant + with HasClientId { + override def makeFinish(dummy: Int = 0): FinishToDst = { + val f = Wire(new FinishToDst) + f.manager_xact_id := this.manager_xact_id + f.client_id := this.client_id + f + } +} + /** [[uncore.GrantMetadata]] with an extra field containing an entire cache block */ class BufferedGrant(implicit p: Parameters) extends GrantMetadata with HasTileLinkBlock @@ -868,6 +879,10 @@ class Finish(implicit p: Parameters) extends ClientToManagerChannel()(p) def hasMultibeatData(dummy: Int = 0) = Bool(false) } +/** [[uncore.Finish]] with an extra field stating its destination */ +class FinishToDst(implicit p: Parameters) extends Finish + with HasClientId + /** Complete IO definition for incoherent TileLink, including networking headers */ class UncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val acquire = new DecoupledIO(new LogicalNetworkIO(new Acquire)) @@ -901,9 +916,12 @@ class ClientUncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { /** This version of TileLinkIO does not contain network headers. * It is intended for use within client agents. */ -class ClientTileLinkIO(implicit p: Parameters) extends ClientUncachedTileLinkIO()(p) { +class ClientTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { + val acquire = new DecoupledIO(new Acquire) val probe = new DecoupledIO(new Probe).flip val release = new DecoupledIO(new Release) + val grant = new DecoupledIO(new GrantFromSrc).flip + val finish = new DecoupledIO(new FinishToDst) } /** This version of TileLinkIO does not contain network headers, but From c292a07ace9a7349a2c67739b128379acf885b8b Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Fri, 1 Apr 2016 19:40:11 -0700 Subject: [PATCH 553/688] Bugfix for merged voluntary releases in L2Cache. Track pending release beats for voluntary releases that are merged by Acquire Trackers. Closes #23 and #24. --- uncore/src/main/scala/cache.scala | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index dd219469..3f1d5b37 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -524,12 +524,13 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends val xact_old_meta = Reg{ new L2Metadata } val coh = xact_old_meta.coh - val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_irel_beats = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_ignt = Reg(init=Bool(false)) val all_pending_done = !(pending_writes.orR || + pending_irel_beats.orR || pending_ignt) // These IOs are used for routing in the parent @@ -538,8 +539,8 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends io.matches.oprb := (state =/= s_idle) && io.oprb().conflicts(xact) // Accept a voluntary Release (and any further beats of data) - pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) - io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR + pending_irel_beats := (pending_irel_beats & dropPendingBitWhenBeatHasData(io.inner.release)) + io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irel_beats.orR when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } // Begin a transaction by getting the current block metadata @@ -561,7 +562,7 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends io.data.write.bits.data := xact.data_buffer(curr_write_beat) // Send an acknowledgement - io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels + io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irel_beats.orR io.inner.grant.bits := coh.inner.makeGrant(xact) when(io.inner.grant.fire()) { pending_ignt := Bool(false) } @@ -579,11 +580,9 @@ class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends // State machine updates and transaction handler metadata intialization when(state === s_idle && io.inner.release.valid && io.alloc.irel) { xact := io.irel() - when(io.irel().hasMultibeatData()) { - pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) - }. otherwise { - pending_irels := UInt(0) - } + pending_irel_beats := Mux(io.irel().hasMultibeatData(), + dropPendingBitWhenBeatHasData(io.inner.release), + UInt(0)) pending_writes := addPendingBitWhenBeatHasData(io.inner.release) pending_ignt := io.irel().requiresAck() state := s_meta_read @@ -680,6 +679,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) + val pending_irel_beats = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) val ignt_data_ready = Reg(init=Bits(0, width = io.inner.tlDataBeats)) @@ -691,6 +691,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra pending_writes.orR || pending_resps.orR || pending_puts.orR || + pending_irel_beats.orR || pending_ognt || ignt_q.io.count > UInt(0) || pending_vol_ignt || @@ -879,7 +880,11 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra xact_vol_ir_r_type := io.irel().r_type xact_vol_ir_src := io.irel().client_id xact_vol_ir_client_xact_id := io.irel().client_xact_id + pending_irel_beats := Mux(io.irel().hasMultibeatData(), + dropPendingBitWhenBeatHasData(io.inner.release), + UInt(0)) } + pending_irel_beats := (pending_irel_beats & dropPendingBitWhenBeatHasData(io.inner.release)) // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: // From f956d4edfb03c1d5b41c51f9738f3a0e0902df54 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 1 Apr 2016 17:42:13 -0700 Subject: [PATCH 554/688] NASTI does not right-justify data; fix in converter --- uncore/src/main/scala/converters.scala | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index ef35ecc8..e6f28b52 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -408,7 +408,6 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { val addr_beat = UInt(width = tlBeatAddrBits) - val byteOff = UInt(width = tlByteAddrBits) val subblock = Bool() } @@ -472,7 +471,6 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) roq.io.enq.bits.tag := io.nasti.ar.bits.id roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat - roq.io.enq.bits.data.byteOff := io.tl.acquire.bits.addr_byte() roq.io.enq.bits.data.subblock := is_subblock roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) roq.io.deq.tag := io.nasti.r.bits.id @@ -518,10 +516,6 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out - val r_aligned_data = Mux(roq.io.deq.data.subblock, - io.nasti.r.bits.data << Cat(roq.io.deq.data.byteOff, UInt(0, 3)), - io.nasti.r.bits.data) - gnt_arb.io.in(0).valid := io.nasti.r.valid io.nasti.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( @@ -531,7 +525,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) client_xact_id = io.nasti.r.bits.id, manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = r_aligned_data) + data = io.nasti.r.bits.data) assert(!gnt_arb.io.in(0).valid || roq.io.deq.matches, "NASTI tag error") gnt_arb.io.in(1).valid := io.nasti.b.valid From f68a7dabdf4f20907ce550596bc50233ab0897ca Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 4 Apr 2016 19:42:25 -0700 Subject: [PATCH 555/688] fix AXI -> TL converter --- uncore/src/main/scala/converters.scala | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index e6f28b52..46b7d7af 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -584,13 +584,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { val base = w.strb & size_mask(aw.size) val addr_byte = nasti_addr_byte(aw) - base << addr_byte - } - - def nasti_wdata(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { - val base = w.data & FillInterleaved(8, size_mask(aw.size)) - val addr_byte = nasti_addr_byte(aw) - base << Cat(addr_byte, UInt(0, 3)) + w.strb & (size_mask(aw.size) << addr_byte) } def tl_last(gnt: GrantMetadata): Bool = @@ -645,7 +639,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) client_xact_id = aw_req.id, addr_block = nasti_addr_block(aw_req), addr_beat = nasti_addr_beat(aw_req), - data = nasti_wdata(aw_req, io.nasti.w.bits), + data = io.nasti.w.bits.data, wmask = nasti_wmask(aw_req, io.nasti.w.bits))) io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) @@ -655,19 +649,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) - val acq = io.tl.acquire.bits val nXacts = tlMaxClientXacts * tlMaxClientsPerPort - val get_align = Reg(Vec(nXacts, UInt(width = tlByteAddrBits))) - val is_narrow_get = acq.a_type === Acquire.getType - - when (io.tl.acquire.fire() && is_narrow_get) { - get_align(acq.client_xact_id) := acq.addr_byte() - } - - def tl_data(gnt: Grant): UInt = - Mux(gnt.g_type === Grant.getDataBeatType, - gnt.data >> Cat(get_align(gnt.client_xact_id), UInt(0, 3)), - gnt.data) io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) io.nasti.b.bits := NastiWriteResponseChannel( @@ -676,7 +658,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) io.nasti.r.bits := NastiReadDataChannel( id = io.tl.grant.bits.client_xact_id, - data = tl_data(io.tl.grant.bits), + data = io.tl.grant.bits.data, last = tl_last(io.tl.grant.bits)) io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), From 31e145eaf0e1cca94da0e27d7849a5f91a47c93f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Apr 2016 16:21:18 -0700 Subject: [PATCH 556/688] fix BroadcastHub allocation and routing --- uncore/src/main/scala/broadcast.scala | 3 ++- uncore/src/main/scala/uncore.scala | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 520c68af..5ecaa7e2 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -73,7 +73,8 @@ class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) trackerList.map(_.io.matches.iacq), trackerList.map(_.io.alloc.iacq), Some(sdqLoc), - Some(sdq_rdy && !irel_vs_iacq_conflict)) + Some(sdq_rdy && !irel_vs_iacq_conflict), + Some(sdq_rdy)) // Queue to store impending Voluntary Release data val voluntary = io.irel().isVoluntary() diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 6444f54f..e64ad083 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -77,18 +77,20 @@ trait HasCoherenceAgentWiringHelpers { matches: Seq[Bool], allocs: Seq[Bool], dataOverrides: Option[Seq[UInt]] = None, - allocOverride: Option[Bool] = None) { + allocOverride: Option[Bool] = None, + matchOverride: Option[Bool] = None) { val ready_bits = Vec(outs.map(_.ready)).toBits val alloc_bits = PriorityEncoderOH(ready_bits) val match_bits = Vec(matches).toBits val no_matches = !match_bits.orR - val do_alloc = allocOverride.getOrElse(Bool(true)) - in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && do_alloc + val alloc_ok = allocOverride.getOrElse(Bool(true)) + val match_ok = matchOverride.getOrElse(Bool(true)) + in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && alloc_ok && match_ok outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => - out.valid := in.valid + out.valid := in.valid && match_ok out.bits := in.bits dataOverrides foreach { d => out.bits.data := d(i) } - a := alloc_bits(i) & no_matches & do_alloc + a := alloc_bits(i) & no_matches & alloc_ok } } } From f88b6932ce7f2d1b502c187938d0a139b6cf3d93 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 6 Apr 2016 15:43:21 -0700 Subject: [PATCH 557/688] don't add pending reads if data is already available --- uncore/src/main/scala/cache.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 3f1d5b37..e30cf486 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -927,8 +927,9 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra dropPendingBit(io.data.read) & dropPendingBitWhenBeatHasData(io.inner.release) & dropPendingBitWhenBeatHasData(io.outer.grant)) | - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | - addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire) + (~ignt_data_ready & ( + addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | + addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire))) val curr_read_beat = PriorityEncoder(pending_reads) io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt io.data.read.bits.id := UInt(trackerId) From 152645b1bc7d87f3ac069502384978c867cbf607 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Apr 2016 11:20:16 -0700 Subject: [PATCH 558/688] use manager_id instead of client_id in GrantFromSrc and FinishToDst --- uncore/src/main/scala/converters.scala | 10 +++++----- uncore/src/main/scala/tilelink.scala | 10 +++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 46b7d7af..100c0412 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -78,10 +78,10 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e val q = Module(new FinishQueue(outstanding)) q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) q.io.enq.bits := g.makeFinish() - q.io.enq.bits.client_id := io.grant.bits.header.src + q.io.enq.bits.manager_id := io.grant.bits.header.src io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.client_id + io.finish.bits.header.dst := q.io.deq.bits.manager_id io.finish.bits.payload := q.io.deq.bits io.finish.valid := q.io.deq.valid q.io.deq.ready := io.finish.ready @@ -123,7 +123,7 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) io.network.release <> rel_with_header io.network.finish <> fin_with_header io.client.probe <> prb_without_header - io.client.grant.bits.client_id := io.network.grant.bits.header.src + io.client.grant.bits.manager_id := io.network.grant.bits.header.src io.client.grant <> gnt_without_header } @@ -159,14 +159,14 @@ class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt } object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasClientId]( + def apply[T <: ClientToManagerChannel with HasManagerId]( in: DecoupledIO[T], clientId: Int) (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) out.bits.payload := in.bits out.bits.header.src := UInt(clientId) - out.bits.header.dst := in.bits.client_id + out.bits.header.dst := in.bits.manager_id out.valid := in.valid in.ready := out.ready out diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index d24765b3..361e6fb4 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -147,6 +147,10 @@ trait HasClientId extends HasTileLinkParameters { val client_id = UInt(width = tlClientIdBits) } +trait HasManagerId extends HasTileLinkParameters { + val manager_id = UInt(width = tlManagerIdBits) +} + trait HasAcquireUnion extends HasTileLinkParameters { val union = Bits(width = tlAcquireUnionBits) @@ -787,11 +791,11 @@ class GrantToDst(implicit p: Parameters) extends Grant /** [[uncore.Grant]] with an extra field stating its destination */ class GrantFromSrc(implicit p: Parameters) extends Grant - with HasClientId { + with HasManagerId { override def makeFinish(dummy: Int = 0): FinishToDst = { val f = Wire(new FinishToDst) f.manager_xact_id := this.manager_xact_id - f.client_id := this.client_id + f.manager_id := this.manager_id f } } @@ -881,7 +885,7 @@ class Finish(implicit p: Parameters) extends ClientToManagerChannel()(p) /** [[uncore.Finish]] with an extra field stating its destination */ class FinishToDst(implicit p: Parameters) extends Finish - with HasClientId + with HasManagerId /** Complete IO definition for incoherent TileLink, including networking headers */ class UncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { From 9b3faff5a5fa82674fcbf20732c693391ccce702 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 19 Apr 2016 09:46:31 -0700 Subject: [PATCH 559/688] add id field to write data channel in TL -> AXI converter --- uncore/src/main/scala/converters.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 100c0412..255f78ac 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -494,6 +494,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) io.nasti.w.bits := NastiWriteDataChannel( + id = io.tl.acquire.bits.client_xact_id, data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) From f9de99ed40547dfc34336549e41681e08e9742f1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Apr 2016 15:35:37 -0700 Subject: [PATCH 560/688] changes to match junctions no-mmio-base --- uncore/src/main/scala/interconnect.scala | 2 +- uncore/src/main/scala/scr.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index c9ad3f1e..428474ba 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -291,7 +291,7 @@ class TileLinkRecursiveInterconnect( addrmap.zip(realAddrMap).zip(xbar.io.out).zipWithIndex.foreach { case (((entry, (start, size)), xbarOut), i) => { entry.region match { - case MemSize(_, _) => + case MemSize(_, _, _) => io.out(outInd) <> xbarOut outInd += 1 case MemSubmap(_, submap) => diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index 9f1ed7d1..f076364d 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -1,7 +1,7 @@ package uncore import Chisel._ -import junctions.{SmiIO, MMIOBase} +import junctions.SmiIO import cde.Parameters import scala.collection.mutable.HashMap import scala.collection.mutable.ArrayBuffer From f6e44b13484639e8c406f89f62a6036da93c5546 Mon Sep 17 00:00:00 2001 From: Wei Song Date: Fri, 22 Apr 2016 17:47:34 +0100 Subject: [PATCH 561/688] avoid logical to physical header conversion overflow --- uncore/src/main/scala/interconnect.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 428474ba..2539d461 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -107,6 +107,7 @@ class PortedTileLinkCrossbar( (implicit p: Parameters) extends PortedTileLinkNetwork(addrToManagerId, sharerToClientId, clientDepths, managerDepths)(p) { val n = p(LNEndpoints) + val phyHdrWidth = log2Up(n) val count = tlDataBeats // Actually instantiate the particular networks required for TileLink val acqNet = Module(new BasicCrossbar(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData()))) @@ -134,12 +135,12 @@ class PortedTileLinkCrossbar( } def managerToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { val out = DefaultToPhysicalShim(n, in) - out.bits.header.dst := in.bits.header.dst + UInt(nManagers) + out.bits.header.dst := in.bits.header.dst + UInt(nManagers, phyHdrWidth) out } def clientToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = { val out = DefaultToPhysicalShim(n, in) - out.bits.header.src := in.bits.header.src + UInt(nManagers) + out.bits.header.src := in.bits.header.src + UInt(nManagers, phyHdrWidth) out } From 356efe2fd5413bc36a828944c1d31527cb746e02 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 26 Apr 2016 16:44:54 -0700 Subject: [PATCH 562/688] Simplify TileLink Narrower It's not necessary to use addr_beat to determine where to put the Grant data. Just stripe it across all lanes. This also gets rid of a dependence on addr_beat in Grant. If we move towards a regime where TileLink is only narrowed, not widened, we may be able to drop the field altogether. --- uncore/src/main/scala/converters.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 255f78ac..29c86038 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -1011,9 +1011,6 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) data = gnt_data_buffer.toBits)(innerConfig) val smallget_grant = ognt.g_type === Grant.getDataBeatType - val get_grant_align = io.out.grant.bits.addr_beat( - innerByteAddrBits - outerByteAddrBits - 1, 0) - val get_grant_shift = Cat(get_grant_align, UInt(0, outerByteAddrBits + 3)) val get_grant = Grant( is_builtin_type = Bool(true), @@ -1021,7 +1018,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), - data = ognt.data << get_grant_shift)(innerConfig) + data = Fill(factor, ognt.data))(innerConfig) io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) From 9044a4a4b7a78598a51972b0b29c23e9c7ea4a1e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 27 Apr 2016 00:15:00 -0700 Subject: [PATCH 563/688] Replace NastiROM with ROMSlave, which uses TileLink I'm not wedded to the name. --- uncore/src/main/scala/NastiROM.scala | 30 -------------------------- uncore/src/main/scala/rom.scala | 32 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 30 deletions(-) delete mode 100644 uncore/src/main/scala/NastiROM.scala create mode 100644 uncore/src/main/scala/rom.scala diff --git a/uncore/src/main/scala/NastiROM.scala b/uncore/src/main/scala/NastiROM.scala deleted file mode 100644 index 2c68340f..00000000 --- a/uncore/src/main/scala/NastiROM.scala +++ /dev/null @@ -1,30 +0,0 @@ -package uncore - -import Chisel._ -import junctions._ -import cde.{Parameters, Field} - -class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { - val io = new NastiIO().flip - val ar = Queue(io.ar, 1) - - // This assumes ROMs are in read-only parts of the address map. - // Reuse b_queue code from NastiErrorSlave if this assumption is bad. - when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } - assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") - io.aw.ready := Bool(false) - io.w.ready := Bool(false) - io.b.valid := Bool(false) - - val byteWidth = io.r.bits.nastiXDataBits / 8 - val rows = (contents.size + byteWidth - 1)/byteWidth + 1 - val rom = Vec.tabulate(rows) { i => - val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) - UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) - } - val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) - val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data - - io.r <> ar - io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) -} diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/rom.scala new file mode 100644 index 00000000..4e27db8c --- /dev/null +++ b/uncore/src/main/scala/rom.scala @@ -0,0 +1,32 @@ +package uncore + +import Chisel._ +import junctions._ +import cde.{Parameters, Field} + +class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new ClientUncachedTileLinkIO().flip + + val acq = Queue(io.acquire, 1) + assert(!acq.valid || acq.bits.a_type === Acquire.getType, "unsupported ROMSlave operation") + + val byteWidth = tlDataBits / 8 + val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rom = Vec.tabulate(rows) { i => + val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) + } + val rdata = rom(if (rows == 1) UInt(0) else acq.bits.full_addr()(log2Up(contents.size)-1,log2Up(byteWidth))) + + io.grant.valid := acq.valid + acq.ready := io.grant.ready + io.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = acq.bits.getBuiltInGrantType(), + client_xact_id = acq.bits.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = rdata) +} From eb0b5ec61e6db1e34fa8bfa025ca27adea9031f8 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 27 Apr 2016 00:16:21 -0700 Subject: [PATCH 564/688] Remove stats CSR --- uncore/src/main/scala/htif.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 42f6547d..5ffffd11 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -37,7 +37,6 @@ class HostIO(w: Int) extends Bundle { val clk_edge = Bool(OUTPUT) val in = Decoupled(Bits(width = w)).flip val out = Decoupled(Bits(width = w)) - val debug_stats_csr = Bool(OUTPUT) override def cloneType = new HostIO(w).asInstanceOf[this.type] } @@ -46,9 +45,6 @@ class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) val csr = new SmiIO(csrDataBits, 12).flip - val debug_stats_csr = Bool(OUTPUT) - // wired directly to stats register - // expected to be used to quickly indicate to testbench to do logging b/c in 'interesting' work } class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { @@ -59,9 +55,6 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val scr = new SmiIO(scrDataBits, scrAddrBits) } - io.host.debug_stats_csr := io.cpu.map(_.debug_stats_csr).reduce(_||_) - // system is 'interesting' if any tile is 'interesting' - val short_request_bits = 64 val long_request_bits = short_request_bits + dataBits*dataBeats require(short_request_bits % w == 0) From 87cecc336fd307789a2d1e749045ed1e368c39ea Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 27 Apr 2016 11:55:35 -0700 Subject: [PATCH 565/688] Add new RTC as TileLink slave, not AXI master --- uncore/src/main/scala/htif.scala | 1 + uncore/src/main/scala/rtc.scala | 78 +++++++++++++------------------- 2 files changed, 33 insertions(+), 46 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5ffffd11..e13bdcbc 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -44,6 +44,7 @@ class HostIO(w: Int) extends Bundle { class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) + val timerIRQ = Bool(INPUT) val csr = new SmiIO(csrDataBits, 12).flip } diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 0942e2a4..8c05d418 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -6,59 +6,45 @@ import cde.{Parameters, Field} case object RTCPeriod extends Field[Int] -class RTC(csr_MTIME: Int)(implicit p: Parameters) extends HtifModule +class RTC(nHarts: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters with HasAddrMapParameters { - val io = new ClientUncachedTileLinkIO - - val addrTable = Vec.tabulate(nCores) { i => - UInt(addrMap(s"conf:csr$i").start + csr_MTIME * csrDataBytes, p(PAddrBits)) + val io = new Bundle { + val tl = new ClientUncachedTileLinkIO().flip + val irqs = Vec(nHarts, Bool()).asOutput } - val rtc = Reg(init=UInt(0, csrDataBits)) - val rtc_tick = Counter(p(RTCPeriod)).inc() + val w = 64 + val regs = Reg(Vec(nHarts+1, UInt(width = w))) + require(w == tlDataBits) // TODO relax this constraint for narrower TL - val sending = Reg(init = Bool(false)) - val send_acked = Reg(init = Vec.fill(nCores)(Bool(true))) - val coreId = Wire(UInt(width = log2Up(nCores))) + val acq = Queue(io.tl.acquire, 1) - when (rtc_tick) { - rtc := rtc + UInt(1) - send_acked := Vec.fill(nCores)(Bool(false)) - sending := Bool(true) - } + val full_addr = acq.bits.full_addr() + val byte_addr = full_addr(log2Up(w/8)-1,0) + val size = w/8*(nHarts+1) + val addr = full_addr(log2Up(size)-1,log2Up(w/8)) + val rdata = regs(addr) + val wdata = acq.bits.data + val read = acq.bits.a_type === Acquire.getType + val write = acq.bits.a_type === Acquire.putType + val wmask = acq.bits.full_wmask() + assert(!acq.valid || read || write, "unsupported RTC operation") - if (nCores > 1) { - val (send_cnt, send_done) = Counter(io.acquire.fire(), nCores) + io.tl.grant.valid := acq.valid + acq.ready := io.tl.grant.ready + io.tl.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = acq.bits.getBuiltInGrantType(), + client_xact_id = acq.bits.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = rdata) - when (send_done) { sending := Bool(false) } + for ((irq, cmp) <- io.irqs zip regs.tail) + irq := (regs(0) >= cmp) - coreId := send_cnt - } else { - when (io.acquire.fire()) { sending := Bool(false) } - - coreId := UInt(0) - } - - when (io.grant.fire()) { send_acked(io.grant.bits.client_xact_id) := Bool(true) } - - val addr_full = addrTable(coreId) - val addr_block = addr_full(p(PAddrBits) - 1, p(CacheBlockOffsetBits)) - val addr_beat = addr_full(p(CacheBlockOffsetBits) - 1, tlByteAddrBits) - val addr_byte = addr_full(tlByteAddrBits - 1, 0) - val wmask = Fill(csrDataBytes, UInt(1, 1)) << addr_byte - - io.acquire.valid := sending - io.acquire.bits := Put( - client_xact_id = coreId, - addr_block = addr_block, - addr_beat = addr_beat, - wmask = wmask, - data = Fill(tlDataBytes / csrDataBytes, rtc)) - io.grant.ready := Bool(true) - - require(tlClientXactIdBits >= log2Up(nCores)) - - assert(!rtc_tick || send_acked.reduce(_ && _), - "Not all clocks were updated for rtc tick") + when (Counter(p(RTCPeriod)).inc()) { regs(0) := regs(0) + UInt(1) } + when (acq.valid && write) { regs(addr) := wdata & wmask | rdata & ~wmask } + when (reset) { regs(0) := UInt(0) } } From 81ff127dc3ab36fdc79dee7451064a91065bb54d Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 27 Apr 2016 14:53:11 -0700 Subject: [PATCH 566/688] Clean up TileLinkRecursiveInterconnect a bit --- uncore/src/main/scala/interconnect.scala | 38 ++++++++++-------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 2539d461..13f55df4 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -257,27 +257,25 @@ abstract class TileLinkInterconnect(implicit p: Parameters) extends TLModule()(p } class TileLinkRecursiveInterconnect( - val nInner: Int, val nOuter: Int, - addrmap: AddrMap, base: BigInt) + val nInner: Int, addrmap: AddrMap, base: BigInt) (implicit p: Parameters) extends TileLinkInterconnect()(p) { var lastEnd = base - var outInd = 0 val levelSize = addrmap.size - val realAddrMap = new ArraySeq[(BigInt, BigInt)](addrmap.size) + val nOuter = addrmap.countSlaves - addrmap.zipWithIndex.foreach { case (AddrMapEntry(name, startOpt, region), i) => - val start = startOpt.getOrElse(lastEnd) + val realAddrMap = addrmap map { case AddrMapEntry(name, region) => + val start = lastEnd val size = region.size - require(bigIntPow2(size), + require(isPow2(size), s"Region $name size $size is not a power of 2") require(start % size == 0, f"Region $name start address 0x$start%x not divisible by 0x$size%x" ) require(start >= lastEnd, f"Region $name start address 0x$start%x before previous region end") - realAddrMap(i) = (start, size) lastEnd = start + size + (start, size) } val routeSel = (addr: UInt) => { @@ -289,25 +287,19 @@ class TileLinkRecursiveInterconnect( val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, levelSize, routeSel)) xbar.io.in <> io.in - addrmap.zip(realAddrMap).zip(xbar.io.out).zipWithIndex.foreach { + io.out <> addrmap.zip(realAddrMap).zip(xbar.io.out).zipWithIndex.flatMap { case (((entry, (start, size)), xbarOut), i) => { entry.region match { case MemSize(_, _, _) => - io.out(outInd) <> xbarOut - outInd += 1 + Some(xbarOut) + case MemSubmap(_, submap) if submap.isEmpty => + xbarOut.acquire.ready := Bool(false) + xbarOut.grant.valid := Bool(false) + None case MemSubmap(_, submap) => - if (submap.isEmpty) { - xbarOut.acquire.ready := Bool(false) - xbarOut.grant.valid := Bool(false) - } else { - val subSlaves = submap.countSlaves - val outputs = io.out.drop(outInd).take(subSlaves) - val ic = Module(new TileLinkRecursiveInterconnect(1, subSlaves, submap, start)) - ic.io.in.head <> xbarOut - for ((o, s) <- outputs zip ic.io.out) - o <> s - outInd += subSlaves - } + val ic = Module(new TileLinkRecursiveInterconnect(1, submap, start)) + ic.io.in.head <> xbarOut + ic.io.out } } } From ed5bdf3c233ae3d644d88d608da12e772cd385c4 Mon Sep 17 00:00:00 2001 From: Wei Song Date: Thu, 28 Apr 2016 16:31:56 +0100 Subject: [PATCH 567/688] print the base address of each SCR as indicated --- uncore/src/main/scala/scr.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala index f076364d..29a591af 100644 --- a/uncore/src/main/scala/scr.scala +++ b/uncore/src/main/scala/scr.scala @@ -29,9 +29,11 @@ class SCRFileMap(prefix: String, maxAddress: Int, baseAddress: BigInt) { def as_c_header(): String = { addr2name.map{ case(address, name) => - "#define " + prefix + "__" + name + "__PADDR 0x%x".format(baseAddress + address) - "#define " + prefix + "__" + name + "__OFFSET 0x%x".format(address) - }.mkString("\n") + "\n" + List( + "#define " + prefix + "__" + name + "__PADDR 0x%x".format(baseAddress + address), + "#define " + prefix + "__" + name + "__OFFSET 0x%x".format(address) + ) + }.flatten.mkString("\n") + "\n" } } From 1df68a25fd8ccaeebcf68e2f8b3b6c85f2e7d322 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 28 Apr 2016 16:08:58 -0700 Subject: [PATCH 568/688] Address Map refactoring --- uncore/src/main/scala/interconnect.scala | 30 +++++++----------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 13f55df4..fcb87b64 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -259,45 +259,31 @@ abstract class TileLinkInterconnect(implicit p: Parameters) extends TLModule()(p class TileLinkRecursiveInterconnect( val nInner: Int, addrmap: AddrMap, base: BigInt) (implicit p: Parameters) extends TileLinkInterconnect()(p) { - var lastEnd = base val levelSize = addrmap.size val nOuter = addrmap.countSlaves - val realAddrMap = addrmap map { case AddrMapEntry(name, region) => - val start = lastEnd - val size = region.size - - require(isPow2(size), - s"Region $name size $size is not a power of 2") - require(start % size == 0, - f"Region $name start address 0x$start%x not divisible by 0x$size%x" ) - require(start >= lastEnd, - f"Region $name start address 0x$start%x before previous region end") - - lastEnd = start + size - (start, size) - } - + val addrHashMap = new AddrHashMap(addrmap, base) val routeSel = (addr: UInt) => { - Cat(realAddrMap.map { case (start, size) => - addr >= UInt(start) && addr < UInt(start + size) + Cat(addrmap.map { case entry => + val hashEntry = addrHashMap(entry.name) + addr >= UInt(hashEntry.start) && addr < UInt(hashEntry.start + hashEntry.region.size) }.reverse) } val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, levelSize, routeSel)) xbar.io.in <> io.in - io.out <> addrmap.zip(realAddrMap).zip(xbar.io.out).zipWithIndex.flatMap { - case (((entry, (start, size)), xbarOut), i) => { + io.out <> addrmap.zip(xbar.io.out).flatMap { + case (entry, xbarOut) => { entry.region match { - case MemSize(_, _, _) => + case _: MemSize => Some(xbarOut) case MemSubmap(_, submap) if submap.isEmpty => xbarOut.acquire.ready := Bool(false) xbarOut.grant.valid := Bool(false) None case MemSubmap(_, submap) => - val ic = Module(new TileLinkRecursiveInterconnect(1, submap, start)) + val ic = Module(new TileLinkRecursiveInterconnect(1, submap, addrHashMap(entry.name).start)) ic.io.in.head <> xbarOut ic.io.out } From 6f052a740c167d70804010cc4bc181c32a00b2fe Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Fri, 29 Apr 2016 14:10:44 -0700 Subject: [PATCH 569/688] Add TileLink BRAM slave --- uncore/src/main/scala/bram.scala | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 uncore/src/main/scala/bram.scala diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala new file mode 100644 index 00000000..b97124ab --- /dev/null +++ b/uncore/src/main/scala/bram.scala @@ -0,0 +1,64 @@ +package uncore + +import Chisel._ +import cde.{Parameters, Field} + +class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module + with HasTileLinkParameters { + val io = new ClientUncachedTileLinkIO().flip + + val bram = SeqMem(depth, Bits(width = tlDataBits)) + + val (s0_get :: s0_getblk :: s0_put :: s0_putblk :: Nil) = Seq( + Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType + ).map(io.acquire.bits.isBuiltInType _) + + val fire_acq = io.acquire.fire() + val fire_gnt = io.grant.fire() + + val multibeat = Reg(init = Bool(false)) + when (fire_acq) { + multibeat := s0_getblk + } + + val s0_valid = io.acquire.valid || multibeat + val s1_valid = Reg(next = s0_valid, init = Bool(false)) + val s1_acq = RegEnable(io.acquire.bits, fire_acq) + + val s0_addr = Cat(io.acquire.bits.addr_block, io.acquire.bits.addr_beat) + val s1_beat = s1_acq.addr_beat + Mux(io.grant.ready, UInt(1), UInt(0)) + val s1_addr = Cat(s1_acq.addr_block, s1_beat) + val raddr = Mux(multibeat, s1_addr, s0_addr) + + val last = (s1_acq.addr_beat === UInt(tlDataBeats-1)) + val ren = (io.acquire.valid && (s0_get || s0_getblk)) || (multibeat && !last) + val wen = (io.acquire.valid && (s0_put || s0_putblk)) + + val rdata = bram.read(raddr, ren) + val wdata = io.acquire.bits.data + val wmask = io.acquire.bits.wmask() + assert(!io.acquire.valid || wmask === Fill(tlDataBytes, Bool(true)), + "bram: subblock writes not supported") + when (wen) { + bram.write(s0_addr, wdata) + } + + when (multibeat && fire_gnt) { + s1_acq.addr_beat := s1_beat + when (last) { + multibeat := Bool(false) + } + } + + io.grant.valid := s1_valid + io.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = s1_acq.getBuiltInGrantType(), + client_xact_id = s1_acq.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = s1_acq.addr_beat, + data = rdata) + + val stall = multibeat || (io.grant.valid && !io.grant.ready) + io.acquire.ready := !stall +} From 695c4c5096e2c0cf819a2b93d1c05e62ab21f7f0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 30 Apr 2016 17:34:12 -0700 Subject: [PATCH 570/688] Support both Get and GetBlock on ROMSlave --- uncore/src/main/scala/rom.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/rom.scala index 4e27db8c..6df60ef1 100644 --- a/uncore/src/main/scala/rom.scala +++ b/uncore/src/main/scala/rom.scala @@ -10,7 +10,13 @@ class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module val io = new ClientUncachedTileLinkIO().flip val acq = Queue(io.acquire, 1) - assert(!acq.valid || acq.bits.a_type === Acquire.getType, "unsupported ROMSlave operation") + val single_beat = acq.bits.isBuiltInType(Acquire.getType) + val multi_beat = acq.bits.isBuiltInType(Acquire.getBlockType) + assert(!acq.valid || single_beat || multi_beat, "unsupported ROMSlave operation") + + val addr_beat = Reg(UInt()) + when (io.grant.fire()) { addr_beat := addr_beat + UInt(1) } + when (io.acquire.fire()) { addr_beat := io.acquire.bits.addr_beat } val byteWidth = tlDataBits / 8 val rows = (contents.size + byteWidth - 1)/byteWidth + 1 @@ -18,15 +24,17 @@ class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) } - val rdata = rom(if (rows == 1) UInt(0) else acq.bits.full_addr()(log2Up(contents.size)-1,log2Up(byteWidth))) + val raddr = Cat(acq.bits.addr_block, addr_beat) + val rdata = rom(if (rows == 1) UInt(0) else raddr(log2Up(rom.size)-1,0)) + val last = !multi_beat || addr_beat === UInt(tlDataBeats-1) io.grant.valid := acq.valid - acq.ready := io.grant.ready + acq.ready := io.grant.ready && last io.grant.bits := Grant( is_builtin_type = Bool(true), g_type = acq.bits.getBuiltInGrantType(), client_xact_id = acq.bits.client_xact_id, manager_xact_id = UInt(0), - addr_beat = UInt(0), + addr_beat = addr_beat, data = rdata) } From 72731de25a6e53f41446a0d1bece875aa97e475a Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 2 May 2016 15:19:43 -0700 Subject: [PATCH 571/688] Take a stab at the PRCI-Rocket interface --- uncore/src/main/scala/htif.scala | 1 - uncore/src/main/scala/prci.scala | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 uncore/src/main/scala/prci.scala diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index e13bdcbc..5ffffd11 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -44,7 +44,6 @@ class HostIO(w: Int) extends Bundle { class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { val reset = Bool(INPUT) val id = UInt(INPUT, log2Up(nCores)) - val timerIRQ = Bool(INPUT) val csr = new SmiIO(csrDataBits, 12).flip } diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/prci.scala new file mode 100644 index 00000000..c66fbde3 --- /dev/null +++ b/uncore/src/main/scala/prci.scala @@ -0,0 +1,25 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import Chisel.ImplicitConversions._ +import junctions._ +import junctions.NastiConstants._ +import cde.{Parameters, Field} + +/** Number of tiles */ +case object NTiles extends Field[Int] + +class PRCICoreIO(implicit p: Parameters) extends Bundle { + val reset = Bool(OUTPUT) + val id = UInt(OUTPUT, log2Up(p(NTiles))) + val interrupts = new Bundle { + val mtip = Bool() + val msip = Bool() + val meip = Bool() + val seip = Bool() + }.asOutput + + override def cloneType: this.type = new PRCICoreIO().asInstanceOf[this.type] +} From cc4102f8ded123ac5a8217eae4a5c5c795672a86 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 2 May 2016 17:49:10 -0700 Subject: [PATCH 572/688] Add trivial version of PRCI block It doesn't really do anything besides deliver deliver IPIs yet. --- uncore/src/main/scala/prci.scala | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/prci.scala index c66fbde3..2dee3ae8 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/prci.scala @@ -11,7 +11,7 @@ import cde.{Parameters, Field} /** Number of tiles */ case object NTiles extends Field[Int] -class PRCICoreIO(implicit p: Parameters) extends Bundle { +class PRCITileIO(implicit p: Parameters) extends Bundle { val reset = Bool(OUTPUT) val id = UInt(OUTPUT, log2Up(p(NTiles))) val interrupts = new Bundle { @@ -19,7 +19,52 @@ class PRCICoreIO(implicit p: Parameters) extends Bundle { val msip = Bool() val meip = Bool() val seip = Bool() + val debug = Bool() }.asOutput - override def cloneType: this.type = new PRCICoreIO().asInstanceOf[this.type] + override def cloneType: this.type = new PRCITileIO().asInstanceOf[this.type] +} + +class PRCI(implicit val p: Parameters) extends Module + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new Bundle { + val id = UInt(INPUT, log2Up(p(NTiles))) + val interrupts = new Bundle { + val mtip = Bool() + val meip = Bool() + val seip = Bool() + val debug = Bool() + }.asInput + val tl = new ClientUncachedTileLinkIO().flip + val tile = new PRCITileIO + } + + val ipi = Reg(init=Bool(false)) + + val acq = Queue(io.tl.acquire, 1) + val read = acq.bits.isBuiltInType(Acquire.getType) + val write = acq.bits.isBuiltInType(Acquire.putType) + val rdata = Wire(init=ipi) + io.tl.grant.valid := acq.valid + acq.ready := io.tl.grant.ready + io.tl.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = acq.bits.getBuiltInGrantType(), + client_xact_id = acq.bits.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = rdata) + + val regSize = 16 + val nRegs = 2 + val addr = acq.bits.full_addr()(log2Up(regSize*nRegs)-1,log2Up(regSize)) + + when (addr === UInt(0) && write) { + ipi := acq.bits.data(0) + } + + io.tile.interrupts := io.interrupts + io.tile.interrupts.msip := ipi + io.tile.id := io.id } From f26c422544591e03f5409d789bb540a48d809da5 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 3 May 2016 12:18:06 -0700 Subject: [PATCH 573/688] assert that TileLink router has valid route --- uncore/src/main/scala/interconnect.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index fcb87b64..88791815 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -214,6 +214,8 @@ class ClientUncachedTileLinkIORouter( new Grant, nOuter, tlDataBeats, Some((gnt: Grant) => gnt.hasMultibeatData()))) gnt_arb.io.in <> io.out.map(_.grant) io.in.grant <> gnt_arb.io.out + + assert(!io.in.acquire.valid || acq_route.orR, "No valid route") } class TileLinkInterconnectIO(val nInner: Int, val nOuter: Int) From 77e859760c38333cf99108b434ef827c9ded1d24 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 29 Apr 2016 14:30:56 -0700 Subject: [PATCH 574/688] add a Hasti RAM alongside the Nasti ROM --- uncore/src/main/scala/mem.scala | 74 +++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 uncore/src/main/scala/mem.scala diff --git a/uncore/src/main/scala/mem.scala b/uncore/src/main/scala/mem.scala new file mode 100644 index 00000000..a2cbf481 --- /dev/null +++ b/uncore/src/main/scala/mem.scala @@ -0,0 +1,74 @@ +package uncore + +import Chisel._ +import junctions._ +import cde.{Parameters, Field} + +class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { + val io = new NastiIO().flip + val ar = Queue(io.ar, 1) + + // This assumes ROMs are in read-only parts of the address map. + // Reuse b_queue code from NastiErrorSlave if this assumption is bad. + when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } + assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") + io.aw.ready := Bool(false) + io.w.ready := Bool(false) + io.b.valid := Bool(false) + + val byteWidth = io.r.bits.nastiXDataBits / 8 + val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rom = Vec.tabulate(rows) { i => + val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) + } + val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) + val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data + + io.r <> ar + io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) +} + +class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { + val io = new HastiSlaveIO + + val hastiDataBytes = hastiDataBits/8 + + val wdata = Vec.tabulate(hastiDataBytes)(i => io.hwdata(8*(i+1)-1,8*i)) + val waddr = Reg(UInt(width = hastiAddrBits)) + val wvalid = Reg(init = Bool(false)) + val wsize = Reg(UInt(width = SZ_HSIZE)) + val ram = SeqMem(depth, Vec(hastiDataBytes, Bits(width = 8))) + + val max_wsize = log2Ceil(hastiDataBytes) + val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, + (0 until max_wsize).map(1 << _).map(sz => (UInt(sz) -> UInt((1 << sz << sz) - 1)))) + val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) + + val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) + val raddr = io.haddr >> UInt(2) + val ren = is_trans && !io.hwrite + val bypass = Reg(init = Bool(false)) + val last_wdata = Reg(next = wdata) + val last_wmask = Reg(next = wmask) + + when (is_trans && io.hwrite) { + waddr := io.haddr + wsize := io.hsize + wvalid := Bool(true) + } .otherwise { wvalid := Bool(false) } + + when (ren) { bypass := wvalid && (waddr >> UInt(2)) === raddr } + + when (wvalid) { + ram.write(waddr >> UInt(2), wdata, wmask.toBools) + } + + val rdata = ram.read(raddr, ren) + io.hrdata := Cat(rdata.zip(wmask.toBools).zip(wdata).map { + case ((rbyte, wsel), wbyte) => Mux(wsel && bypass, wbyte, rbyte) + }.reverse) + + io.hreadyout := Bool(true) + io.hresp := HRESP_OKAY +} From 1ed6d6646ddcd8579213ec10b4f277d14b50bd4a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 2 May 2016 13:58:41 -0700 Subject: [PATCH 575/688] move NastiROM and HastiRAM into rom.scala and bram.scala --- uncore/src/main/scala/bram.scala | 45 +++++++++++++++++++ uncore/src/main/scala/mem.scala | 74 -------------------------------- uncore/src/main/scala/rom.scala | 25 +++++++++++ 3 files changed, 70 insertions(+), 74 deletions(-) delete mode 100644 uncore/src/main/scala/mem.scala diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index b97124ab..0176a6b6 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -2,6 +2,7 @@ package uncore import Chisel._ import cde.{Parameters, Field} +import junctions._ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters { @@ -62,3 +63,47 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module val stall = multibeat || (io.grant.valid && !io.grant.ready) io.acquire.ready := !stall } + +class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { + val io = new HastiSlaveIO + + val hastiDataBytes = hastiDataBits/8 + + val wdata = Vec.tabulate(hastiDataBytes)(i => io.hwdata(8*(i+1)-1,8*i)) + val waddr = Reg(UInt(width = hastiAddrBits)) + val wvalid = Reg(init = Bool(false)) + val wsize = Reg(UInt(width = SZ_HSIZE)) + val ram = SeqMem(depth, Vec(hastiDataBytes, Bits(width = 8))) + + val max_wsize = log2Ceil(hastiDataBytes) + val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, + (0 until max_wsize).map(1 << _).map(sz => (UInt(sz) -> UInt((1 << sz << sz) - 1)))) + val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) + + val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) + val raddr = io.haddr >> UInt(2) + val ren = is_trans && !io.hwrite + val bypass = Reg(init = Bool(false)) + val last_wdata = Reg(next = wdata) + val last_wmask = Reg(next = wmask) + + when (is_trans && io.hwrite) { + waddr := io.haddr + wsize := io.hsize + wvalid := Bool(true) + } .otherwise { wvalid := Bool(false) } + + when (ren) { bypass := wvalid && (waddr >> UInt(2)) === raddr } + + when (wvalid) { + ram.write(waddr >> UInt(2), wdata, wmask.toBools) + } + + val rdata = ram.read(raddr, ren) + io.hrdata := Cat(rdata.zip(wmask.toBools).zip(wdata).map { + case ((rbyte, wsel), wbyte) => Mux(wsel && bypass, wbyte, rbyte) + }.reverse) + + io.hreadyout := Bool(true) + io.hresp := HRESP_OKAY +} diff --git a/uncore/src/main/scala/mem.scala b/uncore/src/main/scala/mem.scala deleted file mode 100644 index a2cbf481..00000000 --- a/uncore/src/main/scala/mem.scala +++ /dev/null @@ -1,74 +0,0 @@ -package uncore - -import Chisel._ -import junctions._ -import cde.{Parameters, Field} - -class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { - val io = new NastiIO().flip - val ar = Queue(io.ar, 1) - - // This assumes ROMs are in read-only parts of the address map. - // Reuse b_queue code from NastiErrorSlave if this assumption is bad. - when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } - assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") - io.aw.ready := Bool(false) - io.w.ready := Bool(false) - io.b.valid := Bool(false) - - val byteWidth = io.r.bits.nastiXDataBits / 8 - val rows = (contents.size + byteWidth - 1)/byteWidth + 1 - val rom = Vec.tabulate(rows) { i => - val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) - UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) - } - val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) - val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data - - io.r <> ar - io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) -} - -class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { - val io = new HastiSlaveIO - - val hastiDataBytes = hastiDataBits/8 - - val wdata = Vec.tabulate(hastiDataBytes)(i => io.hwdata(8*(i+1)-1,8*i)) - val waddr = Reg(UInt(width = hastiAddrBits)) - val wvalid = Reg(init = Bool(false)) - val wsize = Reg(UInt(width = SZ_HSIZE)) - val ram = SeqMem(depth, Vec(hastiDataBytes, Bits(width = 8))) - - val max_wsize = log2Ceil(hastiDataBytes) - val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, - (0 until max_wsize).map(1 << _).map(sz => (UInt(sz) -> UInt((1 << sz << sz) - 1)))) - val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) - - val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) - val raddr = io.haddr >> UInt(2) - val ren = is_trans && !io.hwrite - val bypass = Reg(init = Bool(false)) - val last_wdata = Reg(next = wdata) - val last_wmask = Reg(next = wmask) - - when (is_trans && io.hwrite) { - waddr := io.haddr - wsize := io.hsize - wvalid := Bool(true) - } .otherwise { wvalid := Bool(false) } - - when (ren) { bypass := wvalid && (waddr >> UInt(2)) === raddr } - - when (wvalid) { - ram.write(waddr >> UInt(2), wdata, wmask.toBools) - } - - val rdata = ram.read(raddr, ren) - io.hrdata := Cat(rdata.zip(wmask.toBools).zip(wdata).map { - case ((rbyte, wsel), wbyte) => Mux(wsel && bypass, wbyte, rbyte) - }.reverse) - - io.hreadyout := Bool(true) - io.hresp := HRESP_OKAY -} diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/rom.scala index 6df60ef1..b635e127 100644 --- a/uncore/src/main/scala/rom.scala +++ b/uncore/src/main/scala/rom.scala @@ -38,3 +38,28 @@ class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module addr_beat = addr_beat, data = rdata) } + +class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { + val io = new NastiIO().flip + val ar = Queue(io.ar, 1) + + // This assumes ROMs are in read-only parts of the address map. + // Reuse b_queue code from NastiErrorSlave if this assumption is bad. + when (ar.valid) { assert(ar.bits.len === UInt(0), "Can't burst-read from NastiROM") } + assert(!(io.aw.valid || io.w.valid), "Can't write to NastiROM") + io.aw.ready := Bool(false) + io.w.ready := Bool(false) + io.b.valid := Bool(false) + + val byteWidth = io.r.bits.nastiXDataBits / 8 + val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rom = Vec.tabulate(rows) { i => + val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) + } + val rdata_word = rom(if (rows == 1) UInt(0) else ar.bits.addr(log2Up(contents.size)-1,log2Up(byteWidth))) + val rdata = new LoadGen(Cat(UInt(1), ar.bits.size), ar.bits.addr, rdata_word, Bool(false), byteWidth).data + + io.r <> ar + io.r.bits := NastiReadDataChannel(ar.bits.id, rdata) +} From 14a6e470c953e0f9d611d8db25aa50f198f09072 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Sat, 7 May 2016 21:19:27 -0700 Subject: [PATCH 576/688] transform ids in TL -> NASTI converter if necessary --- uncore/src/main/scala/converters.scala | 101 ++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 29c86038..8bc52e63 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -406,6 +406,66 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { io.in.probe.valid := Bool(false) } +class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module + with HasTileLinkParameters with HasNastiParameters { + val io = new Bundle { + val req = new Bundle { + val valid = Bool(INPUT) + val ready = Bool(OUTPUT) + val tl_id = UInt(INPUT, tlClientXactIdBits) + val nasti_id = UInt(OUTPUT, nastiXIdBits) + } + val resp = new Bundle { + val valid = Bool(INPUT) + val matches = Bool(OUTPUT) + val nasti_id = UInt(INPUT, nastiXIdBits) + val tl_id = UInt(OUTPUT, tlClientXactIdBits) + } + } + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + + if (tlClientXactIdBits <= nastiXIdBits) { + io.req.ready := Bool(true) + io.req.nasti_id := io.req.tl_id + io.resp.matches := Bool(true) + io.resp.tl_id := io.resp.nasti_id + } else if (nastiXIdBits <= 2) { + val nQueues = 1 << nastiXIdBits + val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) + io.req.ready := Bool(false) + io.resp.matches := Bool(false) + io.resp.tl_id := UInt(0) + io.req.nasti_id := req_nasti_id + for (i <- 0 until nQueues) { + val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) + queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) + queue.io.enq.bits := io.req.tl_id + when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } + + queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) + when (io.resp.nasti_id === UInt(i)) { + io.resp.matches := queue.io.deq.valid + io.resp.tl_id := queue.io.deq.bits + } + } + } else { + val maxNastiId = 1 << nastiXIdBits + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) + val roq = Module(new ReorderQueue( + UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) + roq.io.enq.valid := io.req.valid + roq.io.enq.bits.data := io.req.tl_id + roq.io.enq.bits.tag := req_nasti_id + io.req.ready := roq.io.enq.ready + io.req.nasti_id := req_nasti_id + roq.io.deq.valid := io.resp.valid + roq.io.deq.tag := io.resp.nasti_id + io.resp.tl_id := roq.io.deq.data + io.resp.matches := roq.io.deq.matches + } +} + class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { val addr_beat = UInt(width = tlBeatAddrBits) val subblock = Bool() @@ -431,7 +491,6 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val dataBits = tlDataBits*tlDataBeats require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - require(tlClientXactIdBits <= nastiXIdBits, "NastiIO converter is going truncate tags: " + tlClientXactIdBits + " > " + nastiXIdBits) val has_data = io.tl.acquire.bits.hasData() @@ -443,17 +502,26 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val get_valid = io.tl.acquire.valid && !has_data val put_valid = io.tl.acquire.valid && has_data + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + // Reorder queue saves extra information needed to send correct // grant back to TL client val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, - nastiRIdBits, tlMaxClientsPerPort)) + new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) + + val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + + val get_id_ready = get_id_mapper.io.req.ready + val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) + val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask // For Get/GetBlock, make sure Reorder queue can accept new entry val get_helper = DecoupledHelper( get_valid, roq.io.enq.ready, - io.nasti.ar.ready) + io.nasti.ar.ready, + get_id_ready) val w_inflight = Reg(init = Bool(false)) @@ -463,7 +531,8 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val put_helper = DecoupledHelper( put_valid, aw_ready, - io.nasti.w.ready) + io.nasti.w.ready, + put_id_ready) val (nasti_cnt_out, nasti_wrap_out) = Counter( io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) @@ -475,10 +544,20 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) roq.io.deq.tag := io.nasti.r.bits.id + get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) + get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last + get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id + + put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) + put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + put_id_mapper.io.resp.valid := io.nasti.b.fire() + put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id + // Decompose outgoing TL Acquires into Nasti address and data channels io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) io.nasti.ar.bits := NastiReadAddressChannel( - id = io.tl.acquire.bits.client_xact_id, + id = get_id_mapper.io.req.nasti_id, addr = io.tl.acquire.bits.full_addr(), size = Mux(is_subblock, opSizeToXSize(io.tl.acquire.bits.op_size()), @@ -487,14 +566,14 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) io.nasti.aw.bits := NastiWriteAddressChannel( - id = io.tl.acquire.bits.client_xact_id, + id = put_id_mapper.io.req.nasti_id, addr = io.tl.acquire.bits.full_addr(), size = UInt(log2Ceil(tlDataBytes)), len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) io.nasti.w.bits := NastiWriteDataChannel( - id = io.tl.acquire.bits.client_xact_id, + id = put_id_mapper.io.req.nasti_id, data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) @@ -523,21 +602,23 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) is_builtin_type = Bool(true), g_type = Mux(roq.io.deq.data.subblock, Grant.getDataBeatType, Grant.getDataBlockType), - client_xact_id = io.nasti.r.bits.id, + client_xact_id = get_id_mapper.io.resp.tl_id, manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), data = io.nasti.r.bits.data) assert(!gnt_arb.io.in(0).valid || roq.io.deq.matches, "NASTI tag error") + assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, "NASTI tag error") gnt_arb.io.in(1).valid := io.nasti.b.valid io.nasti.b.ready := gnt_arb.io.in(1).ready gnt_arb.io.in(1).bits := Grant( is_builtin_type = Bool(true), g_type = Grant.putAckType, - client_xact_id = io.nasti.b.bits.id, + client_xact_id = put_id_mapper.io.resp.tl_id, manager_xact_id = UInt(0), addr_beat = UInt(0), data = Bits(0)) + assert(!gnt_arb.io.in(1).valid || put_id_mapper.io.resp.matches, "NASTI tag error") assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") From e15e9c5085fadfb2565f7cdd49dfe091aac0cd12 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 May 2016 00:25:13 -0700 Subject: [PATCH 577/688] First draft of interrupt controller --- uncore/src/main/scala/plic.scala | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 uncore/src/main/scala/plic.scala diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/plic.scala new file mode 100644 index 00000000..128839c3 --- /dev/null +++ b/uncore/src/main/scala/plic.scala @@ -0,0 +1,170 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ +import Chisel.ImplicitConversions._ + +import junctions._ +import cde.Parameters + +class GatewayPLICIO extends Bundle { + val valid = Bool(OUTPUT) + val ready = Bool(INPUT) + val complete = Bool(INPUT) +} + +class LevelGateway extends Module { + val io = new Bundle { + val interrupt = Bool(INPUT) + val plic = new GatewayPLICIO + } + + val inFlight = Reg(init=Bool(false)) + when (io.interrupt && io.plic.ready) { inFlight := true } + when (io.plic.complete) { inFlight := false } + io.plic.valid := io.interrupt && !inFlight +} + +case class PLICConfig(nHartsIn: Int, supervisor: Boolean, nDevices: Int, nPriorities: Int) { + def contextsPerHart = if (supervisor) 2 else 1 + def nHarts = contextsPerHart * nHartsIn + def context(i: Int, mode: Char) = mode match { + case 'M' => i * contextsPerHart + case 'S' => require(supervisor); i * contextsPerHart + 1 + } + def claimAddr(i: Int, mode: Char) = hartBase + hartOffset(context(i, mode)) + claimOffset + def threshAddr(i: Int, mode: Char) = hartBase + hartOffset(context(i, mode)) + def enableAddr(i: Int, mode: Char) = enableBase + enableOffset(context(i, mode)) + def size = hartBase + hartOffset(maxHarts) + + def maxDevices = 1023 + def maxHarts = 992 + def pendingBase = 0x800 + def enableBase = 0x1000 + def enableOffset(i: Int) = i * ((maxDevices+7)/8) + def hartBase = enableBase + enableOffset(maxHarts) + def hartOffset(i: Int) = i * 0x1000 + def claimOffset = 2 + + require(nDevices > 0 && nDevices <= maxDevices) + require(nHarts > 0 && nHarts <= maxHarts) + require(nPriorities > 0 && nPriorities <= nDevices) +} + +/** Platform-Level Interrupt Controller */ +class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new Bundle { + val devices = Vec(cfg.nDevices, new GatewayPLICIO).flip + val harts = Vec(cfg.nHarts, Bool()).asOutput + val tl = new ClientUncachedTileLinkIO().flip + } + + val priority = Reg(Vec(cfg.nDevices+1, UInt(width=log2Up(cfg.nPriorities+1)))) + val pending = Reg(init=Vec.fill(cfg.nDevices+1){Bool(false)}) + val enables = Reg(Vec(cfg.nHarts, UInt(width = cfg.nDevices+1))) + val threshold = Reg(Vec(cfg.nHarts, UInt(width = log2Up(cfg.nPriorities+1)))) + + for ((p, g) <- pending.tail zip io.devices) { + g.ready := !p + g.complete := false + when (g.valid) { p := true } + } + + def findMax(x: Seq[UInt]): (UInt, UInt) = { + if (x.length > 1) { + val half = 1 << (log2Ceil(x.length) - 1) + val lMax = findMax(x take half) + val rMax = findMax(x drop half) + val useLeft = lMax._1 >= rMax._1 + (Mux(useLeft, lMax._1, rMax._1), Mux(useLeft, lMax._2, UInt(half) + rMax._2)) + } else (x.head, UInt(0)) + } + + val maxDevs = Wire(Vec(cfg.nHarts, UInt(width = log2Up(pending.size)))) + for (hart <- 0 until cfg.nHarts) { + val effectivePriority = + for (((p, en), pri) <- pending zip enables(hart).toBools zip priority) + yield Cat(p && en, pri) + val (maxPri, maxDev) = findMax(effectivePriority) + + io.harts(hart) := Reg(next = maxPri > Cat(UInt(1), threshold(hart))) + maxDevs(hart) := maxDev + } + + val acq = Queue(io.tl.acquire, 1) + val read = acq.valid && acq.bits.isBuiltInType(Acquire.getType) + val write = acq.valid && acq.bits.isBuiltInType(Acquire.putType) + assert(!acq.valid || read || write, "unsupported PLIC operation") + val addr = acq.bits.full_addr()(log2Up(cfg.size)-1,0) + + val claimant = + if (cfg.nHarts == 1) UInt(0) + else (addr - cfg.hartBase)(log2Up(cfg.hartOffset(cfg.nHarts))-1,log2Up(cfg.hartOffset(1))) + val hart = Wire(init = claimant) + val myMaxDev = maxDevs(claimant) + UInt(0) // XXX FIRRTL bug w/o the + UInt(0) + val myThresh = Cat(UInt(0, 16-threshold(0).getWidth), threshold(claimant)) + val myEnables = enables(hart) >> 1 << 1 + val rdata_fast = Wire(init = myEnables) + val rdata = Wire(init = rdata_fast) + val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata_fast & ~acq.bits.full_wmask()) + + when (addr >= cfg.hartBase) { + rdata := Cat(myMaxDev, myThresh) + when (read && addr(log2Ceil(cfg.claimOffset))) { + pending(myMaxDev) := false + }.elsewhen (write && acq.bits.wmask()(cfg.claimOffset)) { + val dev = (acq.bits.data >> (8 * cfg.claimOffset))(log2Up(pending.size)-1,0) + when (write && myEnables(dev)) { io.devices(dev-1).complete := true } + }.elsewhen (write) { + val thresh = acq.bits.data(log2Up(pending.size)-1,0) + when (write) { threshold(claimant) := thresh } + } + }.elsewhen (addr >= cfg.enableBase) { + if (cfg.nHarts > 1) + hart := (addr - cfg.enableBase)(log2Up(cfg.enableOffset(cfg.nHarts))-1,log2Up(cfg.enableOffset(1))) + require(enables.size <= tlDataBits) // TODO this can be relaxed + when (write) { enables(hart) := masked_wdata >> 1 << 1 } + }.elsewhen (addr >= cfg.pendingBase) { + for (i <- 0 until pending.size by tlDataBytes) { + val cond = + if (tlDataBytes >= pending.size) Bool(true) + else addr(log2Up(pending.size)-1,log2Up(tlDataBytes)) === i/tlDataBytes + when (cond) { + rdata_fast := Cat(pending.slice(i, i + tlDataBytes).map(p => Cat(UInt(0, 7), p)).reverse) + for (j <- 0 until (tlDataBytes min (pending.size - i))) { + when (write) { pending(i+j) := masked_wdata(j * 8) } + } + } + } + }.otherwise { + val regAddrBits = log2Up(log2Up(cfg.maxDevices+1)) - 3 + val regsPerBeat = tlDataBytes >> regAddrBits + for (i <- 0 until priority.size by regsPerBeat) { + val cond = + if (regsPerBeat >= priority.size) Bool(true) + else addr(log2Up(priority.size)+regAddrBits-1,log2Up(tlDataBytes)) === i/regsPerBeat + when (cond) { + rdata_fast := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 16-p.getWidth), p)).reverse) + for (j <- 0 until (regsPerBeat min (priority.size - i))) { + when (write) { priority(i+j) := masked_wdata >> (j * (8 << regAddrBits)) } + } + } + } + } + + priority(0) := 0 + pending(0) := false + + io.tl.grant.valid := acq.valid + acq.ready := io.tl.grant.ready + io.tl.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = acq.bits.getBuiltInGrantType(), + client_xact_id = acq.bits.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = rdata) +} From 533b2291751abf55fbe2458d8e1533446796f492 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 10 May 2016 14:15:47 -0700 Subject: [PATCH 578/688] Improve PLIC QoR --- uncore/src/main/scala/plic.scala | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/plic.scala index 128839c3..7c1d4287 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/plic.scala @@ -86,12 +86,12 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module val maxDevs = Wire(Vec(cfg.nHarts, UInt(width = log2Up(pending.size)))) for (hart <- 0 until cfg.nHarts) { val effectivePriority = - for (((p, en), pri) <- pending zip enables(hart).toBools zip priority) + for (((p, en), pri) <- (pending zip enables(hart).toBools zip priority).tail) yield Cat(p && en, pri) - val (maxPri, maxDev) = findMax(effectivePriority) + val (_, maxDev) = findMax(Cat(UInt(1), threshold(hart)) +: effectivePriority) - io.harts(hart) := Reg(next = maxPri > Cat(UInt(1), threshold(hart))) - maxDevs(hart) := maxDev + maxDevs(hart) := Reg(next = maxDev) + io.harts(hart) := maxDevs(hart) =/= 0 } val acq = Queue(io.tl.acquire, 1) @@ -105,14 +105,11 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module else (addr - cfg.hartBase)(log2Up(cfg.hartOffset(cfg.nHarts))-1,log2Up(cfg.hartOffset(1))) val hart = Wire(init = claimant) val myMaxDev = maxDevs(claimant) + UInt(0) // XXX FIRRTL bug w/o the + UInt(0) - val myThresh = Cat(UInt(0, 16-threshold(0).getWidth), threshold(claimant)) val myEnables = enables(hart) >> 1 << 1 - val rdata_fast = Wire(init = myEnables) - val rdata = Wire(init = rdata_fast) - val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata_fast & ~acq.bits.full_wmask()) + val rdata = Wire(init = Cat(myMaxDev, UInt(0, 16-threshold(0).getWidth), threshold(claimant))) + val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata & ~acq.bits.full_wmask()) when (addr >= cfg.hartBase) { - rdata := Cat(myMaxDev, myThresh) when (read && addr(log2Ceil(cfg.claimOffset))) { pending(myMaxDev) := false }.elsewhen (write && acq.bits.wmask()(cfg.claimOffset)) { @@ -123,6 +120,7 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module when (write) { threshold(claimant) := thresh } } }.elsewhen (addr >= cfg.enableBase) { + rdata := myEnables if (cfg.nHarts > 1) hart := (addr - cfg.enableBase)(log2Up(cfg.enableOffset(cfg.nHarts))-1,log2Up(cfg.enableOffset(1))) require(enables.size <= tlDataBits) // TODO this can be relaxed @@ -133,7 +131,7 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module if (tlDataBytes >= pending.size) Bool(true) else addr(log2Up(pending.size)-1,log2Up(tlDataBytes)) === i/tlDataBytes when (cond) { - rdata_fast := Cat(pending.slice(i, i + tlDataBytes).map(p => Cat(UInt(0, 7), p)).reverse) + rdata := Cat(pending.slice(i, i + tlDataBytes).map(p => Cat(UInt(0, 7), p)).reverse) for (j <- 0 until (tlDataBytes min (pending.size - i))) { when (write) { pending(i+j) := masked_wdata(j * 8) } } @@ -147,7 +145,7 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module if (regsPerBeat >= priority.size) Bool(true) else addr(log2Up(priority.size)+regAddrBits-1,log2Up(tlDataBytes)) === i/regsPerBeat when (cond) { - rdata_fast := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 16-p.getWidth), p)).reverse) + rdata := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 16-p.getWidth), p)).reverse) for (j <- 0 until (regsPerBeat min (priority.size - i))) { when (write) { priority(i+j) := masked_wdata >> (j * (8 << regAddrBits)) } } From f138819992a12893fc388d7a6eaa544e8b497789 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 11 May 2016 16:44:16 -0700 Subject: [PATCH 579/688] fix order of assignments in ManagerTileLinkNetworkPort --- uncore/src/main/scala/converters.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 8bc52e63..29074b37 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -204,10 +204,10 @@ class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) } io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) - io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) - io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.release.bits.client_id := io.network.release.bits.header.src io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) } From 4f84d8f757cfd6f544f2eaebfbc93f7c53f9e632 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 18 May 2016 13:13:34 -0700 Subject: [PATCH 580/688] make sure to hook up finish in ClientTileLinkEnqueuer --- uncore/src/main/scala/tilelink.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 361e6fb4..82479018 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -987,6 +987,7 @@ class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) ext io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) + io.outer.finish <> (if(depths.fin > 0) Queue(io.inner.finish, depths.fin) else io.inner.finish) } object ClientTileLinkEnqueuer { From d69446e1773c9905f76319a33cb64c134887fcef Mon Sep 17 00:00:00 2001 From: Ken McMillan Date: Tue, 17 May 2016 11:00:08 -0700 Subject: [PATCH 581/688] Add config classes to drive unit testing of L2 TileLink agents. Closes #43 --- uncore/src/main/scala/builder.scala | 114 ++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 uncore/src/main/scala/builder.scala diff --git a/uncore/src/main/scala/builder.scala b/uncore/src/main/scala/builder.scala new file mode 100644 index 00000000..74cb7fa1 --- /dev/null +++ b/uncore/src/main/scala/builder.scala @@ -0,0 +1,114 @@ +package uncore + +import Chisel._ +import cde.{Config, Parameters, ParameterDump, Knob, Dump} +import junctions.PAddrBits + +object UncoreBuilder extends App with FileSystemUtilities { + val topModuleName = args(0) + val configClassName = args(1) + val config = try { + Class.forName(s"uncore.$configClassName").newInstance.asInstanceOf[Config] + } catch { + case e: java.lang.ClassNotFoundException => + throwException("Unable to find configClassName \"" + configClassName + + "\", did you misspell it?", e) + } + val world = config.toInstance + val paramsFromConfig: Parameters = Parameters.root(world) + + val gen = () => + Class.forName(s"uncore.$topModuleName") + .getConstructor(classOf[cde.Parameters]) + .newInstance(paramsFromConfig) + .asInstanceOf[Module] + + chiselMain.run(args.drop(2), gen) + + val pdFile = createOutputFile(s"$topModuleName.prm") + pdFile.write(ParameterDump.getDump) + pdFile.close + +} + +class DefaultL2Config extends Config ( + topDefinitions = { (pname,site,here) => + pname match { + case PAddrBits => 32 + case CacheId => 0 + case CacheName => "L2Bank" + case TLId => "L1toL2" + case InnerTLId => "L1toL2" + case OuterTLId => "L2toMC" + case "N_CACHED" => Dump("N_CACHED",here[Int]("CACHED_CLIENTS_PER_PORT")) + case "N_UNCACHED" => Dump("N_UNCACHED",here[Int]("MAX_CLIENTS_PER_PORT") - here[Int]("N_CACHED")) + case "MAX_CLIENT_XACTS" => 4 + case "MAX_CLIENTS_PER_PORT" => Knob("NTILES") + case "CACHED_CLIENTS_PER_PORT" => Knob("N_CACHED_TILES") + case TLKey("L1toL2") => + TileLinkParameters( + coherencePolicy = new MESICoherence(site(L2DirectoryRepresentation)), + nManagers = 1, + nCachingClients = here[Int]("N_CACHED"), + nCachelessClients = here[Int]("N_UNCACHED"), + maxClientXacts = here[Int]("MAX_CLIENT_XACTS"), + maxClientsPerPort = here[Int]("MAX_CLIENTS_PER_PORT"), + maxManagerXacts = site(NAcquireTransactors) + 2, + dataBits = site(CacheBlockBytes)*8, + dataBeats = 2) + case TLKey("L2toMC") => + TileLinkParameters( + coherencePolicy = new MEICoherence(new NullRepresentation(1)), + nManagers = 1, + nCachingClients = 1, + nCachelessClients = 0, + maxClientXacts = 1, + maxClientsPerPort = site(NAcquireTransactors) + 2, + maxManagerXacts = 1, + dataBits = site(CacheBlockBytes)*8, + dataBeats = 2) + case CacheBlockBytes => 64 + case CacheBlockOffsetBits => log2Up(here(CacheBlockBytes)) + case "L2_SETS" => Knob("L2_SETS") + case NSets => Dump("L2_SETS",here[Int]("L2_SETS")) + case NWays => Knob("L2_WAYS") + case RowBits => site(TLKey(site(TLId))).dataBitsPerBeat + case CacheIdBits => Dump("CACHE_ID_BITS",1) + case L2StoreDataQueueDepth => 1 + case NAcquireTransactors => Dump("N_ACQUIRE_TRANSACTORS",2) + case NSecondaryMisses => 4 + case L2DirectoryRepresentation => new FullRepresentation(here[Int]("N_CACHED")) + case L2Replacer => () => new SeqRandom(site(NWays)) + case ECCCode => None + case AmoAluOperandBits => 64 + case SplitMetadata => false + // case XLen => 128 + }}, + knobValues = { + case "L2_WAYS" => 1 + case "L2_SETS" => 1024 + case "NTILES" => 2 + case "N_CACHED_TILES" => 2 + case "L2_CAPACITY_IN_KB" => 256 + } +) + +class WithPLRU extends Config( + (pname, site, here) => pname match { + case L2Replacer => () => new SeqPLRU(site(NSets), site(NWays)) + }) + +class PLRUL2Config extends Config(new WithPLRU ++ new DefaultL2Config) + +class With1L2Ways extends Config(knobValues = { case "L2_WAYS" => 1 }) +class With2L2Ways extends Config(knobValues = { case "L2_WAYS" => 2 }) +class With4L2Ways extends Config(knobValues = { case "L2_WAYS" => 4 }) + +class With1Cached extends Config(knobValues = { case "N_CACHED_TILES" => 1 }) +class With2Cached extends Config(knobValues = { case "N_CACHED_TILES" => 2 }) + + +class W1Cached1WaysConfig extends Config(new With1L2Ways ++ new With1Cached ++ new DefaultL2Config) +class W1Cached2WaysConfig extends Config(new With2L2Ways ++ new With1Cached ++ new DefaultL2Config) +class W2Cached1WaysConfig extends Config(new With1L2Ways ++ new With2Cached ++ new DefaultL2Config) +class W2Cached2WaysConfig extends Config(new With2L2Ways ++ new With2Cached ++ new DefaultL2Config) From fd83d20857f2a28d24623ce4d51416cd7d3dc8b0 Mon Sep 17 00:00:00 2001 From: Ken McMillan Date: Tue, 17 May 2016 17:13:48 -0700 Subject: [PATCH 582/688] Use a def instead of a lazy val in ManagerCoherenceAgent. Prevents C++ emulator from randomizing inputs in unit testing. Closes #44 --- uncore/src/main/scala/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index e64ad083..98eba33f 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -133,7 +133,7 @@ abstract class ManagerCoherenceAgent(implicit p: Parameters) extends CoherenceAg with HasCoherenceAgentWiringHelpers { val io = new ManagerTLIO def innerTL = io.inner - lazy val outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) + def outerTL = TileLinkIOWrapper(io.outer)(p.alterPartial({case TLId => p(OuterTLId)})) def incoherent = io.incoherent } From 3e0b5d6fd94d1a0ef276e0a5463889e7f8721952 Mon Sep 17 00:00:00 2001 From: Colin Schmidt Date: Thu, 19 May 2016 10:55:00 -0700 Subject: [PATCH 583/688] Ensure that a TSHR doesn't see a valid Acquire if that is blocked by a Release, but would otherwise be allocated. Closes #45 --- uncore/src/main/scala/uncore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 98eba33f..8def2a1c 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -87,7 +87,7 @@ trait HasCoherenceAgentWiringHelpers { val match_ok = matchOverride.getOrElse(Bool(true)) in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && alloc_ok && match_ok outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => - out.valid := in.valid && match_ok + out.valid := in.valid && match_ok && alloc_ok out.bits := in.bits dataOverrides foreach { d => out.bits.data := d(i) } a := alloc_bits(i) & no_matches & alloc_ok From ee0acc1d074c8334e582fb66af9fb7525af7ad8d Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Mon, 23 May 2016 13:19:53 -0700 Subject: [PATCH 584/688] Fix BRAM assertion condition --- uncore/src/main/scala/bram.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index 0176a6b6..f6dc7c74 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -38,7 +38,7 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module val rdata = bram.read(raddr, ren) val wdata = io.acquire.bits.data val wmask = io.acquire.bits.wmask() - assert(!io.acquire.valid || wmask === Fill(tlDataBytes, Bool(true)), + assert(!wen || (wmask === Fill(tlDataBytes, Bool(true))), "bram: subblock writes not supported") when (wen) { bram.write(s0_addr, wdata) From 00d31dc5c5e5702fb3708043215bfedea47fa888 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Tue, 24 May 2016 13:26:26 -0700 Subject: [PATCH 585/688] bram: use new hasti definitions --- uncore/src/main/scala/bram.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index 0176a6b6..7cfc2d8a 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import cde.{Parameters, Field} import junctions._ +import HastiConstants._ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters { @@ -67,8 +68,6 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { val io = new HastiSlaveIO - val hastiDataBytes = hastiDataBits/8 - val wdata = Vec.tabulate(hastiDataBytes)(i => io.hwdata(8*(i+1)-1,8*i)) val waddr = Reg(UInt(width = hastiAddrBits)) val wvalid = Reg(init = Bool(false)) @@ -104,6 +103,6 @@ class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { case ((rbyte, wsel), wbyte) => Mux(wsel && bypass, wbyte, rbyte) }.reverse) - io.hreadyout := Bool(true) + io.hready := Bool(true) io.hresp := HRESP_OKAY } From ace9362d8138d460dff0710a25c547308e8416af Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Thu, 12 May 2016 18:51:02 -0700 Subject: [PATCH 586/688] ahb: amoalu does not need so many parameters! (i want to reuse it) --- uncore/src/main/scala/amoalu.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/amoalu.scala index d0eaecd7..6459d558 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/amoalu.scala @@ -51,8 +51,9 @@ class LoadGen(typ: UInt, addr: UInt, dat: UInt, zero: Bool, maxSize: Int) { def data = genData(0) } -class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends CacheModule()(p) { +class AMOALU(rhsIsAligned: Boolean = false)(implicit p: Parameters) extends Module { val operandBits = p(AmoAluOperandBits) + val blockOffBits = p(CacheBlockOffsetBits) require(operandBits == 32 || operandBits == 64) val io = new Bundle { val addr = Bits(INPUT, blockOffBits) From a012341d96a407045bf6d8e627245548bf1ec33b Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Thu, 12 May 2016 12:18:47 -0700 Subject: [PATCH 587/688] ahb: TileLink => AHB bridge, including atomics and bursts --- uncore/src/main/scala/ahb.scala | 403 ++++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 uncore/src/main/scala/ahb.scala diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/ahb.scala new file mode 100644 index 00000000..1e1a9455 --- /dev/null +++ b/uncore/src/main/scala/ahb.scala @@ -0,0 +1,403 @@ +package uncore + +import Chisel._ +import junctions._ +import cde.{Parameters, Field} +import HastiConstants._ + +/* We need to translate TileLink requests into operations we can actually execute on AHB. + * The general plan of attack is: + * get => one AHB=>TL read + * put => [multiple AHB write fragments=>nill], one AHB write=>TL + * getBlock => AHB burst reads =>TL + * putBlock => AHB burst writes=>TL + * getPrefetch => noop=>TL + * putPrefetch => noop=>TL + * putAtomic => one AHB=>TL read, one idle, one AHB atom_write=>nill, one idle + * + * This requires that we support a pipeline of optional AHB requests with optional TL responses + */ +class AHBRequestIO(implicit p: Parameters) extends HastiMasterIO + with HasGrantType + with HasClientTransactionId + with HasTileLinkBeatId { + val executeAHB = Bool() + val respondTL = Bool() + val latchAtom = Bool() + val firstBurst = Bool() + val finalBurst = Bool() + val cmd = Bits(width = M_SZ) // atomic op +} + +// AHB stage1: translate TileLink Acquires into AHBRequests +class AHBTileLinkIn(implicit val p: Parameters) extends Module + with HasHastiParameters + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new Bundle { + val acquire = new DecoupledIO(new Acquire).flip // NOTE: acquire must be either a Queue or a Pipe + val request = new DecoupledIO(new AHBRequestIO) + } + + // Match the AHB burst with a TileLink {Put,Get}Block + val burstSize = tlDataBeats match { + case 1 => HBURST_SINGLE + // case 2 not supported by AHB + case 4 => HBURST_WRAP4 + case 8 => HBURST_WRAP8 + case 16 => HBURST_WRAP16 + case _ => throw new java.lang.AssertionError("TileLink beats unsupported by AHB") + } + + // Bursts start at 0 and wrap-around back to 0 + val finalBurst = SInt(-1, width = log2Up(tlDataBeats)).asUInt + val firstBurst = UInt(0, width = log2Up(tlDataBeats)) + val next_wmask = Wire(UInt(width = tlDataBytes)) // calculated below + + // State variables for processing more complicated TileLink Acquires + val s_atom_r :: s_atom_idle1 :: s_atom_w :: s_atom_idle2 :: Nil = Enum(UInt(), 4) + val atom_state = Reg(init = s_atom_r) + val done_wmask = Reg(init = UInt(0, width = tlDataBytes)) + val burst = Reg(init = firstBurst) + + // Grab some view of the TileLink acquire + val acq_wmask = io.acquire.bits.wmask() + val isReadBurst = io.acquire.bits.is(Acquire.getBlockType) + val isWriteBurst = io.acquire.bits.is(Acquire.putBlockType) + val isBurst = isWriteBurst || isReadBurst + val isAtomic = io.acquire.bits.is(Acquire.putAtomicType) + val isPut = io.acquire.bits.is(Acquire.putType) + + // Final states? + val last_wmask = next_wmask === acq_wmask + val last_atom = atom_state === s_atom_idle2 + val last_burst = burst === finalBurst + + // Block the incoming request until we've fully consumed it + // NOTE: the outgoing grant.valid may happen while acquire.ready is still false; + // for this reason it is essential to have a Queue or a Pipe infront of acquire + io.acquire.ready := io.request.ready && MuxLookup(io.acquire.bits.a_type, Bool(true), Array( + Acquire.getType -> Bool(true), + Acquire.getBlockType -> last_burst, // hold it until the last beat is burst + Acquire.putType -> last_wmask, // only accept the put if we can fully consume its wmask + Acquire.putBlockType -> Bool(true), + Acquire.putAtomicType -> last_atom, // atomic operation stages complete + Acquire.getPrefetchType -> Bool(true), + Acquire.putPrefetchType -> Bool(true))) + + // Advance the fragment state + when (io.request.ready && io.acquire.valid && isPut) { + when (last_wmask) { // if this was the last fragment, restart FSM + done_wmask := UInt(0) + } .otherwise { + done_wmask := next_wmask + } + } + + // Advance the burst state + // We assume here that TileLink gives us all putBlock beats with nothing between them + when (io.request.ready && io.acquire.valid && isBurst) { + burst := burst + UInt(1) // overflow => wraps around to 0 + } + + // Advance the atomic state machine + when (io.request.ready && io.acquire.valid && isAtomic) { + switch (atom_state) { + is (s_atom_r) { atom_state := s_atom_idle1 } + is (s_atom_idle1) { atom_state := s_atom_w } // idle1 => AMOALU runs on a different clock than AHB slave read + is (s_atom_w) { atom_state := s_atom_idle2 } + is (s_atom_idle2) { atom_state := s_atom_r } // idle2 state is required by AHB after hmastlock is lowered + } + } + + // Returns (range=0, range=-1, aligned_wmask, size) + def mask_helper(in_0 : Bool, range : UInt): (Bool, Bool, UInt, UInt) = { + val len = range.getWidth + if (len == 1) { + (range === UInt(0), range === UInt(1), in_0.asUInt() & range, UInt(0)) + } else { + val mid = len / 2 + val lo = range(mid-1, 0) + val hi = range(len-1, mid) + val (lo_0, lo_1, lo_m, lo_s) = mask_helper(in_0, lo) + val (hi_0, hi_1, hi_m, hi_s) = mask_helper(in_0 && lo_0, hi) + val out_0 = lo_0 && hi_0 + val out_1 = lo_1 && hi_1 + val out_m = Cat(hi_m, lo_m) | Fill(len, (in_0 && out_1).asUInt()) + val out_s = Mux(out_1, UInt(log2Up(len)), Mux(lo_0, hi_s, lo_s)) + (out_0, out_1, out_m, out_s) + } + } + + val pending_wmask = acq_wmask & ~done_wmask + val put_addr = PriorityEncoder(pending_wmask) + val (wmask_0, _, exec_wmask, put_size) = mask_helper(Bool(true), pending_wmask) + next_wmask := done_wmask | exec_wmask + + // Calculate the address, with consideration to put fragments and bursts + val addr_block = io.acquire.bits.addr_block + val addr_beat = io.acquire.bits.addr_beat + val addr_burst = Mux(isReadBurst, addr_beat + burst, addr_beat) + val addr_byte = Mux(isPut, put_addr, io.acquire.bits.addr_byte()) + val ahbAddr = Cat(addr_block, addr_burst, addr_byte) + val ahbSize = Mux(isPut, put_size, Mux(isBurst, UInt(log2Up(tlDataBytes)), io.acquire.bits.op_size())) + + val ahbBurst = MuxLookup(io.acquire.bits.a_type, HBURST_SINGLE, Array( + Acquire.getType -> HBURST_SINGLE, + Acquire.getBlockType -> burstSize, + Acquire.putType -> HBURST_SINGLE, + Acquire.putBlockType -> burstSize, + Acquire.putAtomicType -> HBURST_SINGLE, + Acquire.getPrefetchType -> HBURST_SINGLE, + Acquire.putPrefetchType -> HBURST_SINGLE)) + + val ahbWrite = MuxLookup(io.acquire.bits.a_type, Bool(false), Array( + Acquire.getType -> Bool(false), + Acquire.getBlockType -> Bool(false), + Acquire.putType -> Bool(true), + Acquire.putBlockType -> Bool(true), + Acquire.putAtomicType -> MuxLookup(atom_state, Bool(false), Array( + s_atom_r -> Bool(false), + s_atom_idle1 -> Bool(false), // don't care + s_atom_w -> Bool(true), + s_atom_idle2 -> Bool(true))), // don't care + Acquire.getPrefetchType -> Bool(false), // don't care + Acquire.putPrefetchType -> Bool(true))) // don't care + + val ahbExecute = MuxLookup(io.acquire.bits.a_type, Bool(false), Array( + Acquire.getType -> Bool(true), + Acquire.getBlockType -> Bool(true), + Acquire.putType -> !wmask_0, // handle the case of a Put with no bytes! + Acquire.putBlockType -> Bool(true), + Acquire.putAtomicType -> MuxLookup(atom_state, Bool(false), Array( + s_atom_r -> Bool(true), + s_atom_idle1 -> Bool(false), + s_atom_w -> Bool(true), + s_atom_idle2 -> Bool(false))), + Acquire.getPrefetchType -> Bool(false), + Acquire.putPrefetchType -> Bool(false))) + + val respondTL = MuxLookup(io.acquire.bits.a_type, Bool(false), Array( + Acquire.getType -> Bool(true), + Acquire.getBlockType -> Bool(true), + Acquire.putType -> last_wmask, + Acquire.putBlockType -> Bool(true), + Acquire.putAtomicType -> MuxLookup(atom_state, Bool(false), Array( + s_atom_r -> Bool(true), // they want the old data + s_atom_idle1 -> Bool(false), + s_atom_w -> Bool(false), + s_atom_idle2 -> Bool(false))), + Acquire.getPrefetchType -> Bool(true), + Acquire.putPrefetchType -> Bool(true))) + + io.request.valid := io.acquire.valid + io.request.bits.htrans := HTRANS_IDLE // unused/ignored + io.request.bits.haddr := ahbAddr + io.request.bits.hmastlock := isAtomic && atom_state =/= s_atom_idle2 + io.request.bits.hwrite := ahbWrite + io.request.bits.hburst := ahbBurst + io.request.bits.hsize := ahbSize + io.request.bits.hprot := HPROT_DATA | HPROT_PRIVILEGED + io.request.bits.hwdata := io.acquire.bits.data + io.request.bits.executeAHB := ahbExecute + io.request.bits.respondTL := respondTL + io.request.bits.latchAtom := isAtomic && atom_state === s_atom_r + io.request.bits.firstBurst := burst === firstBurst + io.request.bits.finalBurst := burst === finalBurst || !isBurst + io.request.bits.cmd := io.acquire.bits.op_code() + io.request.bits.is_builtin_type := Bool(true) + io.request.bits.g_type := io.acquire.bits.getBuiltInGrantType() + io.request.bits.client_xact_id := io.acquire.bits.client_xact_id + io.request.bits.addr_beat := addr_burst + + val debugBurst = Reg(UInt()) + debugBurst := addr_burst - burst + + // We only support built-in TileLink requests + assert(!io.acquire.valid || io.acquire.bits.is_builtin_type, "AHB bridge only supports builtin TileLink types") + // Ensure alignment of address to size + assert(!io.acquire.valid || (ahbAddr & ((UInt(1) << ahbSize) - UInt(1))) === UInt(0), "TileLink operation misaligned") + // If this is a putBlock, make sure it moves properly + assert(!io.acquire.valid || !isBurst || burst === firstBurst || debugBurst === addr_burst - burst, "TileLink putBlock beats not sequential") + // We better not get an incomplete TileLink acquire + assert(!io.acquire.valid || isBurst || burst === firstBurst, "TileLink never completed a putBlock") +} + +// AHB stage2: execute AHBRequests +class AHBBusMaster(implicit val p: Parameters) extends Module + with HasHastiParameters + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new Bundle { + val request = new DecoupledIO(new AHBRequestIO).flip + val grant = new DecoupledIO(new Grant) + val ahb = new HastiMasterIO() + } + + // All AHB outputs are registered (they might be IOs) + val midBurst = Reg(init = Bool(false)) + val htrans = Reg(init = HTRANS_IDLE) + val haddr = Reg(UInt()) + val hmastlock = Reg(init = Bool(false)) + val hwrite = Reg(Bool()) + val hburst = Reg(UInt()) + val hsize = Reg(UInt()) + val hprot = Reg(UInt()) + val hwdata0 = Reg(Bits()) + val hwdata1 = Reg(Bits()) + val hrdata = Reg(Bits()) + + io.ahb.htrans := htrans + io.ahb.haddr := haddr + io.ahb.hmastlock := hmastlock + io.ahb.hwrite := hwrite + io.ahb.hburst := hburst + io.ahb.hsize := hsize + io.ahb.hprot := hprot + io.ahb.hwdata := hwdata1 // one cycle after the address phase + + // TileLink response data needed in data phase + val respondTL0 = Reg(init = Bool(false)) + val respondTL1 = Reg(init = Bool(false)) + val latchAtom0 = Reg(init = Bool(false)) + val latchAtom1 = Reg(init = Bool(false)) + val bubble = Reg(init = Bool(true)) // nothing useful in address phase + val cmd = Reg(Bits()) + val g_type0 = Reg(UInt()) + val g_type1 = Reg(UInt()) + val client_xact_id0 = Reg(Bits()) + val client_xact_id1 = Reg(Bits()) + val addr_beat0 = Reg(UInt()) + val addr_beat1 = Reg(UInt()) + val grant1 = Reg(new Grant) + + // It is allowed to progress from Idle/Busy during a wait state + val addrReady = io.ahb.hready || bubble + val dataReady = io.ahb.hready + + // Only accept a new AHBRequest if we have enough buffer space in the pad + // to accomodate a persistent drop in TileLink's grant.ready + io.request.ready := addrReady && io.grant.ready + + // htrans must be updated even if no request is valid + when (addrReady) { + when (io.request.fire() && io.request.bits.executeAHB) { + midBurst := !io.request.bits.finalBurst + when (io.request.bits.firstBurst) { + htrans := HTRANS_NONSEQ + } .otherwise { + htrans := HTRANS_SEQ + } + } .otherwise { + when (midBurst) { + htrans := HTRANS_BUSY + } .otherwise { + htrans := HTRANS_IDLE + } + } + } + + // Address phase, clear repondTL when we have nothing to do + when (addrReady) { + when (io.request.fire()) { + respondTL0 := io.request.bits.respondTL + latchAtom0 := io.request.bits.latchAtom + bubble := Bool(false) + } .otherwise { + respondTL0 := Bool(false) + latchAtom0 := Bool(false) + bubble := Bool(true) // an atom-injected Idle is not a bubble! + } + } + + // Transfer bulk address phase + when (io.request.fire()) { + haddr := io.request.bits.haddr + hmastlock := io.request.bits.hmastlock + hwrite := io.request.bits.hwrite + hburst := io.request.bits.hburst + hsize := io.request.bits.hsize + hprot := io.request.bits.hprot + hwdata0 := io.request.bits.hwdata + cmd := io.request.bits.cmd + g_type0 := io.request.bits.g_type + client_xact_id0 := io.request.bits.client_xact_id + addr_beat0 := io.request.bits.addr_beat + } + + // Execute Atomic ops + val amo_p = p.alterPartial({ + case CacheBlockOffsetBits => hastiAddrBits + case AmoAluOperandBits => hastiDataBits + }) + val alu = Module(new AMOALU(rhsIsAligned = false)(amo_p)) + alu.io.addr := haddr + alu.io.cmd := cmd + alu.io.typ := hsize + alu.io.rhs := hwdata0 + alu.io.lhs := hrdata + + // Transfer bulk data phase + // NOTE: this introduces no bubbles because addrReady is a superset of dataReady + when (dataReady) { + hwdata1 := alu.io.out // hwdata1 := hwdata0 + respondTL1 := respondTL0 + latchAtom1 := latchAtom0 + g_type1 := g_type0 + client_xact_id1 := client_xact_id0 + addr_beat1 := addr_beat0 + } + + // Latch the read result for an atomic operation + when (dataReady && latchAtom1) { + hrdata := io.ahb.hrdata + } + + // Only issue TL grant when the slave has provided data + io.grant.valid := dataReady && respondTL1 + io.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = g_type1, + client_xact_id = client_xact_id1, + manager_xact_id = UInt(0), + addr_beat = addr_beat1, + data = io.ahb.hrdata) + + // We cannot support errors from AHB to TileLink + assert(!io.ahb.hresp, "AHB hresp error detected and cannot be reported via TileLink") +} + +class AHBBridge(implicit val p: Parameters) extends Module + with HasHastiParameters + with HasTileLinkParameters + with HasAddrMapParameters { + val io = new Bundle { + val tl = new ClientUncachedTileLinkIO().flip + val ahb = new HastiMasterIO() + } + + // Hasti and TileLink widths must agree at this point in the topology + require (tlDataBits == hastiDataBits) + require (p(PAddrBits) == hastiAddrBits) + + // AHB does not permit bursts to cross a 1KB boundary + require (tlDataBits * tlDataBeats <= 1024*8) + // tlDataBytes must be a power of 2 + require (1 << log2Up(tlDataBytes) == tlDataBytes) + + // Create the sub-blocks + val fsm = Module(new AHBTileLinkIn) + val bus = Module(new AHBBusMaster) + val pad = Module(new Queue(new Grant, 4)) + + fsm.io.acquire <> Queue(io.tl.acquire, 2) // Pipe is also acceptable + bus.io.request <> fsm.io.request + io.ahb <> bus.io.ahb + io.tl.grant <> pad.io.deq + + // The pad is needed to absorb AHB progress while !grant.ready + // We are only 'ready' if the pad has at least 3 cycles of space + bus.io.grant.ready := pad.io.count <= UInt(1) + pad.io.enq.bits := bus.io.grant.bits + pad.io.enq.valid := bus.io.grant.valid +} From 88cc91db75daeb79f727b6f07f9af5ae0fc681b9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 24 May 2016 15:46:51 -0700 Subject: [PATCH 588/688] Ignore way_en in MetadataArray for direct-mapped caches --- uncore/src/main/scala/cache.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index e30cf486..27d96ab8 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -149,8 +149,8 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext val rst = rst_cnt < UInt(nSets) val waddr = Mux(rst, rst_cnt, io.write.bits.idx) val wdata = Mux(rst, rstVal, io.write.bits.data).toBits - val wmask = Mux(rst, SInt(-1), io.write.bits.way_en.toSInt).toBools - val rmask = Mux(rst, SInt(-1), io.read.bits.way_en.toSInt).toBools + val wmask = Mux(rst || Bool(nWays == 1), SInt(-1), io.write.bits.way_en.toSInt).toBools + val rmask = Mux(rst || Bool(nWays == 1), SInt(-1), io.read.bits.way_en.toSInt).toBools when (rst) { rst_cnt := rst_cnt+UInt(1) } val metabits = rstVal.getWidth From 7f1792cba3c006ec36df9c22c1bc95328952459f Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 25 May 2016 12:25:59 -0700 Subject: [PATCH 589/688] ahb: backport bridge to chisel2 Closes #47 --- uncore/src/main/scala/ahb.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/ahb.scala index 1e1a9455..80442dbe 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/ahb.scala @@ -241,7 +241,7 @@ class AHBBusMaster(implicit val p: Parameters) extends Module val hmastlock = Reg(init = Bool(false)) val hwrite = Reg(Bool()) val hburst = Reg(UInt()) - val hsize = Reg(UInt()) + val hsize = Reg(init = UInt(0, width = SZ_HSIZE)) val hprot = Reg(UInt()) val hwdata0 = Reg(Bits()) val hwdata1 = Reg(Bits()) From 3e238adc67e6e38a23c9cdda39f1ed2709fa767f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 25 May 2016 20:37:48 -0700 Subject: [PATCH 590/688] rtc: fix acquire message type check --- uncore/src/main/scala/rtc.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala index 8c05d418..d2940578 100644 --- a/uncore/src/main/scala/rtc.scala +++ b/uncore/src/main/scala/rtc.scala @@ -26,8 +26,8 @@ class RTC(nHarts: Int)(implicit val p: Parameters) extends Module val addr = full_addr(log2Up(size)-1,log2Up(w/8)) val rdata = regs(addr) val wdata = acq.bits.data - val read = acq.bits.a_type === Acquire.getType - val write = acq.bits.a_type === Acquire.putType + val read = acq.bits.isBuiltInType(Acquire.getType) + val write = acq.bits.isBuiltInType(Acquire.putType) val wmask = acq.bits.full_wmask() assert(!acq.valid || read || write, "unsupported RTC operation") From e2755a0f0aa0d4a5d2a69394704651e3ca742ef0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 25 May 2016 20:39:53 -0700 Subject: [PATCH 591/688] Work around zero-width wire limitation in HTIF --- uncore/src/main/scala/htif.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala index 5ffffd11..82c75a30 100644 --- a/uncore/src/main/scala/htif.scala +++ b/uncore/src/main/scala/htif.scala @@ -150,12 +150,14 @@ class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHt val n = dataBits/short_request_bits val mem_req_data = (0 until n).map { i => - val ui = UInt(i, log2Up(n)) + def addr(offset: UInt) = + if (dataBits == short_request_bits) offset + else Cat(offset, UInt(i, log2Up(n))) when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(Cat(io.mem.grant.bits.addr_beat, ui)) := + packet_ram(addr(io.mem.grant.bits.addr_beat)) := io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) } - packet_ram(Cat(cnt, ui)) + packet_ram(addr(cnt)) }.reverse.reduce(_##_) val init_addr = addr.toUInt >> (offsetBits-3) From 22568de5f38af444d9daf430db965a3af9be02ee Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 25 May 2016 21:42:02 -0700 Subject: [PATCH 592/688] Work around another zero-width wire limitation --- uncore/src/main/scala/tilelink.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 82479018..f6a497b2 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -170,16 +170,15 @@ trait HasAcquireUnion extends HasTileLinkParameters { def op_size(dummy: Int = 0) = union(addrByteOff-1, opSizeOff) /** Byte address for [[uncore.PutAtomic]] operand */ def addr_byte(dummy: Int = 0) = union(addrByteMSB-1, addrByteOff) - def amo_offset(dummy: Int = 0) = addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBytes)) + def amo_offset(dummy: Int = 0) = + if (tlByteAddrBits > log2Up(amoAluOperandBytes)) addr_byte()(tlByteAddrBits-1, log2Up(amoAluOperandBytes)) + else UInt(0) /** Bit offset of [[uncore.PutAtomic]] operand */ def amo_shift_bytes(dummy: Int = 0) = UInt(amoAluOperandBytes)*amo_offset() /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0): UInt = { - val amo_word_mask = - if (amoAluOperandBytes == tlWriteMaskBits) UInt(1) - else UIntToOH(amo_offset()) Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBytes, amo_word_mask), + FillInterleaved(amoAluOperandBytes, UIntToOH(amo_offset())), Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), union(tlWriteMaskBits, 1), UInt(0, width = tlWriteMaskBits))) From 8139f71dfb1c13593b2032298f03e2250422edfc Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 May 2016 12:37:31 -0700 Subject: [PATCH 593/688] Work around Chisel2 bug This code is correct, but Chisel2 erroneously flags it as a Chisel3 compatibility error because it looks like Vec(Reg) when factor=1. --- uncore/src/main/scala/converters.scala | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 29074b37..129835d1 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -878,9 +878,6 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) val gnt_client_id = Reg(UInt(width = outerClientIdBits)) val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) val gnt_data = Reg(UInt(width = outerDataBits)) - val gnt_data_vec = Vec.tabulate(factor) { i => - gnt_data(innerDataBits * (i + 1) - 1, innerDataBits * i) - } when (io.out.grant.fire() && stretch) { gnt_data := ognt.data @@ -892,12 +889,8 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) when (send_done) { returning_data := Bool(false) } - def select_data(data: UInt, sel: UInt): UInt = { - val data_vec = Vec.tabulate(factor) { i => - data(innerDataBits * (i + 1) - 1, innerDataBits * i) - } - data_vec(sel) - } + def select_data(data: UInt, sel: UInt): UInt = + data >> (sel << log2Up(innerDataBits)) val gnt_switch = smallget_switch(ognt.client_xact_id) @@ -907,7 +900,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) client_xact_id = gnt_client_id, manager_xact_id = gnt_manager_id, addr_beat = Cat(gnt_beat, send_idx), - data = gnt_data_vec(send_idx)) + data = select_data(gnt_data, send_idx)) val get_grant = Grant( is_builtin_type = Bool(true), From b6d26e90f844cb92fd8029ac9b1975ffb5dee2fc Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 May 2016 15:59:42 -0700 Subject: [PATCH 594/688] Add generic TileLink width adapter --- uncore/src/main/scala/converters.scala | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 129835d1..167bebb7 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -747,6 +747,24 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.b.ready, io.nasti.r.ready) } +object TileLinkWidthAdapter { + def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { + require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) + + if (out.tlDataBits > in.tlDataBits) { + val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + widener.io.in <> in + out <> widener.io.out + } else if (out.tlDataBits < in.tlDataBits) { + val narrower = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + narrower.io.in <> in + out <> narrower.io.out + } else { + out <> in + } + } +} + class TileLinkIOWidener(innerTLId: String, outerTLId: String) (implicit p: Parameters) extends TLModule()(p) { From 391a9b91100869b41e706c7a662c6a062e4528ce Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 May 2016 16:10:42 -0700 Subject: [PATCH 595/688] Use buses, rather than crossbars, by default in TLInterconnect We should eventually parameterize this, of course. --- uncore/src/main/scala/interconnect.scala | 10 +++++----- uncore/src/main/scala/network.scala | 25 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index 88791815..c9fdb478 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -110,11 +110,11 @@ class PortedTileLinkCrossbar( val phyHdrWidth = log2Up(n) val count = tlDataBeats // Actually instantiate the particular networks required for TileLink - val acqNet = Module(new BasicCrossbar(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData()))) - val relNet = Module(new BasicCrossbar(n, new Release, count, Some((r: PhysicalNetworkIO[Release]) => r.payload.hasMultibeatData()))) - val prbNet = Module(new BasicCrossbar(n, new Probe)) - val gntNet = Module(new BasicCrossbar(n, new Grant, count, Some((g: PhysicalNetworkIO[Grant]) => g.payload.hasMultibeatData()))) - val ackNet = Module(new BasicCrossbar(n, new Finish)) + val acqNet = Module(new BasicBus(CrossbarConfig(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData())))) + val relNet = Module(new BasicBus(CrossbarConfig(n, new Release, count, Some((r: PhysicalNetworkIO[Release]) => r.payload.hasMultibeatData())))) + val prbNet = Module(new BasicBus(CrossbarConfig(n, new Probe))) + val gntNet = Module(new BasicBus(CrossbarConfig(n, new Grant, count, Some((g: PhysicalNetworkIO[Grant]) => g.payload.hasMultibeatData())))) + val ackNet = Module(new BasicBus(CrossbarConfig(n, new Finish))) // Aliases for the various network IO bundle types type PNIO[T <: Data] = DecoupledIO[PhysicalNetworkIO[T]] diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala index 64eff1d3..88f1a1b6 100644 --- a/uncore/src/main/scala/network.scala +++ b/uncore/src/main/scala/network.scala @@ -19,19 +19,34 @@ class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { } class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { - val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip - val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) + val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip + val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) } abstract class PhysicalNetwork extends Module -class BasicCrossbar[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) extends PhysicalNetwork { - val io = new BasicCrossbarIO(n, dType) +case class CrossbarConfig[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) +abstract class AbstractCrossbar[T <: Data](conf: CrossbarConfig[T]) extends PhysicalNetwork { + val io = new BasicCrossbarIO(conf.n, conf.dType) +} + +class BasicBus[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { + val arb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) + arb.io.in <> io.in + + arb.io.out.ready := io.out(arb.io.out.bits.header.dst).ready + for ((out, i) <- io.out zipWithIndex) { + out.valid := arb.io.out.valid && arb.io.out.bits.header.dst === UInt(i) + out.bits := arb.io.out.bits + } +} + +class BasicCrossbar[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { io.in.foreach { _.ready := Bool(false) } io.out.zipWithIndex.map{ case (out, i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, n, count, needsLock)) + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { val destined = in.bits.header.dst === UInt(i) arb.valid := in.valid && destined From 8afdd7e3da810796bf9b1d661600a1ac6b0d11f0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 26 May 2016 23:05:44 -0700 Subject: [PATCH 596/688] Work around PutBlocks draining into data array prematurely --- uncore/src/main/scala/cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 27d96ab8..d5efde13 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -952,6 +952,7 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra val curr_write_beat = PriorityEncoder(pending_writes) io.data.write.valid := state === s_busy && pending_writes.orR && + !pending_puts.orR && !pending_ognt && !pending_reads(curr_write_beat) && !pending_resps(curr_write_beat) From 6d82c0d1569f29b3ec1fcf7213d03027bfb7f2b7 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Tue, 31 May 2016 19:25:31 -0700 Subject: [PATCH 597/688] Add M_FLUSH_ALL command --- uncore/src/main/scala/consts.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/consts.scala index ed6db122..14904463 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/consts.scala @@ -26,7 +26,7 @@ trait MemoryOpConstants { val M_PFR = UInt("b00010"); // prefetch with intent to read val M_PFW = UInt("b00011"); // prefetch with intent to write val M_XA_SWAP = UInt("b00100"); - val M_NOP = UInt("b00101"); + val M_FLUSH_ALL = UInt("b00101") // flush all lines val M_XLR = UInt("b00110"); val M_XSC = UInt("b00111"); val M_XA_ADD = UInt("b01000"); From e8408f0a8a8059751618d6b3224a4de05a5e11be Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 1 Jun 2016 10:33:59 -0700 Subject: [PATCH 598/688] fix HastiRAM --- uncore/src/main/scala/bram.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index 76af2802..480f4a73 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -76,15 +76,13 @@ class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { val max_wsize = log2Ceil(hastiDataBytes) val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, - (0 until max_wsize).map(1 << _).map(sz => (UInt(sz) -> UInt((1 << sz << sz) - 1)))) + (0 until max_wsize).map(sz => (UInt(sz) -> UInt((1 << (1 << sz)) - 1)))) val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) val raddr = io.haddr >> UInt(2) val ren = is_trans && !io.hwrite val bypass = Reg(init = Bool(false)) - val last_wdata = Reg(next = wdata) - val last_wmask = Reg(next = wmask) when (is_trans && io.hwrite) { waddr := io.haddr From 740a6073f6b7cde65d014bba3eb32e6df4c9c7a7 Mon Sep 17 00:00:00 2001 From: mwachs5 Date: Wed, 1 Jun 2016 16:33:34 -0700 Subject: [PATCH 599/688] Add Debug Module (#49) * Add Debug Module * [debug] Remove unit tests, update System Bus register addresses, parameterize nComponents * [debug] Update Debug ROM contents to match updated addresses --- uncore/src/main/scala/debug.scala | 962 ++++++++++++++++++++++++++++++ 1 file changed, 962 insertions(+) create mode 100644 uncore/src/main/scala/debug.scala diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala new file mode 100644 index 00000000..e4f96765 --- /dev/null +++ b/uncore/src/main/scala/debug.scala @@ -0,0 +1,962 @@ +// See LICENSE for license details. + +package uncore + +import Chisel._ + +import junctions._ +import cde.{Parameters, Config, Field} +import Math.{ceil, max, min} + +// ***************************************** +// Constants which are interesting even +// outside of this module +// ***************************************** + +object DbRegAddrs{ + + def DMCONTROL = UInt(0x10) + + def DMINFO = UInt(0x11) + def AUTHDATA0 = UInt(0x12) + def AUTHDATA1 = UInt(0x13) + def SERDATA = UInt(0x14) + def SERSTATUS = UInt(0x15) + def SBUSADDRESS0 = UInt(0x16) + def SBUSADDRESS1 = UInt(0x17) + def SBDATA0 = UInt(0x18) + def SBDATA1 = UInt(0x19) + //1a + def HALTSUM = UInt(0x1B) + //1c - 3b are the halt notification registers. + def SBADDRESS2 = UInt(0x3d) + // 3c + def SBDATA2 = UInt(0x3e) + def SBDATA3 = UInt(0x3f) +} + +/** Constant values used by both Debug Bus Response & Request + */ + +object DbBusConsts{ + + def dbDataSize = 34 + + def dbRamWordBits = 32 + + def dbOpSize = 2 + def db_OP_NONE = UInt("b00") + def db_OP_READ = UInt("b01") + def db_OP_READ_WRITE = UInt("b10") + def db_OP_READ_COND_WRITE = UInt("b11") + + def dbRespSize = 2 + def db_RESP_SUCCESS = UInt("b00") + def db_RESP_FAILURE = UInt("b01") + def db_RESP_HW_FAILURE = UInt("b10") + // This is used outside this block + // to indicate 'busy'. + def db_RESP_RESERVED = UInt("b11") + +} + +object DsbBusConsts { + + def sbAddrWidth = 12 + def sbIdWidth = 10 + + /* These are the default ROM contents, which support + * RV32 and RV64. RV128 are implemented as NOP. + * See the Debug Specification for the code. + */ + + def defaultRomContents : Array[Byte] = Array(0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0xf3, 0x24, 0x40, 0xf1, + 0x23, 0x20, 0x90, 0x10, 0x0f, 0x00, 0xf0, 0x0f, 0xf3, 0x24, 0x00, 0xf1, + 0x63, 0xc6, 0x04, 0x00, 0x83, 0x24, 0xc0, 0x43, 0x6f, 0x00, 0xc0, 0x01, + 0x93, 0x94, 0x14, 0x00, 0x63, 0xc6, 0x04, 0x00, 0x83, 0x34, 0x80, 0x43, + 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x00, 0x00, 0x00, 0x23, 0x2e, 0x80, 0x42, + 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, 0x63, 0x04, 0x04, 0x00, + 0x6f, 0x00, 0x80, 0x05, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, + 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, + 0x13, 0x04, 0xd4, 0xff, 0x63, 0x18, 0x04, 0x02, 0x0f, 0x10, 0x00, 0x00, + 0x73, 0x24, 0x00, 0xf1, 0x63, 0x46, 0x04, 0x00, 0x23, 0x2e, 0x90, 0x42, + 0x67, 0x00, 0x00, 0x40, 0x13, 0x14, 0x14, 0x00, 0x63, 0x46, 0x04, 0x00, + 0x23, 0x3c, 0x90, 0x42, 0x67, 0x00, 0x00, 0x40, 0x13, 0x00, 0x00, 0x00, + 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, + 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, + 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0xdf, 0xfb).map(x => x.toByte) + +} + + + +object DsbRegAddrs{ + + def CLEARDEBINT = UInt(0x100) + def SETHALTNOT = UInt(0x10C) + def SERINFO = UInt(0x110) + def SERBASE = UInt(0x114) + // For each serial, there are + // 3 registers starting here: + // SERSEND0 + // SERRECEIVE0 + // SERSTATUS0 + // ... + // SERSTATUS7 + def SERTX_OFFSET = UInt(0) + def SERRX_OFFSET = UInt(4) + def SERSTAT_OFFSET = UInt(8) + +} + + +// ***************************************** +// Configuration & Parameters for this Module +// +// ***************************************** + +/** Enumerations used both in the hardware + * and in the configuration specification. + */ + +object DebugModuleAuthType extends scala.Enumeration { + type DebugModuleAuthType = Value + val None, Password, ChallengeResponse, Reserved = Value +} +import DebugModuleAuthType._ + +object DebugModuleAccessType extends scala.Enumeration { + type DebugModuleAccessType = Value + val Access8Bit, Access16Bit, Access32Bit, Access64Bit, Access128Bit = Value +} +import DebugModuleAccessType._ + + +/** Parameters exposed to the top-level design, set based on + * external requirements, etc. + * + * This object checks that the parameters conform to the + * full specification. The implementation which receives this + * object can perform more checks on what that implementation + * actually supports. + * nComponents : The number of components to support debugging. + * nDebugBusAddrSize : Size of the Debug Bus Address + * nDebugRam Bytes: Size of the Debug RAM (depends on the XLEN of the machine). + * debugRomContents: + * hasBusMaster: Whether or not a bus master should be included + * The size of the accesses supported by the Bus Master. + * nSerialPorts : Number of serial ports to instantiate + * authType : The Authorization Type + * Number of cycles to assert ndreset when pulsed. + **/ + + +case class DebugModuleConfig ( + nComponents : Int, + nDebugBusAddrSize : Int, + nDebugRamBytes : Int, + debugRomContents : Option[Seq[Byte]], + hasBusMaster : Boolean, + hasAccess128 : Boolean, + hasAccess64 : Boolean, + hasAccess32 : Boolean, + hasAccess16 : Boolean, + hasAccess8 : Boolean, + nSerialPorts : Int, + authType : DebugModuleAuthType, + nNDResetCycles : Int +) { + + if (hasBusMaster == false){ + require (hasAccess128 == false) + require (hasAccess64 == false) + require (hasAccess32 == false) + require (hasAccess16 == false) + require (hasAccess8 == false) + } + + require (nSerialPorts <= 8) + + require ((nDebugBusAddrSize >= 5) && (nDebugBusAddrSize <= 7)) + + private val maxComponents = nDebugBusAddrSize match { + case 5 => (32*4) + case 6 => (32*32) + case 7 => (32*32) + } + require (nComponents > 0 && nComponents <= maxComponents) + + private val maxRam = nDebugBusAddrSize match { + case 5 => (4 * 16) + case 6 => (4 * 16) + case 7 => (4 * 64) + } + + require (nDebugRamBytes > 0 && nDebugRamBytes <= maxRam) + + val hasHaltSum = (nComponents > 64) || (nSerialPorts > 0) + + val hasDebugRom = debugRomContents match{ + case Some(_) => true + case None => false + } + + if (hasDebugRom) { + require (debugRomContents.size > 0) + require (debugRomContents.size <= 512) + } + + require (nNDResetCycles > 0) + +} + +class DefaultDebugModuleConfig (val ncomponents : Int, val xlen:Int) + extends DebugModuleConfig( + nComponents = ncomponents, + nDebugBusAddrSize = 5, + nDebugRamBytes = xlen match{ + case 32 => 28 + case 64 => 40 + case 128 => 64 + }, + debugRomContents = Some(DsbBusConsts.defaultRomContents), + hasBusMaster = false, + hasAccess128 = false, + hasAccess64 = false, + hasAccess32 = false, + hasAccess16 = false, + hasAccess8 = false, + nSerialPorts = 0, + authType = DebugModuleAuthType.None, + nNDResetCycles = 1) + +case object DMKey extends Field[DebugModuleConfig] + + +// ***************************************** +// Module Interfaces +// +// ***************************************** + + +/** Structure to define the contents of a Debug Bus Request + */ + +class DebugBusReq(addrBits : Int) extends Bundle { + val addr = UInt(width = addrBits) + val op = UInt(width = DbBusConsts.dbOpSize) + val data = UInt(width = DbBusConsts.dbDataSize) + + override def cloneType = new DebugBusReq(addrBits).asInstanceOf[this.type] +} + + +/** Structure to define the contents of a Debug Bus Response + */ +class DebugBusResp( ) extends Bundle { + val resp = UInt(width = DbBusConsts.dbRespSize) + val data = UInt(width = DbBusConsts.dbDataSize) +} + +/** Structure to define the top-level DebugBus interface + * of DebugModule. + * DebugModule is the consumer of this interface. + * Therefore it has the 'flipped' version of this. + */ + +class DebugBusIO(implicit val p: cde.Parameters) extends ParameterizedBundle()(p) { + val req = new DecoupledIO(new DebugBusReq(p(DMKey).nDebugBusAddrSize)) + val resp = new DecoupledIO(new DebugBusResp).flip() +} + +// ***************************************** +// The Module +// +// ***************************************** + +/** Parameterized version of the Debug Module defined in the + * RISC-V Debug Specification + * + * DebugModule is a slave to two masters: + * The Debug Bus -- implemented as a generic Decoupled IO with request + * and response channels + * The System Bus -- implemented as Uncached Tile Link. + * + * DebugModule is responsible for holding registers, RAM, and ROM + * to support debug interactions, as well as driving interrupts + * to a configurable number of components in the system. + * It is also responsible for some reset lines. + */ + +class DebugModule ()(implicit val p:cde.Parameters) + extends Module + with HasTileLinkParameters { + val cfg = p(DMKey) + + //-------------------------------------------------------------- + // Import constants for shorter variable names + //-------------------------------------------------------------- + + import DbRegAddrs._ + import DsbRegAddrs._ + import DsbBusConsts._ + import DbBusConsts._ + + //-------------------------------------------------------------- + // Sanity Check Configuration For this implementation. + //-------------------------------------------------------------- + + require (cfg.nComponents <= 128) + require (cfg.nSerialPorts == 0) + require (cfg.hasBusMaster == false) + require (cfg.nDebugRamBytes <= 64) + require (cfg.authType == DebugModuleAuthType.None) + + //-------------------------------------------------------------- + // Private Classes (Register Fields) + //-------------------------------------------------------------- + + class RAMFields() extends Bundle { + val interrupt = Bool() + val haltnot = Bool() + val data = Bits(width = 32) + + override def cloneType = new RAMFields().asInstanceOf[this.type] + } + + class CONTROLFields() extends Bundle { + val interrupt = Bool() + val haltnot = Bool() + val reserved0 = Bits(width = 31-22 + 1) + val buserror = Bits(width = 3) + val serial = Bits(width = 3) + val autoincrement = Bool() + val access = UInt(width = 3) + val hartid = Bits(width = 10) + val ndreset = Bool() + val fullreset = Bool() + + override def cloneType = new CONTROLFields().asInstanceOf[this.type] + + } + + class DMINFOFields() extends Bundle { + val reserved0 = Bits(width = 2) + val abussize = UInt(width = 7) + val serialcount = UInt(width = 4) + val access128 = Bool() + val access64 = Bool() + val access32 = Bool() + val access16 = Bool() + val accesss8 = Bool() + val haltsum = Bool() + val reserved1 = Bits(width = 3) + val authenticated = Bool() + val authbusy = Bool() + val authtype = UInt(width = 2) + val version = UInt(width = 2) + + override def cloneType = new DMINFOFields().asInstanceOf[this.type] + + } + + class HALTSUMFields() extends Bundle { + val serialfull = Bool() + val serialvalid = Bool() + val acks = Bits(width = 32) + + override def cloneType = new HALTSUMFields().asInstanceOf[this.type] + + } + + //-------------------------------------------------------------- + // Module I/O + //-------------------------------------------------------------- + + val io = new Bundle { + val db = new DebugBusIO()(p).flip() + val debugInterrupts = Vec(cfg.nComponents, Bool()).asOutput + val tl = new ClientUncachedTileLinkIO().flip + val ndreset = Bool(OUTPUT) + val fullreset = Bool(OUTPUT) + } + + //-------------------------------------------------------------- + // Register & Wire Declarations + //-------------------------------------------------------------- + + // --- Debug Bus Registers + val CONTROLReset = Wire(new CONTROLFields()) + val CONTROLWrEn = Wire(Bool()) + val CONTROLReg = Reg(new CONTROLFields()) + val CONTROLWrData = Wire (new CONTROLFields()) + val CONTROLRdData = Wire (new CONTROLFields()) + val ndresetCtrReg = Reg(UInt(cfg.nNDResetCycles)) + + val DMINFORdData = Wire (new DMINFOFields()) + + val HALTSUMRdData = Wire (new HALTSUMFields()) + + val RAMWrData = Wire (new RAMFields()) + val RAMRdData = Wire (new RAMFields()) + + // --- System Bus Registers + + val SETHALTNOTWrEn = Wire(Bool()) + val SETHALTNOTWrData = Wire(UInt(width = sbIdWidth)) + val CLEARDEBINTWrEn = Wire(Bool()) + val CLEARDEBINTWrData = Wire(UInt(width = sbIdWidth)) + + // --- Interrupt & Halt Notification Registers + + val interruptRegs = Reg(init=Vec.fill(cfg.nComponents){Bool(false)}) + + val haltnotRegs = Reg(init=Vec.fill(cfg.nComponents){Bool(false)}) + val numHaltnotStatus = (Math.ceil(cfg.nComponents / 32)).toInt + + val haltnotStatus = Wire(Vec(numHaltnotStatus, Bits(width = 32))) + val rdHaltnotStatus = Wire(Bits(width = 32)) + + val haltnotSummary = Wire(Bits(width = 32)) + + // --- Debug RAM + + // Since the access size from Debug Bus and System Bus may not be consistent, + // use the maximum to build the RAM, and then select as needed for the smaller + // size. + + val dbRamDataWidth = DbBusConsts.dbRamWordBits + val sbRamDataWidth = tlDataBits + val dbRamAddrWidth = log2Up((cfg.nDebugRamBytes * 8) / dbRamDataWidth) + val sbRamAddrWidth = log2Up((cfg.nDebugRamBytes * 8) / sbRamDataWidth) + val sbRamAddrOffset = log2Up(tlDataBits/8) + + val ramDataWidth = max(dbRamDataWidth, sbRamDataWidth) + val ramAddrWidth = min(dbRamAddrWidth, sbRamAddrWidth) + val ramMem = Mem(1 << ramAddrWidth , UInt(width=ramDataWidth)) + val ramAddr = Wire(UInt(width=ramAddrWidth)) + val ramRdData = Wire(UInt(width=ramDataWidth)) + val ramWrData = Wire(UInt(width=ramDataWidth)) + val ramWrMask = Wire(UInt(width=ramDataWidth)) + val ramWrEn = Wire(Bool()) + + val dbRamAddr = Wire(UInt(width=dbRamAddrWidth)) + val dbRamRdData = Wire (UInt(width=dbRamDataWidth)) + val dbRamWrData = Wire(UInt(width=dbRamDataWidth)) + val dbRamWrEn = Wire(Bool()) + val dbRamRdEn = Wire(Bool()) + + val sbRamAddr = Wire(UInt(width=sbRamAddrWidth)) + val sbRamRdData = Reg (UInt(width=sbRamDataWidth)) + val sbRamWrData = Wire(UInt(width=sbRamDataWidth)) + val sbRamWrEn = Wire(Bool()) + val sbRamRdEn = Wire(Bool()) + + val sbRomRdData = Wire(UInt(width=tlDataBits)) + val sbRomAddrOffset = log2Up(tlDataBits/8) + + // --- Debug Bus Accesses + + val dbRdEn = Wire(Bool()) + val dbWrEn = Wire(Bool()) + val dbRdData = Wire(UInt(width = DbBusConsts.dbDataSize)) + + val s_DB_READY :: s_DB_RESP :: Nil = Enum(Bits(), 2) + val dbStateReg = Reg(init = s_DB_READY) + + val dbResult = Wire(io.db.resp.bits) + + val dbReq = Wire(io.db.req.bits) + val dbRespReg = Reg(io.db.resp.bits) + + val rdCondWrFailure = Wire(Bool()) + val dbWrNeeded = Wire(Bool()) + + // --- System Bus Access + val sbAddr = Wire(UInt(width=sbAddrWidth)) + val sbRdData = Wire(UInt(width=tlDataBits)) + val sbWrData = Wire(UInt(width=tlDataBits)) + val sbWrMask = Wire(UInt(width=tlDataBits)) + val sbWrEn = Wire(Bool()) + val sbRdEn = Wire(Bool()) + + val stallFromDb = Wire(Bool()) + val stallFromSb = Wire(Bool()) + //-------------------------------------------------------------- + // Interrupt Registers + //-------------------------------------------------------------- + + for (component <- 0 until cfg.nComponents) { + io.debugInterrupts(component) := interruptRegs(component) + } + + // Interrupt Registers are written by write to CONTROL or debugRAM addresses + // for Debug Bus, and cleared by writes to CLEARDEBINT by System Bus. + // It is "unspecified" what should happen if both + // SET and CLEAR happen at the same time. In this + // implementation, the SET wins. + + for (component <- 0 until cfg.nComponents) { + when (CONTROLWrEn) { + when (CONTROLWrData.hartid === UInt(component)) { + interruptRegs(component) := interruptRegs(component) | CONTROLWrData.interrupt + } + }.elsewhen (dbRamWrEn) { + when (CONTROLReg.hartid === UInt(component)){ + interruptRegs(component) := interruptRegs(component) | RAMWrData.interrupt + } + }.elsewhen (CLEARDEBINTWrEn){ + when (CLEARDEBINTWrData === UInt(component, width = sbIdWidth)) { + interruptRegs(component) := Bool(false) + } + } + } + + //-------------------------------------------------------------- + // Halt Notification Registers + //-------------------------------------------------------------- + + // Halt Notifications Registers are cleared by zero write to CONTROL or debugRAM addresses + // for Debug Bus, and set by write to SETHALTNOT by System Bus. + // It is "unspecified" what should happen if both + // SET and CLEAR happen at the same time. In this + // implementation, the SET wins. + + for (component <- 0 until cfg.nComponents) { + when (SETHALTNOTWrEn){ + when (SETHALTNOTWrData === UInt(component, width = sbIdWidth)) { + haltnotRegs(component) := Bool(true) + } + } .elsewhen (CONTROLWrEn) { + when (CONTROLWrData.hartid === UInt(component)) { + haltnotRegs(component) := haltnotRegs(component) & CONTROLWrData.haltnot + } + }.elsewhen (dbRamWrEn) { + when (CONTROLReg.hartid === UInt(component)){ + haltnotRegs(component) := haltnotRegs(component) & RAMWrData.haltnot + } + } + } + + for (ii <- 0 until numHaltnotStatus) { + for (jj <- 0 until 32) { + val component = ii * 32 + jj + if (component < cfg.nComponents){ + haltnotStatus(ii)(jj) := haltnotRegs(component) + } else { + haltnotStatus(ii)(jj) := Bool(false) + } + } + } + + haltnotSummary := Bits(0) + for (ii <- 0 until numHaltnotStatus) { + haltnotSummary(ii) := haltnotStatus(ii).orR + } + + //-------------------------------------------------------------- + // Other Registers + //-------------------------------------------------------------- + + CONTROLReset.interrupt := Bool(false) + CONTROLReset.haltnot := Bool(false) + CONTROLReset.reserved0 := Bits(0) + CONTROLReset.buserror := Bits(0) + CONTROLReset.serial := Bits(0) + CONTROLReset.autoincrement := Bool(false) + CONTROLReset.access := UInt(DebugModuleAccessType.Access32Bit.id) + CONTROLReset.hartid := Bits(0) + CONTROLReset.ndreset := Bool(false) + CONTROLReset.fullreset := Bool(false) + + // Because this version of DebugModule doesn't + // support authentication, this entire register is + // Read-Only constant wires. + DMINFORdData.reserved0 := Bits(0) + DMINFORdData.abussize := UInt(0) // Not Implemented. + DMINFORdData.serialcount := UInt(cfg.nSerialPorts) + DMINFORdData.access128 := Bool(cfg.hasAccess128) + DMINFORdData.access64 := Bool(cfg.hasAccess64) + DMINFORdData.access32 := Bool(cfg.hasAccess32) + DMINFORdData.access16 := Bool(cfg.hasAccess16) + DMINFORdData.accesss8 := Bool(cfg.hasAccess8) + DMINFORdData.haltsum := Bool(cfg.hasHaltSum) + DMINFORdData.reserved1 := Bits(0) + DMINFORdData.authenticated := Bool(true) // Not Implemented. + DMINFORdData.authbusy := Bool(false) // Not Implemented. + DMINFORdData.authtype := UInt(cfg.authType.id) + DMINFORdData.version := UInt(0) + + HALTSUMRdData.serialfull := Bool(false) // Not Implemented + HALTSUMRdData.serialvalid := Bool(false) // Not Implemented + HALTSUMRdData.acks := haltnotSummary + + //-------------------------------------------------------------- + // Debug RAM Access (Debug Bus & System Bus) + //-------------------------------------------------------------- + + dbReq := io.db.req.bits + // Debug Bus RAM Access + // From Specification: Debug RAM is 0x00 - 0x0F + // 0x40 - 0x6F Not Implemented + dbRamAddr := dbReq.addr( dbRamAddrWidth-1 , 0) + dbRamWrData := dbReq.data + sbRamAddr := sbAddr(sbRamAddrWidth + sbRamAddrOffset - 1, sbRamAddrOffset) + sbRamWrData := sbWrData + + require (dbRamAddrWidth >= ramAddrWidth) // SB accesses less than 32 bits Not Implemented. + val dbRamWrMask = Vec.fill(1 << (dbRamAddrWidth - ramAddrWidth)){Fill(dbRamDataWidth, UInt(1, width=1))} + + if (dbRamDataWidth < ramDataWidth){ + + val dbRamSel = dbRamAddr(dbRamAddrWidth - ramAddrWidth - 1 , 0) + val rdDataWords = Vec.tabulate(1 << (dbRamAddrWidth - ramAddrWidth)){ ii => + ramRdData((ii+1)*dbRamDataWidth - 1 , ii*dbRamDataWidth)} + + dbRamWrMask := Vec.fill(1 << (dbRamAddrWidth - ramAddrWidth)){UInt(0, width = dbRamDataWidth)} + dbRamWrMask(dbRamSel) := Fill(dbRamDataWidth, UInt(1, width=1)) + dbRamRdData := rdDataWords(dbRamSel) + } else { + dbRamRdData := ramRdData + } + + sbRamRdData := ramRdData + + ramWrMask := Mux(sbRamWrEn, sbWrMask, dbRamWrMask.toBits()) + + assert (!((dbRamWrEn | dbRamRdEn) & (sbRamRdEn | sbRamWrEn)), "Stall logic should have prevented concurrent SB/DB RAM Access") + + // Make copies of DB RAM data before writing. + val dbRamWrDataVec = Fill(1 << (dbRamAddrWidth - ramAddrWidth), dbRamWrData) + ramWrData := Mux(sbRamWrEn, + (ramWrMask & sbRamWrData ) | (~ramWrMask & ramRdData), + (ramWrMask & dbRamWrDataVec.toBits) | (~ramWrMask & ramRdData)) + + ramAddr := Mux(sbRamWrEn | sbRamRdEn, sbRamAddr, + dbRamAddr >> (dbRamAddrWidth - ramAddrWidth)) + + ramRdData := ramMem(ramAddr) + when (ramWrEn) { ramMem(ramAddr) := ramWrData } + + ramWrEn := sbRamWrEn | dbRamWrEn + + //-------------------------------------------------------------- + // Debug Bus Access + //-------------------------------------------------------------- + + // 0x00 - 0x0F Debug RAM + // 0x10 - 0x1B Registers + // 0x1C - 0x3B Halt Notification Registers + // 0x3C - 0x3F Registers + // 0x40 - 0x6F Debug RAM + + + // ----------------------------------------- + // DB Access Write Decoder + + CONTROLWrData := new CONTROLFields().fromBits(dbReq.data) + RAMWrData := new RAMFields().fromBits(dbReq.data) + + dbRamWrEn := Bool(false) + CONTROLWrEn := Bool(false) + when ((dbReq.addr >> 4) === Bits(0)) { // 0x00 - 0x0F Debug RAM + dbRamWrEn := dbWrEn + }.elsewhen (dbReq.addr === DMCONTROL) { + CONTROLWrEn := dbWrEn + }.otherwise { + //Other registers/RAM are Not Implemented. + } + + when (reset) { + CONTROLReg := CONTROLReset + ndresetCtrReg := UInt(0) + }.elsewhen (CONTROLWrEn) { + // interrupt handled in other logic + // haltnot handled in other logic + if (cfg.hasBusMaster){ + // buserror is set 'until 0 is written to any bit in this field'. + CONTROLReg.buserror := Mux((CONTROLWrData.buserror === SInt(-1).toBits), CONTROLReg.buserror, UInt(0)) + CONTROLReg.autoincrement := CONTROLWrData.autoincrement + CONTROLReg.access := CONTROLWrData.access + } + if (cfg.nSerialPorts > 0){ + CONTROLReg.serial := CONTROLWrData.serial + } + CONTROLReg.hartid := CONTROLWrData.hartid + CONTROLReg.fullreset := CONTROLReg.fullreset | CONTROLWrData.fullreset + when (CONTROLWrData.ndreset){ + ndresetCtrReg := UInt(cfg.nNDResetCycles) + }.otherwise { + ndresetCtrReg := Mux(ndresetCtrReg === UInt(0) , UInt(0), ndresetCtrReg - UInt(1)) + } + }.otherwise { + ndresetCtrReg := Mux(ndresetCtrReg === UInt(0) , UInt(0), ndresetCtrReg - UInt(1)) + } + + // ----------------------------------------- + // DB Access Read Mux + + CONTROLRdData := CONTROLReg; + CONTROLRdData.interrupt := interruptRegs(CONTROLReg.hartid) + CONTROLRdData.haltnot := haltnotRegs(CONTROLReg.hartid) + CONTROLRdData.ndreset := ndresetCtrReg.orR + + RAMRdData.interrupt := interruptRegs(CONTROLReg.hartid) + RAMRdData.haltnot := haltnotRegs(CONTROLReg.hartid) + RAMRdData.data := dbRamRdData + + dbRdData := UInt(0) + + // Higher numbers of numHaltnotStatus Not Implemented. + // This logic assumes only up to 128 components. + rdHaltnotStatus := Bits(0) + for (ii <- 0 until numHaltnotStatus) { + when (dbReq.addr === UInt(ii)) { + rdHaltnotStatus := haltnotStatus(ii) + } + } + + dbRamRdEn := Bool(false) + when ((dbReq.addr >> 4) === Bits(0)) { // 0x00 - 0x0F Debug RAM + dbRdData := RAMRdData.toBits() + dbRamRdEn := dbRdEn + }.elsewhen (dbReq.addr === DMCONTROL) { + dbRdData := CONTROLRdData.toBits() + }.elsewhen (dbReq.addr === DMINFO) { + dbRdData := DMINFORdData.toBits() + }.elsewhen (dbReq.addr === HALTSUM) { + if (cfg.hasHaltSum){ + dbRdData := HALTSUMRdData.toBits() + } else { + dbRdData := UInt(0) + } + }.elsewhen ((dbReq.addr >> 2) === UInt(7)) { // 0x1C - 0x1F Haltnot + dbRdData := rdHaltnotStatus + } .otherwise { + //These Registers are not implemented in this version of DebugModule: + // AUTHDATA0 + // AUTHDATA1 + // SERDATA + // SERSTATUS + // SBUSADDRESS0 + // SBUSADDRESS1 + // SBDATA0 + // SBDATA1 + // SBADDRESS2 + // SBDATA2 + // SBDATA3 + // 0x20 - 0x3B haltnot + // Upper bytes of Debug RAM. + dbRdData := UInt(0) + } + + // Conditional write fails if MSB is set of the read data. + rdCondWrFailure := dbRdData(dbDataSize - 1 ) && + (dbReq.op === db_OP_READ_COND_WRITE) + + dbWrNeeded := (dbReq.op === db_OP_READ_WRITE) || + ((dbReq.op === db_OP_READ_COND_WRITE) && ~rdCondWrFailure) + + // This is only relevant at end of s_DB_READ. + dbResult.resp := Mux(rdCondWrFailure, + db_RESP_FAILURE, + db_RESP_SUCCESS) + dbResult.data := dbRdData + + // ----------------------------------------- + // DB Access State Machine Decode (Combo) + io.db.req.ready := !stallFromSb && ((dbStateReg === s_DB_READY) || + (dbStateReg === s_DB_RESP && io.db.resp.fire())) + + io.db.resp.valid := (dbStateReg === s_DB_RESP) + io.db.resp.bits := dbRespReg + + dbRdEn := io.db.req.fire() + dbWrEn := dbWrNeeded && io.db.req.fire() + + // ----------------------------------------- + // DB Access State Machine Update (Seq) + + when (dbStateReg === s_DB_READY){ + when (io.db.req.fire()){ + dbStateReg := s_DB_RESP + dbRespReg := dbResult + } + } .elsewhen (dbStateReg === s_DB_RESP){ + when (io.db.req.fire()){ + dbStateReg := s_DB_RESP + dbRespReg := dbResult + }.elsewhen (io.db.resp.fire()){ + dbStateReg := s_DB_READY + } + } + + + //-------------------------------------------------------------- + // Debug ROM + //-------------------------------------------------------------- + + sbRomRdData := UInt(0) + if (cfg.hasDebugRom) { + // Inspired by ROMSlave + val romContents = cfg.debugRomContents.get + val romByteWidth = tlDataBits / 8 + val romRows = Math.ceil((romContents.size + romByteWidth)/romByteWidth).toInt + val romMem = Vec.tabulate(romRows) { ii => + val slice = romContents.slice(ii*romByteWidth, (ii+1)*romByteWidth) + UInt(slice.foldRight(BigInt(0)) { case (x,y) => ((y << 8) + (x.toInt & 0xFF))}, width = romByteWidth*8) + } + + val sbRomRdAddr = Wire(UInt()) + + if (romRows == 1) { + sbRomRdAddr := UInt(0) + } else { + sbRomRdAddr := sbAddr(log2Up(romRows) + sbRomAddrOffset - 1, sbRomAddrOffset) + } + sbRomRdData := romMem (sbRomRdAddr) + } + + //-------------------------------------------------------------- + // System Bus Access + //-------------------------------------------------------------- + + + // ----------------------------------------- + // SB Access Write Decoder + + sbRamWrEn := Bool(false) + SETHALTNOTWrEn := Bool(false) + CLEARDEBINTWrEn := Bool(false) + + if (tlDataBits == 32) { + SETHALTNOTWrData := sbWrData + CLEARDEBINTWrData := sbWrData + when (sbAddr(11, 8) === UInt(4)){ // 0x400-0x4ff is Debug RAM + sbRamWrEn := sbWrEn + sbRamRdEn := sbRdEn + }.elsewhen (sbAddr === SETHALTNOT){ + SETHALTNOTWrEn := sbWrEn + }.elsewhen (sbAddr === CLEARDEBINT){ + CLEARDEBINTWrEn := sbWrEn + }.otherwise { + //Other registers/RAM are Not Implemented. + } + } else { + + // Pick out the correct word based on the address. + val sbWrDataWords = Vec.tabulate (tlDataBits / 32) {ii => sbWrData((ii+1)*32 - 1, ii*32)} + val sbWrMaskWords = Vec.tabulate(tlDataBits/32) {ii => sbWrMask ((ii+1)*32 -1, ii*32)} + + val sbWrSelTop = log2Up(tlDataBits/8) + val sbWrSelBottom = 2 + + SETHALTNOTWrData := sbWrDataWords(SETHALTNOT(sbWrSelTop, sbWrSelBottom)) + CLEARDEBINTWrData := sbWrDataWords(CLEARDEBINT(sbWrSelTop, sbWrSelBottom)) + + when (sbAddr(11,8) === UInt(4)){ //0x400-0x4ff is Debug RAM + sbRamWrEn := sbWrEn + sbRamRdEn := sbRdEn + } + + SETHALTNOTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop) === SETHALTNOT(sbAddrWidth-1, sbWrSelTop) && + sbWrMaskWords(SETHALTNOT(sbWrSelTop, sbWrSelBottom)).orR && + sbWrEn + + CLEARDEBINTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop) === CLEARDEBINT(sbAddrWidth-1, sbWrSelTop) && + sbWrMaskWords(CLEARDEBINT(sbWrSelTop, sbWrSelBottom)).orR && + sbWrEn + + } + + // ----------------------------------------- + // SB Access Read Mux + + sbRdData := UInt(0) + sbRamRdEn := Bool(false) + + dbRamRdEn := Bool(false) + when (sbAddr(11, 8) === UInt(4)) { //0x400-0x4FF Debug RAM + sbRdData := sbRamRdData + sbRamRdEn := sbRdEn + }.elsewhen (sbAddr(11,8) === UInt(8) || sbAddr(11,8) === UInt(9)){ //0x800-0x9FF Debug ROM + if (cfg.hasDebugRom) { + sbRdData := sbRomRdData + } else { + sbRdData := UInt(0) + } + }. otherwise { + // All readable registers are Not Implemented. + sbRdData := UInt(0) + } + + // ----------------------------------------- + // SB Access State Machine -- based on BRAM Slave + + val sbAcqReg = Reg(io.tl.acquire.bits) + val sbAcqValidReg = Reg(init = Bool(false)) + + val (sbReg_get :: sbReg_getblk :: sbReg_put :: sbReg_putblk :: Nil) = Seq( + Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType + ).map(sbAcqReg.isBuiltInType _) + + val sbMultibeat = sbReg_getblk + + + val sbBeatInc1 = sbAcqReg.addr_beat + UInt(1) + + val sbLast = (sbAcqReg.addr_beat === UInt(tlDataBeats - 1)) + + sbAddr := sbAcqReg.full_addr() + sbRdEn := (sbAcqValidReg && (sbReg_get || sbReg_getblk)) + sbWrEn := (sbAcqValidReg && (sbReg_put || sbReg_putblk)) + sbWrData := sbAcqReg.data + sbWrMask := sbAcqReg.full_wmask() + + // ----------------------------------------- + // SB Access State Machine Update (Seq) + + when (io.tl.acquire.fire()){ + sbAcqReg := io.tl.acquire.bits + sbAcqValidReg := Bool(true) + } .elsewhen (io.tl.grant.fire()) { + when (sbMultibeat){ + sbAcqReg.addr_beat := sbBeatInc1 + when (sbLast) { + sbAcqValidReg := Bool(false) + } + } . otherwise { + sbAcqValidReg := Bool(false) + } + } + + + io.tl.grant.valid := sbAcqValidReg + io.tl.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = sbAcqReg.getBuiltInGrantType(), + client_xact_id = sbAcqReg.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = sbAcqReg.addr_beat, + data = sbRdData + ) + + stallFromDb := Bool(false) // SB always wins, and DB latches its read data so it is not necessary for SB to wait + + stallFromSb := sbRamRdEn || sbRamWrEn // pessimistically assume that DB/SB are going to conflict on the RAM, + // and SB doesn't latch its read data to it is necessary for DB hold + // off while SB is accessing the RAM and waiting to send its result. + + val sbStall = (sbMultibeat & !sbLast) || (io.tl.grant.valid && !io.tl.grant.ready) || stallFromDb + + io.tl.acquire.ready := !sbStall + + //-------------------------------------------------------------- + // Misc. Outputs + //-------------------------------------------------------------- + + io.ndreset := ndresetCtrReg.orR + io.fullreset := CONTROLReg.fullreset + +} From 11b3cee07a6ddf8414409f85b98ea98ae93e8818 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 1 Jun 2016 16:42:39 -0700 Subject: [PATCH 600/688] Ahb tweaks (#50) * ahb: handle tlDataBytes==1 and tlDataBeats==1 gracefully I only now learned that chisel does not handle 0-width wires properly and that log2Up and log2Ceil differ on 1. Fix-up code to handle this. * ahb: optionally disable atomics => optimize to nothing Trust the compiler the compiler to optimize away unused logic. --- uncore/src/main/scala/ahb.scala | 34 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/ahb.scala index 80442dbe..d7ab75bd 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/ahb.scala @@ -30,7 +30,7 @@ class AHBRequestIO(implicit p: Parameters) extends HastiMasterIO } // AHB stage1: translate TileLink Acquires into AHBRequests -class AHBTileLinkIn(implicit val p: Parameters) extends Module +class AHBTileLinkIn(supportAtomics: Boolean = false)(implicit val p: Parameters) extends Module with HasHastiParameters with HasTileLinkParameters with HasAddrMapParameters { @@ -50,13 +50,13 @@ class AHBTileLinkIn(implicit val p: Parameters) extends Module } // Bursts start at 0 and wrap-around back to 0 - val finalBurst = SInt(-1, width = log2Up(tlDataBeats)).asUInt - val firstBurst = UInt(0, width = log2Up(tlDataBeats)) + val finalBurst = UInt(tlDataBeats-1, width = log2Up(tlDataBeats)).asUInt + val firstBurst = UInt(0, width = log2Up(tlDataBeats)) val next_wmask = Wire(UInt(width = tlDataBytes)) // calculated below // State variables for processing more complicated TileLink Acquires val s_atom_r :: s_atom_idle1 :: s_atom_w :: s_atom_idle2 :: Nil = Enum(UInt(), 4) - val atom_state = Reg(init = s_atom_r) + val atom_state = Reg(init = s_atom_r) // never changes if !supportAtomics val done_wmask = Reg(init = UInt(0, width = tlDataBytes)) val burst = Reg(init = firstBurst) @@ -65,7 +65,7 @@ class AHBTileLinkIn(implicit val p: Parameters) extends Module val isReadBurst = io.acquire.bits.is(Acquire.getBlockType) val isWriteBurst = io.acquire.bits.is(Acquire.putBlockType) val isBurst = isWriteBurst || isReadBurst - val isAtomic = io.acquire.bits.is(Acquire.putAtomicType) + val isAtomic = io.acquire.bits.is(Acquire.putAtomicType) && Bool(supportAtomics) val isPut = io.acquire.bits.is(Acquire.putType) // Final states? @@ -97,7 +97,11 @@ class AHBTileLinkIn(implicit val p: Parameters) extends Module // Advance the burst state // We assume here that TileLink gives us all putBlock beats with nothing between them when (io.request.ready && io.acquire.valid && isBurst) { - burst := burst + UInt(1) // overflow => wraps around to 0 + when (last_burst) { + burst := UInt(0) + } .otherwise { + burst := burst + UInt(1) + } } // Advance the atomic state machine @@ -140,7 +144,7 @@ class AHBTileLinkIn(implicit val p: Parameters) extends Module val addr_burst = Mux(isReadBurst, addr_beat + burst, addr_beat) val addr_byte = Mux(isPut, put_addr, io.acquire.bits.addr_byte()) val ahbAddr = Cat(addr_block, addr_burst, addr_byte) - val ahbSize = Mux(isPut, put_size, Mux(isBurst, UInt(log2Up(tlDataBytes)), io.acquire.bits.op_size())) + val ahbSize = Mux(isPut, put_size, Mux(isBurst, UInt(log2Ceil(tlDataBytes)), io.acquire.bits.op_size())) val ahbBurst = MuxLookup(io.acquire.bits.a_type, HBURST_SINGLE, Array( Acquire.getType -> HBURST_SINGLE, @@ -221,10 +225,12 @@ class AHBTileLinkIn(implicit val p: Parameters) extends Module assert(!io.acquire.valid || !isBurst || burst === firstBurst || debugBurst === addr_burst - burst, "TileLink putBlock beats not sequential") // We better not get an incomplete TileLink acquire assert(!io.acquire.valid || isBurst || burst === firstBurst, "TileLink never completed a putBlock") + // If we disabled atomic support, we better not see a request + assert(!io.acquire.bits.is(Acquire.putAtomicType) || Bool(supportAtomics)) } // AHB stage2: execute AHBRequests -class AHBBusMaster(implicit val p: Parameters) extends Module +class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) extends Module with HasHastiParameters with HasTileLinkParameters with HasAddrMapParameters { @@ -325,7 +331,7 @@ class AHBBusMaster(implicit val p: Parameters) extends Module addr_beat0 := io.request.bits.addr_beat } - // Execute Atomic ops + // Execute Atomic ops; unused and optimized away if !supportAtomics val amo_p = p.alterPartial({ case CacheBlockOffsetBits => hastiAddrBits case AmoAluOperandBits => hastiDataBits @@ -340,7 +346,7 @@ class AHBBusMaster(implicit val p: Parameters) extends Module // Transfer bulk data phase // NOTE: this introduces no bubbles because addrReady is a superset of dataReady when (dataReady) { - hwdata1 := alu.io.out // hwdata1 := hwdata0 + hwdata1 := Mux(Bool(supportAtomics), alu.io.out, hwdata0) respondTL1 := respondTL0 latchAtom1 := latchAtom0 g_type1 := g_type0 @@ -367,7 +373,7 @@ class AHBBusMaster(implicit val p: Parameters) extends Module assert(!io.ahb.hresp, "AHB hresp error detected and cannot be reported via TileLink") } -class AHBBridge(implicit val p: Parameters) extends Module +class AHBBridge(supportAtomics: Boolean = true)(implicit val p: Parameters) extends Module with HasHastiParameters with HasTileLinkParameters with HasAddrMapParameters { @@ -383,11 +389,11 @@ class AHBBridge(implicit val p: Parameters) extends Module // AHB does not permit bursts to cross a 1KB boundary require (tlDataBits * tlDataBeats <= 1024*8) // tlDataBytes must be a power of 2 - require (1 << log2Up(tlDataBytes) == tlDataBytes) + require (1 << log2Ceil(tlDataBytes) == tlDataBytes) // Create the sub-blocks - val fsm = Module(new AHBTileLinkIn) - val bus = Module(new AHBBusMaster) + val fsm = Module(new AHBTileLinkIn(supportAtomics)) + val bus = Module(new AHBBusMaster(supportAtomics)) val pad = Module(new Queue(new Grant, 4)) fsm.io.acquire <> Queue(io.tl.acquire, 2) // Pipe is also acceptable From 8e80d1ec8019095c8bd9531e241ac678152908f4 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 1 Jun 2016 21:55:46 -0700 Subject: [PATCH 601/688] Avoid floating-point arithmetic where integers suffice --- uncore/src/main/scala/debug.scala | 4 ++-- uncore/src/main/scala/ecc.scala | 2 +- uncore/src/main/scala/util.scala | 9 --------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index e4f96765..df382a59 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -432,8 +432,8 @@ class DebugModule ()(implicit val p:cde.Parameters) val sbRamAddrWidth = log2Up((cfg.nDebugRamBytes * 8) / sbRamDataWidth) val sbRamAddrOffset = log2Up(tlDataBits/8) - val ramDataWidth = max(dbRamDataWidth, sbRamDataWidth) - val ramAddrWidth = min(dbRamAddrWidth, sbRamAddrWidth) + val ramDataWidth = dbRamDataWidth max sbRamDataWidth + val ramAddrWidth = dbRamAddrWidth min sbRamAddrWidth val ramMem = Mem(1 << ramAddrWidth , UInt(width=ramDataWidth)) val ramAddr = Wire(UInt(width=ramAddrWidth)) val ramRdData = Wire(UInt(width=ramDataWidth)) diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/ecc.scala index 3f16901c..4a582fc1 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/ecc.scala @@ -47,7 +47,7 @@ class ParityCode extends Code class SECCode extends Code { def width(k: Int) = { - val m = new Unsigned(k).log2 + 1 + val m = log2Floor(k) + 1 k + m + (if((1 << m) < m+k+1) 1 else 0) } def encode(x: UInt) = { diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index 12316458..d0afa758 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -3,15 +3,6 @@ package uncore import Chisel._ -import scala.math._ - -class Unsigned(x: Int) { - require(x >= 0) - def clog2: Int = { require(x > 0); ceil(log(x)/log(2)).toInt } - def log2: Int = { require(x > 0); floor(log(x)/log(2)).toInt } - def isPow2: Boolean = x > 0 && (x & (x-1)) == 0 - def nextPow2: Int = if (x == 0) 1 else 1 << clog2 -} object MuxBundle { def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { From 9518b3d589ce8e5a7f6192d94226692c8b34112e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 1 Jun 2016 21:56:24 -0700 Subject: [PATCH 602/688] Fix arithmetic in ROM row count --- uncore/src/main/scala/debug.scala | 2 +- uncore/src/main/scala/rom.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index df382a59..eb22d2b7 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -802,7 +802,7 @@ class DebugModule ()(implicit val p:cde.Parameters) // Inspired by ROMSlave val romContents = cfg.debugRomContents.get val romByteWidth = tlDataBits / 8 - val romRows = Math.ceil((romContents.size + romByteWidth)/romByteWidth).toInt + val romRows = (romContents.size + romByteWidth - 1)/romByteWidth val romMem = Vec.tabulate(romRows) { ii => val slice = romContents.slice(ii*romByteWidth, (ii+1)*romByteWidth) UInt(slice.foldRight(BigInt(0)) { case (x,y) => ((y << 8) + (x.toInt & 0xFF))}, width = romByteWidth*8) diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/rom.scala index b635e127..1ac56e1d 100644 --- a/uncore/src/main/scala/rom.scala +++ b/uncore/src/main/scala/rom.scala @@ -19,7 +19,7 @@ class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module when (io.acquire.fire()) { addr_beat := io.acquire.bits.addr_beat } val byteWidth = tlDataBits / 8 - val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rows = (contents.size + byteWidth - 1)/byteWidth val rom = Vec.tabulate(rows) { i => val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) @@ -52,7 +52,7 @@ class NastiROM(contents: Seq[Byte])(implicit p: Parameters) extends Module { io.b.valid := Bool(false) val byteWidth = io.r.bits.nastiXDataBits / 8 - val rows = (contents.size + byteWidth - 1)/byteWidth + 1 + val rows = (contents.size + byteWidth - 1)/byteWidth val rom = Vec.tabulate(rows) { i => val slice = contents.slice(i*byteWidth, (i+1)*byteWidth) UInt(slice.foldRight(BigInt(0)) { case (x,y) => (y << 8) + (x.toInt & 0xFF) }, byteWidth*8) From 5629fb62bf17a2a763d94f23fd9c80be30500fb9 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 1 Jun 2016 21:58:00 -0700 Subject: [PATCH 603/688] Avoid bitwise sub-assignment --- uncore/src/main/scala/debug.scala | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index eb22d2b7..13fa1555 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -6,7 +6,6 @@ import Chisel._ import junctions._ import cde.{Parameters, Config, Field} -import Math.{ceil, max, min} // ***************************************** // Constants which are interesting even @@ -413,12 +412,12 @@ class DebugModule ()(implicit val p:cde.Parameters) val interruptRegs = Reg(init=Vec.fill(cfg.nComponents){Bool(false)}) val haltnotRegs = Reg(init=Vec.fill(cfg.nComponents){Bool(false)}) - val numHaltnotStatus = (Math.ceil(cfg.nComponents / 32)).toInt + val numHaltnotStatus = ((cfg.nComponents - 1) / 32) + 1 val haltnotStatus = Wire(Vec(numHaltnotStatus, Bits(width = 32))) val rdHaltnotStatus = Wire(Bits(width = 32)) - - val haltnotSummary = Wire(Bits(width = 32)) + + val haltnotSummary = Vec(haltnotStatus.map(_.orR)).toBits // --- Debug RAM @@ -540,19 +539,7 @@ class DebugModule ()(implicit val p:cde.Parameters) } for (ii <- 0 until numHaltnotStatus) { - for (jj <- 0 until 32) { - val component = ii * 32 + jj - if (component < cfg.nComponents){ - haltnotStatus(ii)(jj) := haltnotRegs(component) - } else { - haltnotStatus(ii)(jj) := Bool(false) - } - } - } - - haltnotSummary := Bits(0) - for (ii <- 0 until numHaltnotStatus) { - haltnotSummary(ii) := haltnotStatus(ii).orR + haltnotStatus(ii) := Vec(haltnotRegs.slice(ii * 32, (ii + 1) * 32)).toBits } //-------------------------------------------------------------- From 20e1de08da5b998032f85b742b4df81da00626ba Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 1 Jun 2016 23:35:49 -0700 Subject: [PATCH 604/688] Avoid chisel2 pitfall This code is erroneously flagged as incompatible with chisel3. In fact, it is correct in both chisel2 and chisel3. D'oh. --- uncore/src/main/scala/debug.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 13fa1555..4cb24eb5 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -417,7 +417,7 @@ class DebugModule ()(implicit val p:cde.Parameters) val haltnotStatus = Wire(Vec(numHaltnotStatus, Bits(width = 32))) val rdHaltnotStatus = Wire(Bits(width = 32)) - val haltnotSummary = Vec(haltnotStatus.map(_.orR)).toBits + val haltnotSummary = Cat(haltnotStatus.map(_.orR).reverse) // --- Debug RAM @@ -539,7 +539,7 @@ class DebugModule ()(implicit val p:cde.Parameters) } for (ii <- 0 until numHaltnotStatus) { - haltnotStatus(ii) := Vec(haltnotRegs.slice(ii * 32, (ii + 1) * 32)).toBits + haltnotStatus(ii) := Cat(haltnotRegs.slice(ii * 32, (ii + 1) * 32).reverse) } //-------------------------------------------------------------- From 0866b4c0452d25b6556f0e46e931ebb39ac2a718 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Wed, 1 Jun 2016 23:36:34 -0700 Subject: [PATCH 605/688] Can't assign to Vec literals --- uncore/src/main/scala/debug.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 4cb24eb5..a2f16360 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -593,7 +593,7 @@ class DebugModule ()(implicit val p:cde.Parameters) sbRamWrData := sbWrData require (dbRamAddrWidth >= ramAddrWidth) // SB accesses less than 32 bits Not Implemented. - val dbRamWrMask = Vec.fill(1 << (dbRamAddrWidth - ramAddrWidth)){Fill(dbRamDataWidth, UInt(1, width=1))} + val dbRamWrMask = Wire(init=Vec.fill(1 << (dbRamAddrWidth - ramAddrWidth)){Fill(dbRamDataWidth, UInt(1, width=1))}) if (dbRamDataWidth < ramDataWidth){ From b7ca2145b3dc072709d281971fa57ba2c36b4a24 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 2 Jun 2016 13:47:59 -0700 Subject: [PATCH 606/688] Fix PLIC control bug when !grant.ready --- uncore/src/main/scala/plic.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/plic.scala index 7c1d4287..b8dfbfc3 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/plic.scala @@ -95,9 +95,9 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module } val acq = Queue(io.tl.acquire, 1) - val read = acq.valid && acq.bits.isBuiltInType(Acquire.getType) - val write = acq.valid && acq.bits.isBuiltInType(Acquire.putType) - assert(!acq.valid || read || write, "unsupported PLIC operation") + val read = acq.fire() && acq.bits.isBuiltInType(Acquire.getType) + val write = acq.fire() && acq.bits.isBuiltInType(Acquire.putType) + assert(!acq.fire() || read || write, "unsupported PLIC operation") val addr = acq.bits.full_addr()(log2Up(cfg.size)-1,0) val claimant = @@ -114,10 +114,10 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module pending(myMaxDev) := false }.elsewhen (write && acq.bits.wmask()(cfg.claimOffset)) { val dev = (acq.bits.data >> (8 * cfg.claimOffset))(log2Up(pending.size)-1,0) - when (write && myEnables(dev)) { io.devices(dev-1).complete := true } + when (myEnables(dev)) { io.devices(dev-1).complete := true } }.elsewhen (write) { val thresh = acq.bits.data(log2Up(pending.size)-1,0) - when (write) { threshold(claimant) := thresh } + threshold(claimant) := thresh } }.elsewhen (addr >= cfg.enableBase) { rdata := myEnables From f1745bf1427cee9ab239398006a72a40b7826e2c Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 2 Jun 2016 13:48:29 -0700 Subject: [PATCH 607/688] Allow PLIC nPriorities=0 (priority fixed at 1) --- uncore/src/main/scala/plic.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/plic.scala index b8dfbfc3..c051b9c9 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/plic.scala @@ -49,7 +49,7 @@ case class PLICConfig(nHartsIn: Int, supervisor: Boolean, nDevices: Int, nPriori require(nDevices > 0 && nDevices <= maxDevices) require(nHarts > 0 && nHarts <= maxHarts) - require(nPriorities > 0 && nPriorities <= nDevices) + require(nPriorities >= 0 && nPriorities <= nDevices) } /** Platform-Level Interrupt Controller */ @@ -62,7 +62,9 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module val tl = new ClientUncachedTileLinkIO().flip } - val priority = Reg(Vec(cfg.nDevices+1, UInt(width=log2Up(cfg.nPriorities+1)))) + val priority = + if (cfg.nPriorities > 0) Reg(Vec(cfg.nDevices+1, UInt(width=log2Up(cfg.nPriorities+1)))) + else Wire(init=Vec.fill(cfg.nDevices+1)(UInt(1))) val pending = Reg(init=Vec.fill(cfg.nDevices+1){Bool(false)}) val enables = Reg(Vec(cfg.nHarts, UInt(width = cfg.nDevices+1))) val threshold = Reg(Vec(cfg.nHarts, UInt(width = log2Up(cfg.nPriorities+1)))) @@ -147,7 +149,7 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module when (cond) { rdata := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 16-p.getWidth), p)).reverse) for (j <- 0 until (regsPerBeat min (priority.size - i))) { - when (write) { priority(i+j) := masked_wdata >> (j * (8 << regAddrBits)) } + if (cfg.nPriorities > 0) when (write) { priority(i+j) := masked_wdata >> (j * (8 << regAddrBits)) } } } } From 2e88ffc3644f1f6035221853799f15b469474d44 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Fri, 3 Jun 2016 13:47:40 -0700 Subject: [PATCH 608/688] Cope with changes to AddrMap --- uncore/src/main/scala/interconnect.scala | 31 +++++++++--------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/interconnect.scala index c9fdb478..eb9773a1 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/interconnect.scala @@ -258,36 +258,29 @@ abstract class TileLinkInterconnect(implicit p: Parameters) extends TLModule()(p lazy val io = new TileLinkInterconnectIO(nInner, nOuter) } -class TileLinkRecursiveInterconnect( - val nInner: Int, addrmap: AddrMap, base: BigInt) +class TileLinkRecursiveInterconnect(val nInner: Int, addrMap: AddrMap) (implicit p: Parameters) extends TileLinkInterconnect()(p) { - val levelSize = addrmap.size - val nOuter = addrmap.countSlaves + def port(name: String) = io.out(addrMap.port(name)) + val nOuter = addrMap.numSlaves + val routeSel = (addr: UInt) => + Cat(addrMap.entries.map(e => addrMap(e.name).containsAddress(addr)).reverse) - val addrHashMap = new AddrHashMap(addrmap, base) - val routeSel = (addr: UInt) => { - Cat(addrmap.map { case entry => - val hashEntry = addrHashMap(entry.name) - addr >= UInt(hashEntry.start) && addr < UInt(hashEntry.start + hashEntry.region.size) - }.reverse) - } - - val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, levelSize, routeSel)) + val xbar = Module(new ClientUncachedTileLinkIOCrossbar(nInner, addrMap.length, routeSel)) xbar.io.in <> io.in - io.out <> addrmap.zip(xbar.io.out).flatMap { + io.out <> addrMap.entries.zip(xbar.io.out).flatMap { case (entry, xbarOut) => { entry.region match { - case _: MemSize => - Some(xbarOut) - case MemSubmap(_, submap) if submap.isEmpty => + case submap: AddrMap if submap.isEmpty => xbarOut.acquire.ready := Bool(false) xbarOut.grant.valid := Bool(false) None - case MemSubmap(_, submap) => - val ic = Module(new TileLinkRecursiveInterconnect(1, submap, addrHashMap(entry.name).start)) + case submap: AddrMap => + val ic = Module(new TileLinkRecursiveInterconnect(1, submap)) ic.io.in.head <> xbarOut ic.io.out + case _ => + Some(xbarOut) } } } From 7e550ab07c1cdfdc61f17a62b24da079d2dac8e4 Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Thu, 2 Jun 2016 18:35:17 -0700 Subject: [PATCH 609/688] [debug] rocket: fix for issue 121, correct debug ROM and stall logic --- uncore/src/main/scala/debug.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index a2f16360..43ba6f50 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -214,9 +214,13 @@ class DefaultDebugModuleConfig (val ncomponents : Int, val xlen:Int) extends DebugModuleConfig( nComponents = ncomponents, nDebugBusAddrSize = 5, + // While smaller numbers are theoretically + // possible as noted in the Spec, + // the ROM image would need to be + // adjusted accordingly. nDebugRamBytes = xlen match{ - case 32 => 28 - case 64 => 40 + case 32 => 64 + case 64 => 64 case 128 => 64 }, debugRomContents = Some(DsbBusConsts.defaultRomContents), @@ -888,8 +892,7 @@ class DebugModule ()(implicit val p:cde.Parameters) Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType ).map(sbAcqReg.isBuiltInType _) - val sbMultibeat = sbReg_getblk - + val sbMultibeat = sbReg_getblk & sbAcqValidReg; val sbBeatInc1 = sbAcqReg.addr_beat + UInt(1) From 3e8322816b8fe16d8d17973642fb0fa54c173eec Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Fri, 3 Jun 2016 10:48:01 -0700 Subject: [PATCH 610/688] Correct DMINFO Fields --- uncore/src/main/scala/debug.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 43ba6f50..66de8a36 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -353,6 +353,7 @@ class DebugModule ()(implicit val p:cde.Parameters) val access32 = Bool() val access16 = Bool() val accesss8 = Bool() + val dramsize = UInt(width = 6) val haltsum = Bool() val reserved1 = Bits(width = 3) val authenticated = Bool() @@ -572,12 +573,13 @@ class DebugModule ()(implicit val p:cde.Parameters) DMINFORdData.access32 := Bool(cfg.hasAccess32) DMINFORdData.access16 := Bool(cfg.hasAccess16) DMINFORdData.accesss8 := Bool(cfg.hasAccess8) + DMINFORdData.dramsize := Bits((cfg.nDebugRamBytes >> 2) - 1) // Size in 32-bit words minus 1. DMINFORdData.haltsum := Bool(cfg.hasHaltSum) DMINFORdData.reserved1 := Bits(0) - DMINFORdData.authenticated := Bool(true) // Not Implemented. + DMINFORdData.authenticated := Bool(true) // Not Implemented. DMINFORdData.authbusy := Bool(false) // Not Implemented. DMINFORdData.authtype := UInt(cfg.authType.id) - DMINFORdData.version := UInt(0) + DMINFORdData.version := UInt(1) // Conforms to RISC-V Debug Spec HALTSUMRdData.serialfull := Bool(false) // Not Implemented HALTSUMRdData.serialvalid := Bool(false) // Not Implemented From 605fb5b92f75b4c7c3f9caf564385e767e60705c Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Fri, 3 Jun 2016 15:45:20 -0700 Subject: [PATCH 611/688] [debug]: fix issue with subword select logic --- uncore/src/main/scala/debug.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 66de8a36..17846a19 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -840,9 +840,9 @@ class DebugModule ()(implicit val p:cde.Parameters) // Pick out the correct word based on the address. val sbWrDataWords = Vec.tabulate (tlDataBits / 32) {ii => sbWrData((ii+1)*32 - 1, ii*32)} - val sbWrMaskWords = Vec.tabulate(tlDataBits/32) {ii => sbWrMask ((ii+1)*32 -1, ii*32)} + val sbWrMaskWords = Vec.tabulate (tlDataBits / 32) {ii => sbWrMask ((ii+1)*32 -1, ii*32)} - val sbWrSelTop = log2Up(tlDataBits/8) + val sbWrSelTop = log2Up(tlDataBits/8) - 1 val sbWrSelBottom = 2 SETHALTNOTWrData := sbWrDataWords(SETHALTNOT(sbWrSelTop, sbWrSelBottom)) @@ -852,13 +852,13 @@ class DebugModule ()(implicit val p:cde.Parameters) sbRamWrEn := sbWrEn sbRamRdEn := sbRdEn } - - SETHALTNOTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop) === SETHALTNOT(sbAddrWidth-1, sbWrSelTop) && - sbWrMaskWords(SETHALTNOT(sbWrSelTop, sbWrSelBottom)).orR && + + SETHALTNOTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop + 1) === SETHALTNOT(sbAddrWidth-1, sbWrSelTop + 1) && + (sbWrMaskWords(SETHALTNOT(sbWrSelTop, sbWrSelBottom))).orR && sbWrEn - CLEARDEBINTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop) === CLEARDEBINT(sbAddrWidth-1, sbWrSelTop) && - sbWrMaskWords(CLEARDEBINT(sbWrSelTop, sbWrSelBottom)).orR && + CLEARDEBINTWrEn := sbAddr(sbAddrWidth - 1, sbWrSelTop + 1) === CLEARDEBINT(sbAddrWidth-1, sbWrSelTop + 1) && + (sbWrMaskWords(CLEARDEBINT(sbWrSelTop, sbWrSelBottom))).orR && sbWrEn } From b832689642cef975735e7cad721b3203062c6d34 Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Fri, 3 Jun 2016 18:26:05 -0700 Subject: [PATCH 612/688] Correct Debug ROM contents --- uncore/src/main/scala/debug.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 17846a19..42d38def 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -69,7 +69,10 @@ object DsbBusConsts { * See the Debug Specification for the code. */ - def defaultRomContents : Array[Byte] = Array(0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h + + def defaultRomContents : Array[Byte] = Array( + 0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0xf3, 0x24, 0x40, 0xf1, 0x23, 0x20, 0x90, 0x10, 0x0f, 0x00, 0xf0, 0x0f, 0xf3, 0x24, 0x00, 0xf1, 0x63, 0xc6, 0x04, 0x00, 0x83, 0x24, 0xc0, 0x43, 0x6f, 0x00, 0xc0, 0x01, @@ -78,14 +81,13 @@ object DsbBusConsts { 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, 0x63, 0x04, 0x04, 0x00, 0x6f, 0x00, 0x80, 0x05, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, - 0x13, 0x04, 0xd4, 0xff, 0x63, 0x18, 0x04, 0x02, 0x0f, 0x10, 0x00, 0x00, + 0x13, 0x04, 0x04, 0xf4, 0x63, 0x18, 0x04, 0x02, 0x0f, 0x10, 0x00, 0x00, 0x73, 0x24, 0x00, 0xf1, 0x63, 0x46, 0x04, 0x00, 0x23, 0x2e, 0x90, 0x42, 0x67, 0x00, 0x00, 0x40, 0x13, 0x14, 0x14, 0x00, 0x63, 0x46, 0x04, 0x00, 0x23, 0x3c, 0x90, 0x42, 0x67, 0x00, 0x00, 0x40, 0x13, 0x00, 0x00, 0x00, 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0xdf, 0xfb).map(x => x.toByte) - } From be7500e4a93d8d255818c5f1bf0d853fd41991c0 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 5 Jun 2016 20:33:51 -0700 Subject: [PATCH 613/688] Update PLIC addr map --- uncore/src/main/scala/plic.scala | 96 +++++++++++++++++++------------- 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/plic.scala index c051b9c9..7635e3aa 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/plic.scala @@ -39,13 +39,16 @@ case class PLICConfig(nHartsIn: Int, supervisor: Boolean, nDevices: Int, nPriori def size = hartBase + hartOffset(maxHarts) def maxDevices = 1023 - def maxHarts = 992 - def pendingBase = 0x800 - def enableBase = 0x1000 + def maxHarts = 15872 + def pendingBase = 0x1000 + def enableBase = 0x2000 + def hartBase = 0x200000 + require(hartBase >= enableBase + enableOffset(maxHarts)) + def enableOffset(i: Int) = i * ((maxDevices+7)/8) - def hartBase = enableBase + enableOffset(maxHarts) def hartOffset(i: Int) = i * 0x1000 - def claimOffset = 2 + def claimOffset = 4 + def priorityBytes = 4 require(nDevices > 0 && nDevices <= maxDevices) require(nHarts > 0 && nHarts <= maxHarts) @@ -65,9 +68,11 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module val priority = if (cfg.nPriorities > 0) Reg(Vec(cfg.nDevices+1, UInt(width=log2Up(cfg.nPriorities+1)))) else Wire(init=Vec.fill(cfg.nDevices+1)(UInt(1))) + val threshold = + if (cfg.nPriorities > 0) Reg(Vec(cfg.nHarts, UInt(width = log2Up(cfg.nPriorities+1)))) + else Wire(init=Vec.fill(cfg.nHarts)(UInt(0))) val pending = Reg(init=Vec.fill(cfg.nDevices+1){Bool(false)}) - val enables = Reg(Vec(cfg.nHarts, UInt(width = cfg.nDevices+1))) - val threshold = Reg(Vec(cfg.nHarts, UInt(width = log2Up(cfg.nPriorities+1)))) + val enables = Reg(Vec(cfg.nHarts, Vec(cfg.nDevices+1, Bool()))) for ((p, g) <- pending.tail zip io.devices) { g.ready := !p @@ -88,12 +93,12 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module val maxDevs = Wire(Vec(cfg.nHarts, UInt(width = log2Up(pending.size)))) for (hart <- 0 until cfg.nHarts) { val effectivePriority = - for (((p, en), pri) <- (pending zip enables(hart).toBools zip priority).tail) + for (((p, en), pri) <- (pending zip enables(hart) zip priority).tail) yield Cat(p && en, pri) - val (_, maxDev) = findMax(Cat(UInt(1), threshold(hart)) +: effectivePriority) + val (maxPri, maxDev) = findMax((UInt(1) << priority(0).getWidth) +: effectivePriority) maxDevs(hart) := Reg(next = maxDev) - io.harts(hart) := maxDevs(hart) =/= 0 + io.harts(hart) := Reg(next = maxPri) > Cat(UInt(1), threshold(hart)) } val acq = Queue(io.tl.acquire, 1) @@ -107,49 +112,58 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module else (addr - cfg.hartBase)(log2Up(cfg.hartOffset(cfg.nHarts))-1,log2Up(cfg.hartOffset(1))) val hart = Wire(init = claimant) val myMaxDev = maxDevs(claimant) + UInt(0) // XXX FIRRTL bug w/o the + UInt(0) - val myEnables = enables(hart) >> 1 << 1 - val rdata = Wire(init = Cat(myMaxDev, UInt(0, 16-threshold(0).getWidth), threshold(claimant))) + val myEnables = enables(hart) + val rdata = Wire(init = UInt(0, tlDataBits)) val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata & ~acq.bits.full_wmask()) when (addr >= cfg.hartBase) { + val word = + if (tlDataBytes > cfg.claimOffset) UInt(0) + else addr(log2Up(cfg.claimOffset),log2Up(tlDataBytes)) + rdata := Cat(myMaxDev, UInt(0, 8*cfg.priorityBytes-threshold(0).getWidth), threshold(claimant)) >> (word * tlDataBits) + when (read && addr(log2Ceil(cfg.claimOffset))) { pending(myMaxDev) := false - }.elsewhen (write && acq.bits.wmask()(cfg.claimOffset)) { - val dev = (acq.bits.data >> (8 * cfg.claimOffset))(log2Up(pending.size)-1,0) - when (myEnables(dev)) { io.devices(dev-1).complete := true } - }.elsewhen (write) { - val thresh = acq.bits.data(log2Up(pending.size)-1,0) - threshold(claimant) := thresh + } + when (write) { + when (if (tlDataBytes > cfg.claimOffset) acq.bits.wmask()(cfg.claimOffset) else addr(log2Ceil(cfg.claimOffset))) { + val dev = (acq.bits.data >> ((8 * cfg.claimOffset) % tlDataBits))(log2Up(pending.size)-1,0) + when (myEnables(dev)) { io.devices(dev-1).complete := true } + }.otherwise { + if (cfg.nPriorities > 0) threshold(claimant) := acq.bits.data + } } }.elsewhen (addr >= cfg.enableBase) { - rdata := myEnables - if (cfg.nHarts > 1) - hart := (addr - cfg.enableBase)(log2Up(cfg.enableOffset(cfg.nHarts))-1,log2Up(cfg.enableOffset(1))) - require(enables.size <= tlDataBits) // TODO this can be relaxed - when (write) { enables(hart) := masked_wdata >> 1 << 1 } - }.elsewhen (addr >= cfg.pendingBase) { - for (i <- 0 until pending.size by tlDataBytes) { - val cond = - if (tlDataBytes >= pending.size) Bool(true) - else addr(log2Up(pending.size)-1,log2Up(tlDataBytes)) === i/tlDataBytes - when (cond) { - rdata := Cat(pending.slice(i, i + tlDataBytes).map(p => Cat(UInt(0, 7), p)).reverse) - for (j <- 0 until (tlDataBytes min (pending.size - i))) { - when (write) { pending(i+j) := masked_wdata(j * 8) } + val enableHart = + if (cfg.nHarts > 1) (addr - cfg.enableBase)(log2Up(cfg.enableOffset(cfg.nHarts))-1,log2Up(cfg.enableOffset(1))) + else UInt(0) + hart := enableHart + val word = + if (tlDataBits >= cfg.nHarts) UInt(0) + else addr(log2Up((cfg.nHarts+7)/8)-1,log2Up(tlDataBytes)) + for (i <- 0 until cfg.nHarts by tlDataBits) { + when (word === i/tlDataBits) { + rdata := Cat(myEnables.slice(i, i + tlDataBits).reverse) + for (j <- 0 until (tlDataBits min (myEnables.size - i))) { + when (write) { enables(enableHart)(i+j) := masked_wdata(j) } } } } + }.elsewhen (addr >= cfg.pendingBase) { + val word = + if (tlDataBytes >= pending.size) UInt(0) + else addr(log2Up(pending.size)-1,log2Up(tlDataBytes)) + rdata := pending.toBits >> (word * tlDataBits) }.otherwise { - val regAddrBits = log2Up(log2Up(cfg.maxDevices+1)) - 3 - val regsPerBeat = tlDataBytes >> regAddrBits + val regsPerBeat = tlDataBytes >> log2Up(cfg.priorityBytes) + val word = + if (regsPerBeat >= priority.size) UInt(0) + else addr(log2Up(priority.size*cfg.priorityBytes)-1,log2Up(tlDataBytes)) for (i <- 0 until priority.size by regsPerBeat) { - val cond = - if (regsPerBeat >= priority.size) Bool(true) - else addr(log2Up(priority.size)+regAddrBits-1,log2Up(tlDataBytes)) === i/regsPerBeat - when (cond) { - rdata := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 16-p.getWidth), p)).reverse) + when (word === i/regsPerBeat) { + rdata := Cat(priority.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 8*cfg.priorityBytes-p.getWidth), p)).reverse) for (j <- 0 until (regsPerBeat min (priority.size - i))) { - if (cfg.nPriorities > 0) when (write) { priority(i+j) := masked_wdata >> (j * (8 << regAddrBits)) } + if (cfg.nPriorities > 0) when (write) { priority(i+j) := masked_wdata >> (j * 8 * cfg.priorityBytes) } } } } @@ -157,6 +171,8 @@ class PLIC(val cfg: PLICConfig)(implicit val p: Parameters) extends Module priority(0) := 0 pending(0) := false + for (e <- enables) + e(0) := false io.tl.grant.valid := acq.valid acq.ready := io.tl.grant.ready From 631e3e2dd9eee607695de7a8dbc5b1cda0858532 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 5 Jun 2016 23:06:21 -0700 Subject: [PATCH 614/688] Make PRCI a singleton, not per-tile Some stuff is densely packed in the address space (e.g. IPI regs), so needs to be on the same TileLink slave port --- uncore/src/main/scala/prci.scala | 42 ++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/prci.scala index 2dee3ae8..3d59d16b 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/prci.scala @@ -25,27 +25,29 @@ class PRCITileIO(implicit p: Parameters) extends Bundle { override def cloneType: this.type = new PRCITileIO().asInstanceOf[this.type] } +/** Power, Reset, Clock, Interrupt */ class PRCI(implicit val p: Parameters) extends Module with HasTileLinkParameters with HasAddrMapParameters { val io = new Bundle { - val id = UInt(INPUT, log2Up(p(NTiles))) - val interrupts = new Bundle { + val interrupts = Vec(p(NTiles), new Bundle { val mtip = Bool() val meip = Bool() val seip = Bool() val debug = Bool() - }.asInput + }).asInput val tl = new ClientUncachedTileLinkIO().flip - val tile = new PRCITileIO + val tiles = Vec(p(NTiles), new PRCITileIO) } - val ipi = Reg(init=Bool(false)) + val ipi = Reg(init=Vec.fill(p(NTiles))(Bool(false))) val acq = Queue(io.tl.acquire, 1) + val addr = acq.bits.full_addr() val read = acq.bits.isBuiltInType(Acquire.getType) val write = acq.bits.isBuiltInType(Acquire.putType) - val rdata = Wire(init=ipi) + val rdata = Wire(init=UInt(0)) + val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata & ~acq.bits.full_wmask()) io.tl.grant.valid := acq.valid acq.ready := io.tl.grant.ready io.tl.grant.bits := Grant( @@ -56,15 +58,25 @@ class PRCI(implicit val p: Parameters) extends Module addr_beat = UInt(0), data = rdata) - val regSize = 16 - val nRegs = 2 - val addr = acq.bits.full_addr()(log2Up(regSize*nRegs)-1,log2Up(regSize)) - - when (addr === UInt(0) && write) { - ipi := acq.bits.data(0) + when (write) { + val ipiRegBytes = 4 + val regsPerBeat = tlDataBytes/ipiRegBytes + val word = + if (regsPerBeat >= ipi.size) UInt(0) + else addr(log2Up(ipi.size*ipiRegBytes)-1,log2Up(tlDataBytes)) + for (i <- 0 until ipi.size by regsPerBeat) { + when (word === i/regsPerBeat) { + rdata := Cat(ipi.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 8*ipiRegBytes-1), p)).reverse) + for (j <- 0 until (regsPerBeat min (ipi.size - i))) { + when (write) { ipi(i+j) := masked_wdata(j*8*ipiRegBytes) } + } + } + } } - io.tile.interrupts := io.interrupts - io.tile.interrupts.msip := ipi - io.tile.id := io.id + for ((tile, i) <- io.tiles zipWithIndex) { + tile.interrupts := io.interrupts(i) + tile.interrupts.msip := ipi(i) + tile.id := UInt(i) + } } From dd85f2410fb8ac54d481b3b9a14f6adb678436df Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sun, 5 Jun 2016 23:47:56 -0700 Subject: [PATCH 615/688] Avoid need for cloneType --- uncore/src/main/scala/prci.scala | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/prci.scala index 3d59d16b..13064c44 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/prci.scala @@ -11,15 +11,18 @@ import cde.{Parameters, Field} /** Number of tiles */ case object NTiles extends Field[Int] +class PRCIInterrupts extends Bundle { + val mtip = Bool() + val meip = Bool() + val seip = Bool() + val debug = Bool() +} + class PRCITileIO(implicit p: Parameters) extends Bundle { val reset = Bool(OUTPUT) val id = UInt(OUTPUT, log2Up(p(NTiles))) - val interrupts = new Bundle { - val mtip = Bool() + val interrupts = new PRCIInterrupts { val msip = Bool() - val meip = Bool() - val seip = Bool() - val debug = Bool() }.asOutput override def cloneType: this.type = new PRCITileIO().asInstanceOf[this.type] @@ -30,12 +33,7 @@ class PRCI(implicit val p: Parameters) extends Module with HasTileLinkParameters with HasAddrMapParameters { val io = new Bundle { - val interrupts = Vec(p(NTiles), new Bundle { - val mtip = Bool() - val meip = Bool() - val seip = Bool() - val debug = Bool() - }).asInput + val interrupts = Vec(p(NTiles), new PRCIInterrupts).asInput val tl = new ClientUncachedTileLinkIO().flip val tiles = Vec(p(NTiles), new PRCITileIO) } From 2d66ac93d3ae586c3e08e5042de7c305da890066 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 2 Jun 2016 09:04:30 -0700 Subject: [PATCH 616/688] make sure HastiRAM cuts off the correct number of bits for word address --- uncore/src/main/scala/bram.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index 480f4a73..a5c4cef9 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -74,13 +74,13 @@ class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { val wsize = Reg(UInt(width = SZ_HSIZE)) val ram = SeqMem(depth, Vec(hastiDataBytes, Bits(width = 8))) - val max_wsize = log2Ceil(hastiDataBytes) + val max_size = log2Ceil(hastiDataBytes) val wmask_lut = MuxLookup(wsize, SInt(-1, hastiDataBytes).asUInt, - (0 until max_wsize).map(sz => (UInt(sz) -> UInt((1 << (1 << sz)) - 1)))) - val wmask = (wmask_lut << waddr(max_wsize - 1, 0))(hastiDataBytes - 1, 0) + (0 until max_size).map(sz => (UInt(sz) -> UInt((1 << (1 << sz)) - 1)))) + val wmask = (wmask_lut << waddr(max_size - 1, 0))(hastiDataBytes - 1, 0) val is_trans = io.hsel && (io.htrans === HTRANS_NONSEQ || io.htrans === HTRANS_SEQ) - val raddr = io.haddr >> UInt(2) + val raddr = io.haddr >> UInt(max_size) val ren = is_trans && !io.hwrite val bypass = Reg(init = Bool(false)) @@ -90,10 +90,10 @@ class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { wvalid := Bool(true) } .otherwise { wvalid := Bool(false) } - when (ren) { bypass := wvalid && (waddr >> UInt(2)) === raddr } + when (ren) { bypass := wvalid && (waddr >> UInt(max_size)) === raddr } when (wvalid) { - ram.write(waddr >> UInt(2), wdata, wmask.toBools) + ram.write(waddr >> UInt(max_size), wdata, wmask.toBools) } val rdata = ram.read(raddr, ren) From 324cabc494a772045c8ac7516e5e957dc03511a7 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Tue, 7 Jun 2016 14:04:01 -0700 Subject: [PATCH 617/688] tilelink: wmask was double the width it should be When amo_offset = UInt(0), UIntToOH(amo_offset) = "b01", not b"1". This meant that the amo wmask was double wide, making wmask() fat. --- uncore/src/main/scala/tilelink.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index f6a497b2..dde0a619 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -177,11 +177,12 @@ trait HasAcquireUnion extends HasTileLinkParameters { def amo_shift_bytes(dummy: Int = 0) = UInt(amoAluOperandBytes)*amo_offset() /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0): UInt = { - Mux(isBuiltInType(Acquire.putAtomicType), - FillInterleaved(amoAluOperandBytes, UIntToOH(amo_offset())), - Mux(isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType), - union(tlWriteMaskBits, 1), - UInt(0, width = tlWriteMaskBits))) + val is_amo = isBuiltInType(Acquire.putAtomicType) + val amo_sel = if (tlByteAddrBits > log2Up(amoAluOperandBytes)) UIntToOH(amo_offset()) else UInt(1) + val amo_mask = FillInterleaved(amoAluOperandBytes, amo_sel) + val is_put = isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType) + val put_mask = union(tlWriteMaskBits, 1) + Mux(is_amo, amo_mask, Mux(is_put, put_mask, UInt(0))) } /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) From f421e2ab112c152787939f4173d8a264e25c13dd Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 8 Jun 2016 09:58:23 -0700 Subject: [PATCH 618/688] fix TileLinkWidthAdapter --- uncore/src/main/scala/converters.scala | 312 ++++++++++++------------- 1 file changed, 155 insertions(+), 157 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 167bebb7..569f4278 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -756,7 +756,7 @@ object TileLinkWidthAdapter { widener.io.in <> in out <> widener.io.out } else if (out.tlDataBits < in.tlDataBits) { - val narrower = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) narrower.io.in <> in out <> narrower.io.out } else { @@ -961,9 +961,9 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val outerMaxClients = outerParams.maxClientsPerPort val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - require(outerDataBeats >= innerDataBeats) + require(outerDataBeats > innerDataBeats) require(outerDataBeats % innerDataBeats == 0) - require(outerDataBits <= innerDataBits) + require(outerDataBits < innerDataBits) require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) val factor = outerDataBeats / innerDataBeats @@ -973,166 +973,164 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) } - if (factor > 1) { - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits - val stretch = iacq.a_type === Acquire.putBlockType - val shrink = iacq.a_type === Acquire.getBlockType - val smallput = iacq.a_type === Acquire.putType - val smallget = iacq.a_type === Acquire.getType + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType + val smallput = iacq.a_type === Acquire.putType + val smallget = iacq.a_type === Acquire.getType - val acq_data_buffer = Reg(UInt(width = innerDataBits)) - val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) - val acq_client_id = Reg(iacq.client_xact_id) - val acq_addr_block = Reg(iacq.addr_block) - val acq_addr_beat = Reg(iacq.addr_beat) - val oacq_ctr = Counter(factor) + val acq_data_buffer = Reg(UInt(width = innerDataBits)) + val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) - val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) - val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) + val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) + val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) - val mask_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerWriteMaskBits - val msb = (i + 1) * outerWriteMaskBits - 1 - iacq.wmask()(msb, lsb) + val mask_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerWriteMaskBits + val msb = (i + 1) * outerWriteMaskBits - 1 + iacq.wmask()(msb, lsb) + } + + val data_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerDataBits + val msb = (i + 1) * outerDataBits - 1 + iacq.data(msb, lsb) + } + + val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) + + val smallput_data = Mux1H(beat_sel, data_chunks) + val smallput_wmask = Mux1H(beat_sel, mask_chunks) + val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) + + assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), + "Can't perform Put wider than outer width") + + val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( + MT_B -> Bool(true), + MT_BU -> Bool(true), + MT_H -> Bool(outerDataBits >= 16), + MT_HU -> Bool(outerDataBits >= 16), + MT_W -> Bool(outerDataBits >= 32), + MT_WU -> Bool(outerDataBits >= 32), + MT_D -> Bool(outerDataBits >= 64), + MT_Q -> Bool(false))) + + assert(!io.in.acquire.valid || !smallget || read_size_ok, + "Can't perform Get wider than outer width") + + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate())(outerConfig) + + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + addr_byte = outer_byte_addr, + operand_size = iacq.op_size(), + alloc = iacq.allocate())(outerConfig) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = smallput_beat, + data = smallput_data, + wmask = Some(smallput_wmask))(outerConfig) + + val sending_put = Reg(init = Bool(false)) + + val pass_valid = io.in.acquire.valid && !stretch + + io.out.acquire.bits := MuxBundle(Wire(io.out.acquire.bits, init=iacq), Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire), + (smallput, put_acquire), + (smallget, get_acquire))) + io.out.acquire.valid := sending_put || pass_valid + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) + } + + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(innerDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits)(innerConfig) + + val smallget_grant = ognt.g_type === Grant.getDataBeatType + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), + data = Fill(factor, ognt.data))(innerConfig) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + + io.in.grant.bits := MuxBundle(Wire(io.in.grant.bits, init=ognt), Seq( + sending_get -> get_block_grant, + smallget_grant -> get_grant)) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) } + } - val data_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerDataBits - val msb = (i + 1) * outerDataBits - 1 - iacq.data(msb, lsb) - } - - val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) - - val smallput_data = Mux1H(beat_sel, data_chunks) - val smallput_wmask = Mux1H(beat_sel, mask_chunks) - val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) - - assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), - "Can't perform Put wider than outer width") - - val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( - MT_B -> Bool(true), - MT_BU -> Bool(true), - MT_H -> Bool(outerDataBits >= 16), - MT_HU -> Bool(outerDataBits >= 16), - MT_W -> Bool(outerDataBits >= 32), - MT_WU -> Bool(outerDataBits >= 32), - MT_D -> Bool(outerDataBits >= 64), - MT_Q -> Bool(false))) - - assert(!io.in.acquire.valid || !smallget || read_size_ok, - "Can't perform Get wider than outer width") - - val outerConfig = p.alterPartial({ case TLId => outerTLId }) - val innerConfig = p.alterPartial({ case TLId => innerTLId }) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - alloc = iacq.allocate())(outerConfig) - - val put_block_acquire = PutBlock( - client_xact_id = acq_client_id, - addr_block = acq_addr_block, - addr_beat = if (factor > 1) - Cat(acq_addr_beat, oacq_ctr.value) - else acq_addr_beat, - data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = outer_beat_addr, - addr_byte = outer_byte_addr, - operand_size = iacq.op_size(), - alloc = iacq.allocate())(outerConfig) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = smallput_beat, - data = smallput_data, - wmask = Some(smallput_wmask))(outerConfig) - - val sending_put = Reg(init = Bool(false)) - - val pass_valid = io.in.acquire.valid && !stretch - - io.out.acquire.bits := MuxBundle(Wire(io.out.acquire.bits, init=iacq), Seq( - (sending_put, put_block_acquire), - (shrink, get_block_acquire), - (smallput, put_acquire), - (smallget, get_acquire))) - io.out.acquire.valid := sending_put || pass_valid - io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) - - when (io.in.acquire.fire() && stretch) { - acq_data_buffer := iacq.data - acq_wmask_buffer := iacq.wmask() - acq_client_id := iacq.client_xact_id - acq_addr_block := iacq.addr_block - acq_addr_beat := iacq.addr_beat - sending_put := Bool(true) - } - - when (sending_put && io.out.acquire.ready) { - acq_data_buffer := acq_data_buffer >> outerDataBits - acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits - when (oacq_ctr.inc()) { sending_put := Bool(false) } - } - - val ognt_block = ognt.hasMultibeatData() - val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) - val gnt_client_id = Reg(ognt.client_xact_id) - val gnt_manager_id = Reg(ognt.manager_xact_id) - - val ignt_ctr = Counter(innerDataBeats) - val ognt_ctr = Counter(factor) - val sending_get = Reg(init = Bool(false)) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits)(innerConfig) - - val smallget_grant = ognt.g_type === Grant.getDataBeatType - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), - data = Fill(factor, ognt.data))(innerConfig) - - io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) - io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - - io.in.grant.bits := MuxBundle(Wire(io.in.grant.bits, init=ognt), Seq( - sending_get -> get_block_grant, - smallget_grant -> get_grant)) - - when (io.out.grant.valid && ognt_block && !sending_get) { - gnt_data_buffer(ognt_ctr.value) := ognt.data - when (ognt_ctr.inc()) { - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - sending_get := Bool(true) - } - } - - when (io.in.grant.ready && sending_get) { - ignt_ctr.inc() - sending_get := Bool(false) - } - } else { io.out <> io.in } + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } } class MMIOTileLinkManagerData(implicit p: Parameters) From 515157089468beea03d6851ed605e0db6eb7c465 Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Wed, 8 Jun 2016 15:13:39 -0700 Subject: [PATCH 619/688] Fix valid signal for multibeat grants --- uncore/src/main/scala/bram.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index a5c4cef9..8a822744 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -23,7 +23,9 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module multibeat := s0_getblk } - val s0_valid = io.acquire.valid || multibeat + + val last = Wire(Bool()) + val s0_valid = io.acquire.valid || (multibeat && !last) val s1_valid = Reg(next = s0_valid, init = Bool(false)) val s1_acq = RegEnable(io.acquire.bits, fire_acq) @@ -32,7 +34,7 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module val s1_addr = Cat(s1_acq.addr_block, s1_beat) val raddr = Mux(multibeat, s1_addr, s0_addr) - val last = (s1_acq.addr_beat === UInt(tlDataBeats-1)) + last := (s1_acq.addr_beat === UInt(tlDataBeats-1)) val ren = (io.acquire.valid && (s0_get || s0_getblk)) || (multibeat && !last) val wen = (io.acquire.valid && (s0_put || s0_putblk)) From 93c1b17b52c8f222aa3c90a4994b888caa0c6117 Mon Sep 17 00:00:00 2001 From: mwachs5 Date: Wed, 8 Jun 2016 20:31:13 -0700 Subject: [PATCH 620/688] [debug] Remove erroneous buffer on SB read data (#56) --- uncore/src/main/scala/debug.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 42d38def..eb7eb5df 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -454,7 +454,7 @@ class DebugModule ()(implicit val p:cde.Parameters) val dbRamRdEn = Wire(Bool()) val sbRamAddr = Wire(UInt(width=sbRamAddrWidth)) - val sbRamRdData = Reg (UInt(width=sbRamDataWidth)) + val sbRamRdData = Wire (UInt(width=sbRamDataWidth)) val sbRamWrData = Wire(UInt(width=sbRamDataWidth)) val sbRamWrEn = Wire(Bool()) val sbRamRdEn = Wire(Bool()) From 7014eef339458548aadddca920ab212820b84391 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 8 Jun 2016 11:19:22 -0700 Subject: [PATCH 621/688] ahb: fix bugs found using comparatortest --- uncore/src/main/scala/ahb.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/ahb.scala index d7ab75bd..a6b445fe 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/ahb.scala @@ -140,9 +140,10 @@ class AHBTileLinkIn(supportAtomics: Boolean = false)(implicit val p: Parameters) // Calculate the address, with consideration to put fragments and bursts val addr_block = io.acquire.bits.addr_block - val addr_beat = io.acquire.bits.addr_beat - val addr_burst = Mux(isReadBurst, addr_beat + burst, addr_beat) + val addr_beatin= io.acquire.bits.addr_beat + val addr_burst = Mux(isReadBurst, addr_beatin + burst, addr_beatin) val addr_byte = Mux(isPut, put_addr, io.acquire.bits.addr_byte()) + val addr_beat = Mux(isWriteBurst, UInt(0), addr_burst) val ahbAddr = Cat(addr_block, addr_burst, addr_byte) val ahbSize = Mux(isPut, put_size, Mux(isBurst, UInt(log2Ceil(tlDataBytes)), io.acquire.bits.op_size())) @@ -185,7 +186,7 @@ class AHBTileLinkIn(supportAtomics: Boolean = false)(implicit val p: Parameters) Acquire.getType -> Bool(true), Acquire.getBlockType -> Bool(true), Acquire.putType -> last_wmask, - Acquire.putBlockType -> Bool(true), + Acquire.putBlockType -> last_burst, Acquire.putAtomicType -> MuxLookup(atom_state, Bool(false), Array( s_atom_r -> Bool(true), // they want the old data s_atom_idle1 -> Bool(false), @@ -212,11 +213,13 @@ class AHBTileLinkIn(supportAtomics: Boolean = false)(implicit val p: Parameters) io.request.bits.is_builtin_type := Bool(true) io.request.bits.g_type := io.acquire.bits.getBuiltInGrantType() io.request.bits.client_xact_id := io.acquire.bits.client_xact_id - io.request.bits.addr_beat := addr_burst + io.request.bits.addr_beat := addr_beat val debugBurst = Reg(UInt()) - debugBurst := addr_burst - burst - + when (io.request.valid) { + debugBurst := addr_burst - burst + } + // We only support built-in TileLink requests assert(!io.acquire.valid || io.acquire.bits.is_builtin_type, "AHB bridge only supports builtin TileLink types") // Ensure alignment of address to size @@ -336,7 +339,7 @@ class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) case CacheBlockOffsetBits => hastiAddrBits case AmoAluOperandBits => hastiDataBits }) - val alu = Module(new AMOALU(rhsIsAligned = false)(amo_p)) + val alu = Module(new AMOALU(rhsIsAligned = true)(amo_p)) alu.io.addr := haddr alu.io.cmd := cmd alu.io.typ := hsize From 31b72625aa5f05587ba42a6967a18f9b30890e5c Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 8 Jun 2016 13:11:12 -0700 Subject: [PATCH 622/688] ahb: allow no-ops to progress also when a slave is !hready --- uncore/src/main/scala/ahb.scala | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/ahb.scala index a6b445fe..adee3f8a 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/ahb.scala @@ -270,6 +270,8 @@ class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) val respondTL1 = Reg(init = Bool(false)) val latchAtom0 = Reg(init = Bool(false)) val latchAtom1 = Reg(init = Bool(false)) + val executeAHB0 = Reg(init = Bool(false)) + val executeAHB1 = Reg(init = Bool(false)) val bubble = Reg(init = Bool(true)) // nothing useful in address phase val cmd = Reg(Bits()) val g_type0 = Reg(UInt()) @@ -281,8 +283,8 @@ class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) val grant1 = Reg(new Grant) // It is allowed to progress from Idle/Busy during a wait state - val addrReady = io.ahb.hready || bubble - val dataReady = io.ahb.hready + val addrReady = io.ahb.hready || bubble || (!executeAHB1 && !executeAHB0) + val dataReady = io.ahb.hready || !executeAHB1 // Only accept a new AHBRequest if we have enough buffer space in the pad // to accomodate a persistent drop in TileLink's grant.ready @@ -311,10 +313,12 @@ class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) when (io.request.fire()) { respondTL0 := io.request.bits.respondTL latchAtom0 := io.request.bits.latchAtom + executeAHB0:= io.request.bits.executeAHB bubble := Bool(false) } .otherwise { respondTL0 := Bool(false) latchAtom0 := Bool(false) + executeAHB0:= Bool(false) bubble := Bool(true) // an atom-injected Idle is not a bubble! } } @@ -347,11 +351,17 @@ class AHBBusMaster(supportAtomics: Boolean = false)(implicit val p: Parameters) alu.io.lhs := hrdata // Transfer bulk data phase - // NOTE: this introduces no bubbles because addrReady is a superset of dataReady when (dataReady) { + when (addrReady) { + respondTL1 := respondTL0 + latchAtom1 := latchAtom0 + executeAHB1 := executeAHB0 + } .otherwise { + respondTL1 := Bool(false) + latchAtom1 := Bool(false) + executeAHB1 := Bool(false) + } hwdata1 := Mux(Bool(supportAtomics), alu.io.out, hwdata0) - respondTL1 := respondTL0 - latchAtom1 := latchAtom0 g_type1 := g_type0 client_xact_id1 := client_xact_id0 addr_beat1 := addr_beat0 From a1ebc7347748313018e8e7ce4d2ad3e087f0c786 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 8 Jun 2016 16:15:51 -0700 Subject: [PATCH 623/688] tilelink: don't accidentally make a malformed union Closes #55 --- uncore/src/main/scala/tilelink.scala | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index dde0a619..95e1ebf2 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -358,13 +358,25 @@ object Acquire { operand_size: UInt, opcode: UInt, wmask: UInt, - alloc: Bool): UInt = { + alloc: Bool) + (implicit p: Parameters): UInt = { + + val tlExternal = p(TLKey(p(TLId))) + val tlWriteMaskBits = tlExternal.writeMaskBits + val tlByteAddrBits = log2Up(tlWriteMaskBits) + + // These had better be the right size when we cat them together! + val my_addr_byte = (UInt(0, tlByteAddrBits) | addr_byte)(tlByteAddrBits-1, 0) + val my_operand_size = (UInt(0, MT_SZ) | operand_size)(MT_SZ-1, 0) + val my_opcode = (UInt(0, M_SZ) | opcode)(M_SZ-1, 0) + val my_wmask = (UInt(0, tlWriteMaskBits) | wmask)(tlWriteMaskBits-1, 0) + MuxLookup(a_type, UInt(0), Array( - Acquire.getType -> Cat(addr_byte, operand_size, opcode, alloc), - Acquire.getBlockType -> Cat(operand_size, opcode, alloc), - Acquire.putType -> Cat(wmask, alloc), - Acquire.putBlockType -> Cat(wmask, alloc), - Acquire.putAtomicType -> Cat(addr_byte, operand_size, opcode, alloc), + Acquire.getType -> Cat(my_addr_byte, my_operand_size, my_opcode, alloc), + Acquire.getBlockType -> Cat(my_operand_size, my_opcode, alloc), + Acquire.putType -> Cat(my_wmask, alloc), + Acquire.putBlockType -> Cat(my_wmask, alloc), + Acquire.putAtomicType -> Cat(my_addr_byte, my_operand_size, my_opcode, alloc), Acquire.getPrefetchType -> Cat(M_XRD, alloc), Acquire.putPrefetchType -> Cat(M_XWR, alloc))) } From cee0cf345ed20e6918926ddd3104bec2c2e720df Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Thu, 9 Jun 2016 10:30:23 -0700 Subject: [PATCH 624/688] [debug] Update Debug ROM contents to write F..F to RAM in case of exception --- uncore/src/main/scala/debug.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index eb7eb5df..76f9e1d8 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -73,11 +73,11 @@ object DsbBusConsts { def defaultRomContents : Array[Byte] = Array( 0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, - 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0xf3, 0x24, 0x40, 0xf1, - 0x23, 0x20, 0x90, 0x10, 0x0f, 0x00, 0xf0, 0x0f, 0xf3, 0x24, 0x00, 0xf1, - 0x63, 0xc6, 0x04, 0x00, 0x83, 0x24, 0xc0, 0x43, 0x6f, 0x00, 0xc0, 0x01, - 0x93, 0x94, 0x14, 0x00, 0x63, 0xc6, 0x04, 0x00, 0x83, 0x34, 0x80, 0x43, - 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x00, 0x00, 0x00, 0x23, 0x2e, 0x80, 0x42, + 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0x0f, 0x00, 0xf0, 0x0f, + 0xf3, 0x24, 0x00, 0xf1, 0x63, 0xc6, 0x04, 0x00, 0x83, 0x24, 0xc0, 0x43, + 0x6f, 0x00, 0x80, 0x01, 0x93, 0x94, 0x14, 0x00, 0x63, 0xc6, 0x04, 0x00, + 0x83, 0x34, 0x80, 0x43, 0x6f, 0x00, 0x80, 0x00, 0x13, 0x00, 0x00, 0x00, + 0x23, 0x2e, 0x80, 0x42, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x20, 0x80, 0x10, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, 0x63, 0x04, 0x04, 0x00, 0x6f, 0x00, 0x80, 0x05, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, From e5cfc2dac1a0b055bac8eb021404e1af4c7ee1a6 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Fri, 10 Jun 2016 14:04:28 -0700 Subject: [PATCH 625/688] Add a Smi to TileLink converter (#59) I'm trying to get someone to attach their stuff to Rocket Chip for the upcoming tapout. TileLink sounded too complicated, but Smi went over well. Since the mmioNetwork in Rocket Chip is based on TileLink, it seemed like the easiest thing to do was to write a TileLink to Smi converter so people could use it. It turns out there was already one inside the groundtest unit tests, so I just moved that into uncore (it was inlined into a test case so you couldn't actually use it before). Internally the converter uses Nasti, but I figured that's good enough for now. --- uncore/src/main/scala/smi.scala | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 uncore/src/main/scala/smi.scala diff --git a/uncore/src/main/scala/smi.scala b/uncore/src/main/scala/smi.scala new file mode 100644 index 00000000..dc8b158e --- /dev/null +++ b/uncore/src/main/scala/smi.scala @@ -0,0 +1,31 @@ +// See LICENSE for details + +package uncore + +import Chisel._ +import junctions._ +import cde.Parameters + +/** Convert TileLink protocol to Smi protocol */ +class SmiIOTileLinkIOConverter(val dataWidth: Int, val addrWidth: Int) + (implicit p: Parameters) extends Module { + val io = new Bundle { + val tl = (new ClientUncachedTileLinkIO).flip + val smi = new SmiIO(dataWidth, addrWidth) + } + + def decoupledNastiConnect(outer: NastiIO, inner: NastiIO) { + outer.ar <> Queue(inner.ar) + outer.aw <> Queue(inner.aw) + outer.w <> Queue(inner.w) + inner.r <> Queue(outer.r) + inner.b <> Queue(outer.b) + } + + val tl2nasti = Module(new NastiIOTileLinkIOConverter()) + val nasti2smi = Module(new SmiIONastiIOConverter(dataWidth, addrWidth)) + + tl2nasti.io.tl <> io.tl + decoupledNastiConnect(nasti2smi.io.nasti, tl2nasti.io.nasti) + io.smi <> nasti2smi.io.smi +} From 2d2096e50918eb2176845ebf60b71488682534b9 Mon Sep 17 00:00:00 2001 From: mwachs5 Date: Wed, 15 Jun 2016 15:07:43 -0700 Subject: [PATCH 626/688] Add smaller ROM/RAM for 32-bit debug (#60) --- uncore/src/main/scala/debug.scala | 38 +++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index 76f9e1d8..daa86b82 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -64,12 +64,10 @@ object DsbBusConsts { def sbAddrWidth = 12 def sbIdWidth = 10 - /* These are the default ROM contents, which support - * RV32 and RV64. RV128 are implemented as NOP. - * See the Debug Specification for the code. - */ - - // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h + //These are the default ROM contents, which support + //RV32 and RV64. RV128 are implemented as NOP. + // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h/S + // The code assumes 4*0xF bytes of Debug RAM. def defaultRomContents : Array[Byte] = Array( 0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, @@ -88,6 +86,24 @@ object DsbBusConsts { 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0xdf, 0xfb).map(x => x.toByte) + + // These ROM contents support only RV32 + // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h/S + // The code assumes only 28 bytes of Debug RAM. + + def xlen32OnlyRomContents : Array[Byte] = Array( + 0x6f, 0x00, 0x00, 0x04, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0x0f, 0x00, 0xf0, 0x0f, + 0x83, 0x24, 0x80, 0x41, 0x23, 0x2c, 0x80, 0x40, 0x73, 0x24, 0x40, 0xf1, + 0x23, 0x20, 0x80, 0x10, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, + 0x63, 0x04, 0x04, 0x00, 0x6f, 0x00, 0x80, 0x03, 0x73, 0x24, 0x20, 0x7b, + 0x73, 0x00, 0x20, 0x7b, 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, + 0x13, 0x74, 0x04, 0x1c, 0x13, 0x04, 0x04, 0xf4, 0x63, 0x18, 0x04, 0x00, + 0x0f, 0x10, 0x00, 0x00, 0x23, 0x2c, 0x90, 0x40, 0x67, 0x00, 0x00, 0x40, + 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, 0x73, 0x60, 0x04, 0x7b, + 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, 0xe3, 0x0c, 0x04, 0xfe, + 0x6f, 0xf0, 0xdf, 0xfd).map(x => x.toByte) + } @@ -144,7 +160,7 @@ import DebugModuleAccessType._ * nComponents : The number of components to support debugging. * nDebugBusAddrSize : Size of the Debug Bus Address * nDebugRam Bytes: Size of the Debug RAM (depends on the XLEN of the machine). - * debugRomContents: + * debugRomContents: Optional Sequence of bytes which form the Debug ROM contents. * hasBusMaster: Whether or not a bus master should be included * The size of the accesses supported by the Bus Master. * nSerialPorts : Number of serial ports to instantiate @@ -221,11 +237,15 @@ class DefaultDebugModuleConfig (val ncomponents : Int, val xlen:Int) // the ROM image would need to be // adjusted accordingly. nDebugRamBytes = xlen match{ - case 32 => 64 + case 32 => 28 case 64 => 64 case 128 => 64 }, - debugRomContents = Some(DsbBusConsts.defaultRomContents), + debugRomContents = xlen match { + case 32 => Some(DsbBusConsts.xlen32OnlyRomContents) + case 64 => Some(DsbBusConsts.defaultRomContents) + case 128 => Some(DsbBusConsts.defaultRomContents) + }, hasBusMaster = false, hasAccess128 = false, hasAccess64 = false, From 16bfbda3c9ff3c4a46ff2705eefb58d98f8597e8 Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 4 Apr 2016 22:17:11 -0700 Subject: [PATCH 627/688] Refactor the TransactionTracker logic in all the L2 TileLink Managers. They now share common sub-transactions within traits, and use a common set of state transitions and scoreboarding logic. Tracker allocation logic has also been updated. No changes to external IOs or the TileLink protocol. A new bufferless Broadcast hub is also included, but does not yet pass fuzzing checks. --- uncore/src/main/scala/broadcast.scala | 512 ++++------- uncore/src/main/scala/bufferless.scala | 141 +++ uncore/src/main/scala/cache.scala | 1117 ++++++++++-------------- uncore/src/main/scala/converters.scala | 38 +- uncore/src/main/scala/metadata.scala | 3 +- uncore/src/main/scala/sdq.scala | 118 +++ uncore/src/main/scala/tilelink.scala | 19 +- uncore/src/main/scala/trackers.scala | 546 ++++++++++++ uncore/src/main/scala/uncore.scala | 106 +-- 9 files changed, 1486 insertions(+), 1114 deletions(-) create mode 100644 uncore/src/main/scala/bufferless.scala create mode 100644 uncore/src/main/scala/sdq.scala create mode 100644 uncore/src/main/scala/trackers.scala diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 5ecaa7e2..1c51287f 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -4,390 +4,198 @@ package uncore import Chisel._ import cde.{Parameters, Field} -case object L2StoreDataQueueDepth extends Field[Int] +class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { -trait HasBroadcastHubParameters extends HasCoherenceAgentParameters { - val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats - val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) - val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases -} - -class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p) - with HasBroadcastHubParameters { - val idx = UInt(width = dqIdxBits) - val loc = UInt(width = log2Ceil(nDataQueueLocations)) -} - -object DataQueueLocation { - def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = { - val d = Wire(new DataQueueLocation) - d.idx := idx - d.loc := loc - d - } -} - -class L2BroadcastHub(implicit p: Parameters) extends ManagerCoherenceAgent()(p) - with HasBroadcastHubParameters { - val internalDataBits = new DataQueueLocation().getWidth - val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) - - val usingStoreDataQueue = p.alterPartial({ - case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) - case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) - }) - - // Create SHRs for outstanding transactions - val trackerList = + // Create TSHRs for outstanding transactions + val irelTrackerList = (0 until nReleaseTransactors).map(id => - Module(new BroadcastVoluntaryReleaseTracker(id)(usingStoreDataQueue))) ++ + Module(new BufferedBroadcastVoluntaryReleaseTracker(id))) + val iacqTrackerList = (nReleaseTransactors until nTransactors).map(id => - Module(new BroadcastAcquireTracker(id)(usingStoreDataQueue))) + Module(new BufferedBroadcastAcquireTracker(id))) + val trackerList = irelTrackerList ++ iacqTrackerList // Propagate incoherence flags - trackerList.map(_.io.incoherent := io.incoherent) + trackerList.map(_.io.incoherent) foreach { _ := io.incoherent } - // Queue to store impending Put data - val sdq = Reg(Vec(sdqDepth, io.iacq().data)) - val sdq_val = Reg(init=Bits(0, sdqDepth)) - val sdq_alloc_id = PriorityEncoder(~sdq_val) - val sdq_rdy = !sdq_val.andR - val sdq_enq = trackerList.map( t => - (t.io.alloc.iacq || t.io.matches.iacq) && - t.io.inner.acquire.fire() && - t.io.iacq().hasData() - ).reduce(_||_) - when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } + // Create an arbiter for the one memory port + val outerList = trackerList.map(_.io.outer) + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size) + (p.alterPartial({ case TLId => p(OuterTLId) }))) + outer_arb.io.in <> outerList + io.outer <> outer_arb.io.out // Handle acquire transaction initiation val irel_vs_iacq_conflict = - io.inner.acquire.valid && - io.inner.release.valid && - io.irel().conflicts(io.iacq()) - val sdqLoc = List.fill(nTransactors) { - DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits - } - doInputRoutingWithAllocation( - io.inner.acquire, - trackerList.map(_.io.inner.acquire), - trackerList.map(_.io.matches.iacq), - trackerList.map(_.io.alloc.iacq), - Some(sdqLoc), - Some(sdq_rdy && !irel_vs_iacq_conflict), - Some(sdq_rdy)) + io.inner.acquire.valid && + io.inner.release.valid && + io.irel().conflicts(io.iacq()) - // Queue to store impending Voluntary Release data - val voluntary = io.irel().isVoluntary() - val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() - val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width - val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1 - when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } + doInputRoutingWithAllocation( + in = io.inner.acquire, + outs = trackerList.map(_.io.inner.acquire), + allocs = trackerList.map(_.io.alloc_iacq), + allocOverride = !irel_vs_iacq_conflict) // Handle releases, which might be voluntary and might have data - val vwbqLoc = (0 until nTransactors).map(i => - (DataQueueLocation(rel_data_cnt, - (if(i < nReleaseTransactors) inVolWBQueue - else inClientReleaseQueue)).toBits)) doInputRoutingWithAllocation( - io.inner.release, - trackerList.map(_.io.inner.release), - trackerList.map(_.io.matches.irel), - trackerList.map(_.io.alloc.irel), - Some(vwbqLoc)) + in = io.inner.release, + outs = trackerList.map(_.io.inner.release), + allocs = trackerList.map(_.io.alloc_irel)) // Wire probe requests and grant reply to clients, finish acks from clients - // Note that we bypass the Grant data subbundles - doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) - io.inner.grant.bits.data := io.outer.grant.bits.data - io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) - - // Create an arbiter for the one memory port - val outer_arb = Module(new ClientUncachedTileLinkIOArbiter(trackerList.size) - (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) }))) - outer_arb.io.in <> trackerList.map(_.io.outer) - // Get the pending data out of the store data queue - val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) - val is_in_sdq = outer_data_ptr.loc === inStoreQueue - val free_sdq = io.outer.acquire.fire() && - io.outer.acquire.bits.hasData() && - outer_data_ptr.loc === inStoreQueue - io.outer <> outer_arb.io.out - io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( - inStoreQueue -> sdq(outer_data_ptr.idx), - inVolWBQueue -> vwbdq(outer_data_ptr.idx))) - - // Update SDQ valid bits - when (io.outer.acquire.valid || sdq_enq) { - sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | - PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) - } } class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { - val io = new ManagerXactTrackerIO + val io = new HierarchicalXactTrackerIO pinAllReadyValidLow(io) } -class BroadcastVoluntaryReleaseTracker(trackerId: Int) - (implicit p: Parameters) extends BroadcastXactTracker()(p) { - val s_idle :: s_busy :: Nil = Enum(UInt(), 2) - val state = Reg(init=s_idle) +trait BroadcastsToAllClients extends HasCoherenceAgentParameters { + val coh = HierarchicalMetadata.onReset + val inner_coh = coh.inner + val outer_coh = coh.outer + def full_representation = ~UInt(0, width = innerNCachingClients) +} - val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({ case TLId => innerTLId }))) - val coh = ManagerMetadata.onReset - - val pending_irels = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_writes = Reg(init=Bits(0, width = io.outer.tlDataBeats)) - val pending_ignt = Reg(init=Bool(false)) - - val all_pending_done = !(pending_irels.orR || pending_writes.orR || pending_ignt) - - // Accept a voluntary Release (and any further beats of data) - pending_irels := (pending_irels & dropPendingBitWhenBeatHasData(io.inner.release)) - io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irels.orR - when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } - - // Write the voluntarily written back data to outer memory using an Acquire.PutBlock - //TODO: Use io.outer.release instead? - pending_writes := (pending_writes & dropPendingBitWhenBeatHasData(io.outer.acquire)) | - addPendingBitWhenBeatHasData(io.inner.release) - val curr_write_beat = PriorityEncoder(pending_writes) - io.outer.acquire.valid := state === s_busy && pending_writes.orR - io.outer.acquire.bits := PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = curr_write_beat, - data = xact.data_buffer(curr_write_beat)) - (p.alterPartial({ case TLId => outerTLId })) - - // Send an acknowledgement - io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irels && io.outer.grant.valid - io.inner.grant.bits := coh.makeGrant(xact) - when(io.inner.grant.fire()) { pending_ignt := Bool(false) } - io.outer.grant.ready := state === s_busy && io.inner.grant.ready - - // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.valid && io.alloc.irel) { - xact := io.irel() - when(io.irel().hasMultibeatData()) { - pending_irels := dropPendingBitWhenBeatHasData(io.inner.release) - }. otherwise { - pending_irels := UInt(0) - } - pending_writes := addPendingBitWhenBeatHasData(io.inner.release) - pending_ignt := io.irel().requiresAck() - state := s_busy - } - when(state === s_busy && all_pending_done) { state := s_idle } - - // These IOs are used for routing in the parent - io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) - io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && io.irel().isVoluntary() - io.matches.oprb := Bool(false) +abstract class BroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) + extends VoluntaryReleaseTracker(trackerId)(p) + with EmitsVoluntaryReleases + with BroadcastsToAllClients { + val io = new HierarchicalXactTrackerIO + pinAllReadyValidLow(io) // Checks for illegal behavior - assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), + assert(!(state === s_idle && io.inner.release.fire() && io.alloc_irel.should && !io.irel().isVoluntary()), "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } -class BroadcastAcquireTracker(trackerId: Int) - (implicit p: Parameters) extends BroadcastXactTracker()(p) { - val s_idle :: s_probe :: s_mem_read :: s_mem_write :: s_make_grant :: s_mem_resp :: s_ack :: Nil = Enum(UInt(), 7) - val state = Reg(init=s_idle) +abstract class BroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) + extends AcquireTracker(trackerId)(p) + with EmitsVoluntaryReleases + with BroadcastsToAllClients { + val io = new HierarchicalXactTrackerIO + pinAllReadyValidLow(io) - val xact = Reg(new BufferedAcquireFromSrc()(p.alterPartial({ case TLId => innerTLId }))) - val coh = ManagerMetadata.onReset + val alwaysWriteFullBeat = false + val nSecondaryMisses = 1 + def iacq_can_merge = Bool(false) - assert(!(state =/= s_idle && xact.isBuiltInType() && - Vec(Acquire.putAtomicType, Acquire.getPrefetchType, Acquire.putPrefetchType).contains(xact.a_type)), - "Broadcast Hub does not support PutAtomics or prefetches") // TODO - - val release_count = Reg(init=UInt(0, width = log2Up(io.inner.tlNCachingClients+1))) - val pending_probes = Reg(init=Bits(0, width = io.inner.tlNCachingClients)) - val curr_p_id = PriorityEncoder(pending_probes) - val mask_self = SInt(-1, width = io.inner.tlNCachingClients) - .toUInt - .bitSet(io.inner.acquire.bits.client_id, io.inner.acquire.bits.requiresSelfProbe()) - val mask_incoherent = mask_self & ~io.incoherent.toBits - - val collect_iacq_data = Reg(init=Bool(false)) - val iacq_data_valid = Reg(init=Bits(0, width = innerDataBeats)) - val iacq_data_done = connectIncomingDataBeatCounter(io.inner.acquire) - val irel_data_done = connectIncomingDataBeatCounter(io.inner.release) - val (ignt_data_cnt, ignt_data_done) = connectOutgoingDataBeatCounter(io.inner.grant) - val (oacq_data_cnt, oacq_data_done) = connectOutgoingDataBeatCounter(io.outer.acquire) - val ognt_data_done = connectIncomingDataBeatCounter(io.outer.grant) - val pending_ognt_ack = Reg(init=Bool(false)) - val pending_outer_write = xact.hasData() - val pending_outer_write_ = io.iacq().hasData() - val pending_outer_read = io.ignt().hasData() - val pending_outer_read_ = coh.makeGrant(io.iacq(), UInt(trackerId)).hasData() - val subblock_type = xact.isSubBlockType() - - // These IOs are used for routing in the parent - io.matches.iacq := (state =/= s_idle) && xact.conflicts(io.iacq()) - io.matches.irel := (state =/= s_idle) && xact.conflicts(io.irel()) && !io.irel().isVoluntary() - io.matches.oprb := Bool(false) - - val outerParams = p.alterPartial({ case TLId => outerTLId }) - - val oacq_probe = PutBlock( - client_xact_id = UInt(trackerId), - addr_block = io.irel().addr_block, - addr_beat = io.irel().addr_beat, - data = io.irel().data)(outerParams) - - val oacq_write_beat = Put( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = xact.addr_beat, - data = xact.data_buffer(0), - wmask = xact.wmask())(outerParams) - - val oacq_write_block = PutBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = oacq_data_cnt, - data = xact.data_buffer(oacq_data_cnt), - wmask = xact.wmask_buffer(oacq_data_cnt))(outerParams) - - val oacq_read_beat = Get( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block, - addr_beat = xact.addr_beat, - addr_byte = xact.addr_byte(), - operand_size = xact.op_size(), - alloc = Bool(false))(outerParams) - - val oacq_read_block = GetBlock( - client_xact_id = UInt(trackerId), - addr_block = xact.addr_block)(outerParams) - - io.outer.acquire.valid := Bool(false) - io.outer.acquire.bits := Mux(state === s_probe, oacq_probe, - Mux(state === s_mem_write, - Mux(subblock_type, oacq_write_beat, oacq_write_block), - Mux(subblock_type, oacq_read_beat, oacq_read_block))) - io.outer.grant.ready := Bool(false) - - io.inner.probe.valid := Bool(false) - io.inner.probe.bits := coh.makeProbe(curr_p_id, xact) - - io.inner.grant.valid := Bool(false) - io.inner.grant.bits := coh.makeGrant(xact, UInt(trackerId)) // Data bypassed in parent - - io.inner.acquire.ready := Bool(false) - io.inner.release.ready := Bool(false) - io.inner.finish.ready := Bool(false) - - assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_id =/= xact.client_id), - "AcquireTracker accepted data beat from different network source than initial request.") - - assert(!(state =/= s_idle && collect_iacq_data && io.inner.acquire.fire() && - io.iacq().client_xact_id =/= xact.client_xact_id), - "AcquireTracker accepted data beat from different client transaction than initial request.") - - assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq && - io.iacq().hasMultibeatData() && io.iacq().addr_beat =/= UInt(0)), + // Checks for illegal behavior + // TODO: this could be allowed, but is a useful check against allocation gone wild + assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc_iacq.should && + io.iacq().hasMultibeatData() && !io.iacq().first()), "AcquireTracker initialized with a tail data beat.") - when(collect_iacq_data) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid) { - xact.data_buffer(io.iacq().addr_beat) := io.iacq().data - xact.wmask_buffer(io.iacq().addr_beat) := io.iacq().wmask() - iacq_data_valid := iacq_data_valid.bitSet(io.iacq().addr_beat, Bool(true)) - } - when(iacq_data_done) { collect_iacq_data := Bool(false) } - } + assert(!(state =/= s_idle && pending_ignt && xact_iacq.isPrefetch()), + "Broadcast Hub does not support Prefetches.") + + assert(!(state =/= s_idle && pending_ignt && xact_iacq.isAtomic()), + "Broadcast Hub does not support PutAtomics.") +} + +class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) + extends BroadcastVoluntaryReleaseTracker(trackerId)(p) + with HasDataBuffer { + + // Tell the parent if any incoming messages conflict with the ongoing transaction + routeInParent() + io.alloc_iacq.can := Bool(false) + + // Start transaction by accepting inner release + innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) + + // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat + io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact + + when(irel_is_allocating) { pending_orel := io.irel().hasData() } + + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } + + // Dispatch outer release + outerRelease( + coh = outer_coh.onHit(M_XWR), + data = data_buffer(vol_ognt_counter.up.idx)) + + quiesce() +} + +class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) + extends BroadcastAcquireTracker(trackerId)(p) + with HasByteWriteMaskBuffer { + + // Setup IOs used for routing in the parent + routeInParent() + io.alloc_irel.can := Bool(false) + + // First, take care of accpeting new acquires or secondary misses + // Handling of primary and secondary misses' data and write mask merging + innerAcquire( + can_alloc = Bool(false), + next = s_inner_probe) + + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + + // Track which clients yet need to be probed and make Probe message + // If a writeback occurs, we can forward its data via the buffer, + // and skip having to go outwards + val skip_outer_acquire = pending_ignt_data.andR + + innerProbe( + inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block), + Mux(!skip_outer_acquire, s_outer_acquire, s_busy)) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data, and may be unexpected voluntary releases + def irel_can_merge = io.irel().conflicts(xact_addr_block) && + io.irel().isVoluntary() && + !Vec(s_idle, s_meta_write).contains(state) && + !all_pending_done && + !io.outer.grant.fire() && + !io.inner.grant.fire() && + !vol_ignt_counter.pending + + innerRelease(block_vol_ignt = vol_ognt_counter.pending) + + //TODO: accept vol irels when state === s_idle, operate like the VolRelTracker + io.inner.release.ready := irel_can_merge || irel_same_xact + + mergeDataInner(io.inner.release) + + // If there was a writeback, forward it outwards + outerRelease( + coh = outer_coh.onHit(M_XWR), + data = data_buffer(vol_ognt_counter.up.idx)) + + // Send outer request for miss + outerAcquire( + caching = !xact_iacq.isBuiltInType(), + coh = outer_coh, + data = data_buffer(ognt_counter.up.idx), + wmask = wmask_buffer(ognt_counter.up.idx), + next = s_busy) + + // Handle the response from outer memory + mergeDataOuter(io.outer.grant) + + // Acknowledge or respond with data + innerGrant( + data = data_buffer(ignt_data_idx), + external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending) + + when(iacq_is_allocating) { + wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle + initializeProbes() + } + + initDataInner(io.inner.acquire) + + // Wait for everything to quiesce + quiesce() - when(pending_ognt_ack) { - io.outer.grant.ready := Bool(true) - when(io.outer.grant.valid) { pending_ognt_ack := Bool(false) } - //TODO add finish queue if this isnt the last level manager - } - - switch (state) { - is(s_idle) { - io.inner.acquire.ready := Bool(true) - when(io.inner.acquire.valid && io.alloc.iacq) { - xact := io.iacq() - xact.data_buffer(UInt(0)) := io.iacq().data - xact.wmask_buffer(UInt(0)) := io.iacq().wmask() - collect_iacq_data := io.iacq().hasMultibeatData() - iacq_data_valid := io.iacq().hasData() << io.iacq().addr_beat - val needs_probes = mask_incoherent.orR - when(needs_probes) { - pending_probes := mask_incoherent - release_count := PopCount(mask_incoherent) - } - state := Mux(needs_probes, s_probe, - Mux(pending_outer_write_, s_mem_write, - Mux(pending_outer_read_, s_mem_read, s_make_grant))) - } - } - is(s_probe) { - // Generate probes - io.inner.probe.valid := pending_probes.orR - when(io.inner.probe.ready) { - pending_probes := pending_probes & ~UIntToOH(curr_p_id) - } - - // Handle releases, which may have data to be written back - val matches = io.matches.irel - io.inner.release.ready := (!io.irel().hasData() || io.outer.acquire.ready) && matches - when(io.inner.release.valid && matches) { - when(io.irel().hasData()) { - io.outer.acquire.valid := Bool(true) - when(io.outer.acquire.ready) { - when(oacq_data_done) { - pending_ognt_ack := Bool(true) - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } .otherwise { - release_count := release_count - UInt(1) - when(release_count === UInt(1)) { - state := Mux(pending_outer_write, s_mem_write, - Mux(pending_outer_read, s_mem_read, s_make_grant)) - } - } - } - } - is(s_mem_write) { // Write data to outer memory - io.outer.acquire.valid := !pending_ognt_ack && (!collect_iacq_data || iacq_data_valid(oacq_data_cnt)) - when(oacq_data_done) { - pending_ognt_ack := Bool(true) - state := Mux(pending_outer_read, s_mem_read, s_mem_resp) - } - } - is(s_mem_read) { // Read data from outer memory (possibly what was just written) - io.outer.acquire.valid := !pending_ognt_ack - when(io.outer.acquire.fire()) { state := s_mem_resp } - } - is(s_mem_resp) { // Wait to forward grants from outer memory - io.outer.grant.ready := io.inner.grant.ready - io.inner.grant.valid := io.outer.grant.valid - when(ignt_data_done) { - state := Mux(io.ignt().requiresAck(), s_ack, s_idle) - } - } - is(s_make_grant) { // Manufacture a local grant (some kind of permission upgrade) - io.inner.grant.valid := Bool(true) - when(io.inner.grant.ready) { - state := Mux(io.ignt().requiresAck(), s_ack, s_idle) - } - } - is(s_ack) { // Wait for transaction to complete - io.inner.finish.ready := Bool(true) - when(io.inner.finish.valid) { state := s_idle } - } - } } diff --git a/uncore/src/main/scala/bufferless.scala b/uncore/src/main/scala/bufferless.scala new file mode 100644 index 00000000..e5ddcb98 --- /dev/null +++ b/uncore/src/main/scala/bufferless.scala @@ -0,0 +1,141 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import cde.{Parameters, Field} + + +class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { + + // Create TSHRs for outstanding transactions + val irelTrackerList = + (0 until nReleaseTransactors).map(id => + Module(new BufferlessBroadcastVoluntaryReleaseTracker(id))) + val iacqTrackerList = + (nReleaseTransactors until nTransactors).map(id => + Module(new BufferlessBroadcastAcquireTracker(id))) + val trackerList = irelTrackerList ++ iacqTrackerList + + // Propagate incoherence flags + trackerList.map(_.io.incoherent) foreach { _ := io.incoherent } + + // Create an arbiter for the one memory port + val outerList = trackerList.map(_.io.outer) + val outer_arb = Module(new ClientTileLinkIOArbiter(outerList.size) + (p.alterPartial({ case TLId => p(OuterTLId) }))) + outer_arb.io.in <> outerList + io.outer <> outer_arb.io.out + + // Handle acquire transaction initiation + val irel_vs_iacq_conflict = + io.inner.acquire.valid && + io.inner.release.valid && + io.irel().conflicts(io.iacq()) + + doInputRoutingWithAllocation( + in = io.inner.acquire, + outs = trackerList.map(_.io.inner.acquire), + allocs = trackerList.map(_.io.alloc_iacq), + allocOverride = !irel_vs_iacq_conflict) + io.outer.acquire.bits.data := io.inner.acquire.bits.data + io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat + + // Handle releases, which might be voluntary and might have data + doInputRoutingWithAllocation( + in = io.inner.release, + outs = trackerList.map(_.io.inner.release), + allocs = trackerList.map(_.io.alloc_irel)) + io.outer.release.bits.data := io.inner.release.bits.data + io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat + + // Wire probe requests and grant reply to clients, finish acks from clients + doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) + + doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) + io.inner.grant.bits.data := io.outer.grant.bits.data + io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat + + doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) +} + +class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) + extends BroadcastVoluntaryReleaseTracker(trackerId)(p) { + + // Tell the parent if any incoming messages conflict with the ongoing transaction + routeInParent() + io.alloc_iacq.can := Bool(false) + + // Start transaction by accepting inner release + innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) + + // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat + // and if the outer relase path is clear + val irel_could_accept = state === s_idle || irel_can_merge || irel_same_xact + io.inner.release.ready := irel_could_accept && + (!io.irel().hasData() || io.outer.release.ready) + + // Dispatch outer release + outerRelease(coh = outer_coh.onHit(M_XWR)) + io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data + + quiesce() +} + +class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) + extends BroadcastAcquireTracker(trackerId)(p) { + + // Setup IOs used for routing in the parent + routeInParent() + io.alloc_irel.can := Bool(false) + + // First, take care of accpeting new acquires or secondary misses + // Handling of primary and secondary misses' data and write mask merging + innerAcquire( + can_alloc = Bool(false), + next = s_inner_probe) + + val iacq_could_accept = state === s_outer_acquire || iacq_can_merge || iacq_same_xact + io.inner.acquire.ready := iacq_could_accept && + (!io.iacq().hasData() || io.outer.acquire.fire()) + + // Track which clients yet need to be probed and make Probe message + innerProbe( + inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block), + s_outer_acquire) + + // Handle incoming releases from clients, which may reduce sharer counts + // and/or write back dirty data, and may be unexpected voluntary releases + def irel_can_merge = io.irel().conflicts(xact_addr_block) && + io.irel().isVoluntary() && + !vol_ignt_counter.pending && + (state =/= s_idle) + + innerRelease(block_vol_ignt = vol_ognt_counter.pending) + + val irel_could_accept = irel_can_merge || irel_same_xact + io.inner.release.ready := irel_could_accept && + (!io.irel().hasData() || io.outer.release.ready) + + // If there was a writeback, forward it outwards + outerRelease( + coh = outer_coh.onHit(M_XWR), + buffering = Bool(false)) + + // Send outer request for miss + outerAcquire( + caching = !xact_iacq.isBuiltInType(), + buffering = Bool(false), + coh = outer_coh, + next = s_busy) + + // Handle the response from outer memory + io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data + + // Acknowledge or respond with data + innerGrant(external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending) + + when(iacq_is_allocating) { initializeProbes() } + + // Wait for everything to quiesce + quiesce() +} diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index d5efde13..2ea72f0d 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -181,13 +181,17 @@ class MetadataArray[T <: Metadata](onReset: () => T)(implicit p: Parameters) ext case object L2DirectoryRepresentation extends Field[DirectoryRepresentation] -trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { +trait HasOuterCacheParameters extends HasCacheParameters with HasCoherenceAgentParameters { val cacheId = p(CacheId) val idxLSB = cacheIdBits val idxMSB = idxLSB + idxBits - 1 val tagLSB = idxLSB + idxBits - def inSameSet(addr1: UInt, addr2: UInt): Bool = addr1(idxMSB,idxLSB) === addr2(idxMSB,idxLSB) - def haveSameTag(addr1: UInt, addr2: UInt): Bool = addr1 >> UInt(tagLSB) === addr2 >> UInt(tagLSB) + def inSameSet(block: HasCacheBlockAddress, addr: UInt): Bool = { + block.addr_block(idxMSB,idxLSB) === addr(idxMSB,idxLSB) + } + def haveSameTag(block: HasCacheBlockAddress, addr: UInt): Bool = { + block.addr_block >> UInt(tagLSB) === addr >> UInt(tagLSB) + } //val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats @@ -198,11 +202,11 @@ trait HasL2HellaCacheParameters extends HasCacheParameters with HasCoherenceAgen require(rowBits == innerDataBits) // TODO: relax this by improving s_data_* states val nSecondaryMisses = p(NSecondaryMisses) val isLastLevelCache = true - val ignoresWriteMask = !p(ECCCode).isEmpty + val alwaysWriteFullBeat = !p(ECCCode).isEmpty } abstract class L2HellaCacheModule(implicit val p: Parameters) extends Module - with HasL2HellaCacheParameters { + with HasOuterCacheParameters { def doInternalOutputArbitration[T <: Data : ClassTag]( out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { @@ -218,40 +222,51 @@ abstract class L2HellaCacheModule(implicit val p: Parameters) extends Module } abstract class L2HellaCacheBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) - with HasL2HellaCacheParameters + with HasOuterCacheParameters trait HasL2Id extends HasCoherenceAgentParameters { val id = UInt(width = log2Up(nTransactors + 1)) } -trait HasL2InternalRequestState extends HasL2HellaCacheParameters { +trait HasL2InternalRequestState extends HasOuterCacheParameters { val tag_match = Bool() val meta = new L2Metadata val way_en = Bits(width = nWays) } -trait HasL2BeatAddr extends HasL2HellaCacheParameters { +trait HasL2BeatAddr extends HasOuterCacheParameters { val addr_beat = UInt(width = log2Up(refillCycles)) } -trait HasL2Data extends HasL2HellaCacheParameters +trait HasL2Data extends HasOuterCacheParameters with HasL2BeatAddr { val data = UInt(width = rowBits) def hasData(dummy: Int = 0) = Bool(true) def hasMultibeatData(dummy: Int = 0) = Bool(refillCycles > 1) } -class L2Metadata(implicit p: Parameters) extends Metadata()(p) with HasL2HellaCacheParameters { +class L2Metadata(implicit p: Parameters) extends Metadata()(p) with HasOuterCacheParameters { val coh = new HierarchicalMetadata } object L2Metadata { - def apply(tag: Bits, coh: HierarchicalMetadata)(implicit p: Parameters) = { + def apply(tag: Bits, coh: HierarchicalMetadata) + (implicit p: Parameters): L2Metadata = { val meta = Wire(new L2Metadata) meta.tag := tag meta.coh := coh meta } + + def apply( + tag: Bits, + inner: ManagerMetadata, + outer: ClientMetadata)(implicit p: Parameters): L2Metadata = { + val coh = Wire(new HierarchicalMetadata) + coh.inner := inner + coh.outer := outer + apply(tag, coh) + } } class L2MetaReadReq(implicit p: Parameters) extends MetaReadReq()(p) with HasL2Id { @@ -267,12 +282,12 @@ class L2MetaResp(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2Id with HasL2InternalRequestState -trait HasL2MetaReadIO extends HasL2HellaCacheParameters { +trait HasL2MetaReadIO extends HasOuterCacheParameters { val read = Decoupled(new L2MetaReadReq) val resp = Valid(new L2MetaResp).flip } -trait HasL2MetaWriteIO extends HasL2HellaCacheParameters { +trait HasL2MetaWriteIO extends HasOuterCacheParameters { val write = Decoupled(new L2MetaWriteReq) } @@ -280,6 +295,10 @@ class L2MetaRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2MetaReadIO with HasL2MetaWriteIO +trait HasL2MetaRWIO extends HasOuterCacheParameters { + val meta = new L2MetaRWIO +} + class L2MetadataArray(implicit p: Parameters) extends L2HellaCacheModule()(p) { val io = new L2MetaRWIO().flip @@ -330,21 +349,55 @@ class L2DataReadReq(implicit p: Parameters) extends L2HellaCacheBundle()(p) val way_en = Bits(width = nWays) } +object L2DataReadReq { + def apply( + id: UInt, + way_en: UInt, + addr_idx: UInt, + addr_beat: UInt)(implicit p: Parameters) = { + val req = Wire(new L2DataReadReq) + req.id := id + req.way_en := way_en + req.addr_idx := addr_idx + req.addr_beat := addr_beat + req + } +} + class L2DataWriteReq(implicit p: Parameters) extends L2DataReadReq()(p) with HasL2Data { val wmask = Bits(width = rowBits/8) } +object L2DataWriteReq { + def apply( + id: UInt, + way_en: UInt, + addr_idx: UInt, + addr_beat: UInt, + wmask: UInt, + data: UInt)(implicit p: Parameters) = { + val req = Wire(new L2DataWriteReq) + req.id := id + req.way_en := way_en + req.addr_idx := addr_idx + req.addr_beat := addr_beat + req.wmask := wmask + req.data := data + req + } +} + class L2DataResp(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2Id with HasL2Data -trait HasL2DataReadIO extends HasL2HellaCacheParameters { +trait HasL2DataReadIO extends HasOuterCacheParameters { val read = Decoupled(new L2DataReadReq) val resp = Valid(new L2DataResp).flip } -trait HasL2DataWriteIO extends HasL2HellaCacheParameters { +trait HasL2DataWriteIO extends HasOuterCacheParameters { val write = Decoupled(new L2DataWriteReq) } @@ -352,6 +405,10 @@ class L2DataRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2DataReadIO with HasL2DataWriteIO +trait HasL2DataRWIO extends HasOuterCacheParameters { + val data = new L2DataRWIO +} + class L2DataArray(delay: Int)(implicit p: Parameters) extends L2HellaCacheModule()(p) { val io = new L2DataRWIO().flip @@ -371,7 +428,7 @@ class L2DataArray(delay: Int)(implicit p: Parameters) extends L2HellaCacheModule } class L2HellaCacheBank(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) - with HasL2HellaCacheParameters { + with HasOuterCacheParameters { require(isPow2(nSets)) require(isPow2(nWays)) @@ -385,21 +442,22 @@ class L2HellaCacheBank(implicit p: Parameters) extends HierarchicalCoherenceAgen data.io <> tshrfile.io.data } -class TSHRFileIO(implicit p: Parameters) extends HierarchicalTLIO()(p) { - val meta = new L2MetaRWIO - val data = new L2DataRWIO -} +class TSHRFileIO(implicit p: Parameters) extends HierarchicalTLIO()(p) + with HasL2MetaRWIO + with HasL2DataRWIO class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) with HasCoherenceAgentWiringHelpers { val io = new TSHRFileIO // Create TSHRs for outstanding transactions - val trackerList = + val irelTrackerList = (0 until nReleaseTransactors).map(id => - Module(new L2VoluntaryReleaseTracker(id))) ++ + Module(new CacheVoluntaryReleaseTracker(id))) + val iacqTrackerList = (nReleaseTransactors until nTransactors).map(id => - Module(new L2AcquireTracker(id))) + Module(new CacheAcquireTracker(id))) + val trackerList = irelTrackerList ++ iacqTrackerList // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors)) @@ -414,25 +472,23 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) val irel_vs_iacq_conflict = io.inner.acquire.valid && io.inner.release.valid && - inSameSet(io.inner.acquire.bits.addr_block, io.inner.release.bits.addr_block) + inSameSet(io.inner.acquire.bits, io.inner.release.bits.addr_block) doInputRoutingWithAllocation( - io.inner.acquire, - trackerList.map(_.io.inner.acquire), - trackerList.map(_.io.matches.iacq), - trackerList.map(_.io.alloc.iacq), + in = io.inner.acquire, + outs = trackerList.map(_.io.inner.acquire), + allocs = trackerList.map(_.io.alloc_iacq), allocOverride = !irel_vs_iacq_conflict) - assert(PopCount(trackerList.map(_.io.alloc.iacq)) <= UInt(1), + assert(PopCount(trackerList.map(_.io.alloc_iacq.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Acquire") // Wire releases from clients doInputRoutingWithAllocation( - io.inner.release, - trackerAndWbIOs.map(_.inner.release), - trackerAndWbIOs.map(_.matches.irel), - trackerAndWbIOs.map(_.alloc.irel)) + in = io.inner.release, + outs = trackerAndWbIOs.map(_.inner.release), + allocs = trackerAndWbIOs.map(_.alloc_irel)) - assert(PopCount(trackerAndWbIOs.map(_.alloc.irel)) <= UInt(1), + assert(PopCount(trackerAndWbIOs.map(_.alloc_irel.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Release") // Wire probe requests and grant reply to clients, finish acks from clients @@ -457,21 +513,13 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) } -class L2XactTrackerIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) { - val data = new L2DataRWIO - val meta = new L2MetaRWIO - val wb = new L2WritebackIO -} +class L2XactTrackerIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) + with HasL2DataRWIO + with HasL2MetaRWIO + with HasL2WritebackIO -abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) - with HasL2HellaCacheParameters { - class CacheBlockBuffer { // TODO - val buffer = Reg(Bits(width = p(CacheBlockBytes)*8)) - - def internal = Vec(internalDataBeats, Bits(width = rowBits)).fromBits(buffer) - def inner = Vec(innerDataBeats, Bits(width = innerDataBits)).fromBits(buffer) - def outer = Vec(outerDataBeats, Bits(width = outerDataBits)).fromBits(buffer) - } +trait HasRowBeatCounters extends HasOuterCacheParameters with HasPendingBitHelpers { + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt): Unit def connectDataBeatCounter[S <: L2HellaCacheBundle](inc: Bool, data: S, beat: UInt, full_block: Bool) = { if(data.refillCycles > 1) { @@ -505,208 +553,115 @@ abstract class L2XactTracker(implicit p: Parameters) extends XactTracker()(p) def dropPendingBitInternal[T <: L2HellaCacheBundle with HasL2BeatAddr] (in: ValidIO[T]) = ~Fill(in.bits.refillCycles, in.valid) | ~UIntToOH(in.bits.addr_beat) - def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = { - val a = in.bits - val isPartial = a.wmask() =/= Acquire.fullWriteMask - addPendingBitWhenBeat(in.fire() && (isPartial || Bool(ignoresWriteMask)), a) + // TODO: Deal with the possibility that rowBits != tlDataBits + def mergeDataInternal[T <: L2HellaCacheBundle with HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { + when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } } } -class L2VoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { - val io = new L2XactTrackerIO - pinAllReadyValidLow(io) +trait ReadsFromOuterCacheDataArray extends HasCoherenceMetadataBuffer + with HasRowBeatCounters + with HasDataBuffer { + def io: HasL2DataRWIO - val s_idle :: s_meta_read :: s_meta_resp :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 5) - val state = Reg(init=s_idle) + val pending_reads = Reg(init=Bits(0, width = innerDataBeats)) + val pending_resps = Reg(init=Bits(0, width = innerDataBeats)) + val curr_read_beat = PriorityEncoder(pending_reads) - val xact = Reg(new BufferedReleaseFromSrc()(p.alterPartial({case TLId => p(InnerTLId)}))) - val xact_way_en = Reg{ Bits(width = nWays) } - val xact_old_meta = Reg{ new L2Metadata } - val coh = xact_old_meta.coh + def readDataArray(drop_pending_bit: UInt, + add_pending_bit: UInt = UInt(0), + block_pending_read: Bool = Bool(false)) { + val port = io.data + pending_reads := (pending_reads & dropPendingBit(port.read) & drop_pending_bit) | add_pending_bit + port.read.valid := state === s_busy && pending_reads.orR && !block_pending_read + port.read.bits := L2DataReadReq( + id = UInt(trackerId), + way_en = xact_way_en, + addr_idx = xact_addr_idx, + addr_beat = curr_read_beat) - val pending_irel_beats = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_ignt = Reg(init=Bool(false)) + pending_resps := (pending_resps & dropPendingBitInternal(port.resp)) | + addPendingBitInternal(port.read) - val all_pending_done = - !(pending_writes.orR || - pending_irel_beats.orR || - pending_ignt) + scoreboard += (pending_reads.orR, pending_resps.orR) - // These IOs are used for routing in the parent - io.matches.iacq := (state =/= s_idle) && inSameSet(io.iacq().addr_block, xact.addr_block) - io.matches.irel := (state =/= s_idle) && io.irel().conflicts(xact) - io.matches.oprb := (state =/= s_idle) && io.oprb().conflicts(xact) + mergeDataInternal(port.resp) + } +} - // Accept a voluntary Release (and any further beats of data) - pending_irel_beats := (pending_irel_beats & dropPendingBitWhenBeatHasData(io.inner.release)) - io.inner.release.ready := ((state === s_idle) && io.irel().isVoluntary()) || pending_irel_beats.orR - when(io.inner.release.fire()) { xact.data_buffer(io.irel().addr_beat) := io.irel().data } +trait WritesToOuterCacheDataArray extends HasCoherenceMetadataBuffer + with HasRowBeatCounters + with HasDataBuffer { + def io: HasL2DataRWIO - // Begin a transaction by getting the current block metadata - io.meta.read.valid := state === s_meta_read - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.read.bits.tag := xact.addr_block >> UInt(tagLSB) - - // Write the voluntarily written back data to this cache - pending_writes := (pending_writes & dropPendingBit(io.data.write)) | - addPendingBitWhenBeatHasData(io.inner.release) + val pending_writes = Reg(init=Bits(0, width = innerDataBeats)) val curr_write_beat = PriorityEncoder(pending_writes) - io.data.write.valid := state === s_busy && pending_writes.orR - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact.addr_block(idxMSB,idxLSB) - io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := ~UInt(0, io.data.write.bits.wmask.getWidth) - io.data.write.bits.data := xact.data_buffer(curr_write_beat) - // Send an acknowledgement - io.inner.grant.valid := state === s_busy && pending_ignt && !pending_irel_beats.orR - io.inner.grant.bits := coh.inner.makeGrant(xact) - when(io.inner.grant.fire()) { pending_ignt := Bool(false) } - - // End a transaction by updating the block metadata - io.meta.write.valid := state === s_meta_write - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact.addr_block(idxMSB,idxLSB) - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact.addr_block >> UInt(tagLSB) - io.meta.write.bits.data.coh.inner := xact_old_meta.coh.inner.onRelease(xact) - io.meta.write.bits.data.coh.outer := Mux(xact.hasData(), - xact_old_meta.coh.outer.onHit(M_XWR), - xact_old_meta.coh.outer) - - // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.release.valid && io.alloc.irel) { - xact := io.irel() - pending_irel_beats := Mux(io.irel().hasMultibeatData(), - dropPendingBitWhenBeatHasData(io.inner.release), - UInt(0)) - pending_writes := addPendingBitWhenBeatHasData(io.inner.release) - pending_ignt := io.irel().requiresAck() - state := s_meta_read + def writeDataArray(add_pending_bit: UInt = UInt(0), + block_pending_write: Bool = Bool(false)) { + val port = io.data + pending_writes := (pending_writes & dropPendingBit(port.write)) | add_pending_bit + port.write.valid := state === s_busy && pending_writes.orR && !block_pending_write + port.write.bits := L2DataWriteReq( + id = UInt(trackerId), + way_en = xact_way_en, + addr_idx = xact_addr_idx, + addr_beat = curr_write_beat, + wmask = ~UInt(0, port.write.bits.wmask.getWidth), + data = data_buffer(curr_write_beat)) + + scoreboard += pending_writes.orR } - when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } - when(state === s_meta_resp && io.meta.resp.valid) { - xact_old_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - state := s_busy - } - when(state === s_busy && all_pending_done) { state := s_meta_write } - when(state === s_meta_write && io.meta.write.ready) { state := s_idle } - - // Checks for illegal behavior - assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), - "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") - assert(!(state === s_idle && io.inner.release.fire() && !io.irel().isVoluntary()), - "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } - -class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { - val io = new L2XactTrackerIO - pinAllReadyValidLow(io) - - val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) - val state = Reg(init=s_idle) - - // State holding transaction metadata - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits/8))) - val xact_addr_block = Reg{ io.inner.acquire.bits.addr_block } - val xact_tag_match = Reg{ Bool() } - val xact_way_en = Reg{ Bits(width = nWays) } - val xact_old_meta = Reg{ new L2Metadata } - val pending_coh = Reg{ xact_old_meta.coh } - val xact_allocate = Reg{ Bool() } - val xact_amo_shift_bytes = Reg{ UInt() } - val xact_op_code = Reg{ UInt() } - val xact_addr_byte = Reg{ UInt() } - val xact_op_size = Reg{ UInt() } - val xact_vol_ir_r_type = Reg{ io.irel().r_type } - val xact_vol_ir_src = Reg{ io.irel().client_id } - val xact_vol_ir_client_xact_id = Reg{ io.irel().client_xact_id } - - // Miss queue holds transaction metadata used to make grants - val ignt_q = Module(new Queue( - new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })), - 1 + nSecondaryMisses)) - - // Some accessor wires derived from the the above state - val xact = ignt_q.io.deq.bits - val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) - val xact_addr_tag = xact_addr_block >> UInt(tagLSB) - val xact_vol_irel = Release( - src = xact_vol_ir_src, - voluntary = Bool(true), - r_type = xact_vol_ir_r_type, - client_xact_id = xact_vol_ir_client_xact_id, - addr_block = xact_addr_block) - (p.alterPartial({ case TLId => p(InnerTLId) })) - - // Counters and scoreboard tracking progress made on processing this transaction - val pending_irels = connectTwoWayBeatCounter( - max = io.inner.tlNCachingClients, - up = io.inner.probe, - down = io.inner.release, - trackDown = (r: Release) => !r.isVoluntary())._1 - - val pending_vol_ignt = connectTwoWayBeatCounter( - max = 1, - up = io.inner.release, - down = io.inner.grant, - trackUp = (r: Release) => r.isVoluntary(), - trackDown = (g: Grant) => g.isVoluntary())._1 - - val (pending_ognt, oacq_data_idx, oacq_data_done, ognt_data_idx, ognt_data_done) = - connectTwoWayBeatCounter( - max = 1, - up = io.outer.acquire, - down = io.outer.grant, - beat = xact.addr_beat) - - val (ignt_data_idx, ignt_data_done) = connectOutgoingDataBeatCounter( - out = io.inner.grant, - beat = ignt_q.io.deq.bits.addr_beat) - - val pending_ifins = connectTwoWayBeatCounter( - max = nSecondaryMisses, - up = io.inner.grant, - down = io.inner.finish, - trackUp = (g: Grant) => g.requiresAck())._1 - - val pending_puts = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) - val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_irel_beats = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_writes = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val ignt_data_ready = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_meta_write = Reg(init = Bool(false)) - - // Used to decide when to escape from s_busy - val all_pending_done = - !(pending_reads.orR || - pending_writes.orR || - pending_resps.orR || - pending_puts.orR || - pending_irel_beats.orR || - pending_ognt || - ignt_q.io.count > UInt(0) || - pending_vol_ignt || - //pending_meta_write || // Has own state: s_meta_write - pending_ifins) +trait HasAMOALU extends HasAcquireMetadataBuffer + with HasByteWriteMaskBuffer + with HasRowBeatCounters { + val io: L2XactTrackerIO // Provide a single ALU per tracker to merge Puts and AMOs with data being // refilled, written back, or extant in the cache val amoalu = Module(new AMOALU(rhsIsAligned = true)) - amoalu.io.addr := Cat(xact_addr_block, xact.addr_beat, xact_addr_byte) - amoalu.io.cmd := xact_op_code - amoalu.io.typ := xact_op_size - amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData - amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData - val amo_result = Reg(init = UInt(0, xact.tlDataBits)) + val amo_result = Reg(init = UInt(0, innerDataBits)) + + def initializeAMOALUIOs() { + amoalu.io.addr := Cat(xact_addr_block, xact_addr_beat, xact_addr_byte) + amoalu.io.cmd := xact_op_code + amoalu.io.typ := xact_op_size + amoalu.io.lhs := io.data.resp.bits.data // default, overwritten by calls to mergeData + amoalu.io.rhs := data_buffer.head // default, overwritten by calls to mergeData + } + + // Utility function for applying any buffered stored data to the cache line + // before storing it back into the data array + override def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + val amo_shift_bits = xact_amo_shift_bytes << 3 + amoalu.io.lhs := old_data >> amo_shift_bits + amoalu.io.rhs := new_data >> amo_shift_bits + val wmask = FillInterleaved(8, wmask_buffer(beat)) + data_buffer(beat) := ~wmask & old_data | + wmask & Mux(xact_iacq.isAtomic(), amoalu.io.out << amo_shift_bits, new_data) + when(xact_iacq.isAtomic() && xact_addr_beat === beat) { amo_result := old_data } + } +} + +trait HasCoherenceMetadataBuffer extends HasOuterCacheParameters + with HasBlockAddressBuffer + with HasXactTrackerStates { + def trackerId: Int + + val xact_way_en = Reg{ Bits(width = nWays) } + val xact_old_meta = Reg{ new L2Metadata } + val pending_coh = Reg{ xact_old_meta.coh } + val pending_meta_write = Reg{ Bool() } // pending_meta_write has own state (s_meta_write) + + val inner_coh = pending_coh.inner + val outer_coh = pending_coh.outer + + val xact_addr_idx = xact_addr_block(idxMSB,idxLSB) + val xact_addr_tag = xact_addr_block >> UInt(tagLSB) // Utility function for updating the metadata that will be kept in this cache def updatePendingCohWhen(flag: Bool, next: HierarchicalMetadata) { @@ -716,58 +671,140 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra } } - def addOtherBits(en: Bool, nBits: Int): UInt = - Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits)) + def metaRead(port: HasL2MetaReadIO, next_state: UInt) { + port.read.valid := state === s_meta_read + port.read.bits.id := UInt(trackerId) + port.read.bits.idx := xact_addr_idx + port.read.bits.tag := xact_addr_tag - def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = - addOtherBits(in.fire() && - in.bits.hasMultibeatData() && - in.bits.addr_beat === UInt(0), - in.bits.tlDataBeats) + when(state === s_meta_read && port.read.ready) { state := s_meta_resp } - def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = - ~addPendingBitsOnFirstBeat(in) + when(state === s_meta_resp && port.resp.valid) { + xact_old_meta := port.resp.bits.meta + xact_way_en := port.resp.bits.way_en + state := next_state + } + } + + def metaWrite(port: HasL2MetaWriteIO, to_write: L2Metadata, next_state: UInt) { + port.write.valid := state === s_meta_write + port.write.bits.id := UInt(trackerId) + port.write.bits.idx := xact_addr_idx + port.write.bits.way_en := xact_way_en + port.write.bits.data := to_write + + when(state === s_meta_write && port.write.ready) { state := next_state } + } +} + +trait TriggersWritebacks extends HasCoherenceMetadataBuffer { + def triggerWriteback(wb: L2WritebackIO, next_state: UInt) { + wb.req.valid := state === s_wb_req + wb.req.bits.id := UInt(trackerId) + wb.req.bits.idx := xact_addr_idx + wb.req.bits.tag := xact_old_meta.tag + wb.req.bits.coh := xact_old_meta.coh + wb.req.bits.way_en := xact_way_en + + when(state === s_wb_req && wb.req.ready) { state := s_wb_resp } + when(state === s_wb_resp && wb.resp.valid) { state := s_outer_acquire } + } +} + +class CacheVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) + extends VoluntaryReleaseTracker(trackerId)(p) + with HasDataBuffer + with WritesToOuterCacheDataArray { + val io = new L2XactTrackerIO + pinAllReadyValidLow(io) + + // Avoid metatdata races with writebacks + routeInParent(iacqMatches = inSameSet(_, xact_addr_block)) + io.alloc_iacq.can := Bool(false) + + // Initialize and accept pending Release beats + innerRelease( + block_vol_ignt = pending_writes.orR, + next = s_meta_read) + + io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact + + // Begin a transaction by getting the current block metadata + metaRead(io.meta, s_busy) + + // Write the voluntarily written back data to this cache + writeDataArray(add_pending_bit = addPendingBitWhenBeatHasData(io.inner.release)) + + // End a transaction by updating the block metadata + metaWrite( + io.meta, + L2Metadata( + tag = xact_addr_tag, + inner = xact_old_meta.coh.inner.onRelease(xact_vol_irel), + outer = Mux(xact_vol_irel.hasData(), + xact_old_meta.coh.outer.onHit(M_XWR), + xact_old_meta.coh.outer)), + s_idle) + + when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } + + when(irel_is_allocating) { + pending_writes := addPendingBitWhenBeatHasData(io.inner.release) + } + + quiesce(s_meta_write) + + // Checks for illegal behavior + assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), + "VoluntaryReleaseTracker accepted Release for a block not resident in this cache!") +} + +class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) + extends AcquireTracker(trackerId)(p) + with HasByteWriteMaskBuffer + with HasAMOALU + with TriggersWritebacks + with ReadsFromOuterCacheDataArray + with WritesToOuterCacheDataArray { + val io = new L2XactTrackerIO + pinAllReadyValidLow(io) + initializeAMOALUIOs() + + + val pending_coh_on_ognt = HierarchicalMetadata( + ManagerMetadata.onReset, + pending_coh.outer.onGrant(io.outer.grant.bits, xact_op_code)) + + val pending_coh_on_ignt = HierarchicalMetadata( + pending_coh.inner.onGrant(io.ignt()), + Mux(ognt_counter.down.done, + pending_coh_on_ognt.outer, + pending_coh.outer)) + + val pending_coh_on_irel = HierarchicalMetadata( + pending_coh.inner.onRelease(io.irel()), // Drop sharer + Mux(io.irel().hasData(), // Dirty writeback + pending_coh.outer.onHit(M_XWR), + pending_coh.outer)) - // Defined here because of Chisel default wire demands, used in s_meta_resp val pending_coh_on_hit = HierarchicalMetadata( - io.meta.resp.bits.meta.coh.inner, - io.meta.resp.bits.meta.coh.outer.onHit(xact_op_code)) + io.meta.resp.bits.meta.coh.inner, + io.meta.resp.bits.meta.coh.outer.onHit(xact_op_code)) val pending_coh_on_miss = HierarchicalMetadata.onReset - // Utility function for applying any buffered stored data to the cache line - // before storing it back into the data array - def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { - val old_data = incoming // Refilled, written back, or de-cached data - val new_data = data_buffer(beat) // Newly Put data is already in the buffer - amoalu.io.lhs := old_data >> (xact_amo_shift_bytes << 3) - amoalu.io.rhs := new_data >> (xact_amo_shift_bytes << 3) - val wmask = FillInterleaved(8, wmask_buffer(beat)) - data_buffer(beat) := ~wmask & old_data | - wmask & Mux(xact.isBuiltInType(Acquire.putAtomicType), - amoalu.io.out << (xact_amo_shift_bytes << 3), - new_data) - when(xact.is(Acquire.putAtomicType) && xact.addr_beat === beat) { amo_result := old_data } - } - // TODO: Deal with the possibility that rowBits != tlDataBits - def mergeDataInternal[T <: L2HellaCacheBundle with HasL2Data with HasL2BeatAddr](in: ValidIO[T]) { - when(in.valid) { mergeData(rowBits)(in.bits.addr_beat, in.bits.data) } - } + // Setup IOs used for routing in the parent + val before_wb_alloc = Vec(s_meta_read, s_meta_resp, s_wb_req).contains(state) - def mergeDataInner[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { - when(in.fire() && in.bits.hasData()) { - mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) - } - } + routeInParent( + iacqMatches = inSameSet(_, xact_addr_block), + irelMatches = (irel: HasCacheBlockAddress) => + Mux(before_wb_alloc, inSameSet(irel, xact_addr_block), exactAddrMatch(irel))) + io.alloc_irel.can := Bool(false) - def mergeDataOuter[T <: TLBundle with HasTileLinkData with HasTileLinkBeatId](in: DecoupledIO[T]) { - when(in.fire() && in.bits.hasData()) { - mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) - } - } - - // and Puts-under-Put, and either may also merge witha preceding prefetch + // TileLink allows for Gets-under-Get + // and Puts-under-Put, and either may also merge with a preceding prefetch // that requested the correct permissions (via op_code) def acquiresAreMergeable(sec: AcquireMetadata): Bool = { val allowedTypes = List((Acquire.getType, Acquire.getType), @@ -778,25 +815,15 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra (Acquire.getPrefetchType, Acquire.getType), (Acquire.putPrefetchType, Acquire.putType), (Acquire.putPrefetchType, Acquire.putBlockType)) - allowedTypes.map { case(a, b) => xact.isBuiltInType(a) && sec.isBuiltInType(b) }.reduce(_||_) && + allowedTypes.map { case(a, b) => xact_iacq.isBuiltInType(a) && sec.isBuiltInType(b) }.reduce(_||_) && xact_op_code === sec.op_code() && sec.conflicts(xact_addr_block) && xact_allocate } - // These IOs are used for routing in the parent - val iacq_in_same_set = inSameSet(xact_addr_block, io.iacq().addr_block) - val irel_in_same_set = inSameSet(xact_addr_block, io.irel().addr_block) - val before_wb_alloc = Vec(s_meta_read, s_meta_resp, s_wb_req).contains(state) - io.matches.iacq := (state =/= s_idle) && iacq_in_same_set - io.matches.irel := (state =/= s_idle) && - Mux(before_wb_alloc, irel_in_same_set, io.irel().conflicts(xact_addr_block)) - io.matches.oprb := Bool(false) //TODO - - // Actual transaction processing logic begins here: - // // First, take care of accpeting new acquires or secondary misses - val iacq_can_merge = acquiresAreMergeable(io.iacq()) && + // Handling of primary and secondary misses' data and write mask merging + def iacq_can_merge = acquiresAreMergeable(io.iacq()) && state =/= s_idle && state =/= s_meta_write && !all_pending_done && !io.inner.release.fire() && @@ -804,293 +831,147 @@ class L2AcquireTracker(trackerId: Int)(implicit p: Parameters) extends L2XactTra !io.data.resp.valid && ignt_q.io.enq.ready && ignt_q.io.deq.valid - val iacq_same_xact = xact.client_xact_id === io.iacq().client_xact_id && - xact.hasMultibeatData() && - ignt_q.io.deq.valid && // i.e. state =/= s_idle - pending_puts(io.iacq().addr_beat) - - val iacq_accepted = io.inner.acquire.fire() && - (io.alloc.iacq || iacq_can_merge || iacq_same_xact) + innerAcquire( + can_alloc = Bool(true), + next = s_meta_read) io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact - // Handling of primary and secondary misses' data and write mask merging - when(iacq_accepted && io.iacq().hasData()) { - val beat = io.iacq().addr_beat - val full = FillInterleaved(8, io.iacq().wmask()) - data_buffer(beat) := (~full & data_buffer(beat)) | (full & io.iacq().data) - wmask_buffer(beat) := io.iacq().wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed - } - - // Enqueue some metadata information that we'll use to make coherence updates with later - ignt_q.io.enq.valid := iacq_accepted && io.iacq().first() - ignt_q.io.enq.bits := io.iacq() - - // Track whether any beats are missing from a PutBlock - pending_puts := (pending_puts & - dropPendingBitWhenBeatHasData(io.inner.acquire)) | - addPendingBitsOnFirstBeat(io.inner.acquire) - // Begin a transaction by getting the current block metadata - io.meta.read.valid := state === s_meta_read - io.meta.read.bits.id := UInt(trackerId) - io.meta.read.bits.idx := xact_addr_idx - io.meta.read.bits.tag := xact_addr_tag + // Defined here because of Chisel default wire demands, used in s_meta_resp + val coh = io.meta.resp.bits.meta.coh + val tag_match = io.meta.resp.bits.tag_match + val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact_op_code) + else tag_match && coh.outer.isValid()) + val needs_writeback = !tag_match && + xact_allocate && + (coh.outer.requiresVoluntaryWriteback() || + coh.inner.requiresProbesOnVoluntaryWriteback()) + val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact_iacq) + val should_update_meta = !tag_match && xact_allocate || + is_hit && pending_coh_on_hit =/= coh + def full_representation = io.meta.resp.bits.meta.coh.inner.full() + + metaRead( + io.meta, + Mux(needs_writeback, s_wb_req, + Mux(needs_inner_probes, s_inner_probe, + Mux(!is_hit, s_outer_acquire, s_busy)))) + + updatePendingCohWhen( + io.meta.resp.valid, + Mux(is_hit, pending_coh_on_hit, + Mux(tag_match, coh, pending_coh_on_miss))) // Issue a request to the writeback unit - io.wb.req.valid := state === s_wb_req - io.wb.req.bits.id := UInt(trackerId) - io.wb.req.bits.idx := xact_addr_idx - io.wb.req.bits.tag := xact_old_meta.tag - io.wb.req.bits.coh := xact_old_meta.coh - io.wb.req.bits.way_en := xact_way_en + triggerWriteback(io.wb, s_outer_acquire) // Track which clients yet need to be probed and make Probe message - pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) - val curr_probe_dst = PriorityEncoder(pending_iprbs) - io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits := pending_coh.inner.makeProbe(curr_probe_dst, xact, xact_addr_block) + // If we're probing, we know the tag matches, so if this is the + // last level cache, we can use the data without upgrading permissions + val skip_outer_acquire = + (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact_op_code) + else xact_old_meta.coh.outer.isValid()) + + innerProbe( + inner_coh.makeProbe(curr_probe_dst, xact_iacq, xact_addr_block), + Mux(!skip_outer_acquire, s_outer_acquire, s_busy)) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data, and may be unexpected voluntary releases - val irel_can_merge = io.irel().conflicts(xact_addr_block) && + + innerRelease() // Don't block on pending_writes because they won't happen until s_busy + + def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && !Vec(s_idle, s_meta_read, s_meta_resp, s_meta_write).contains(state) && !all_pending_done && !io.outer.grant.fire() && !io.inner.grant.fire() && - !pending_vol_ignt - - val irel_same_xact = io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() && - state === s_inner_probe - - val irel_accepted = io.inner.release.fire() && - (io.alloc.irel || irel_can_merge || irel_same_xact) + !vol_ignt_counter.pending io.inner.release.ready := irel_can_merge || irel_same_xact - val pending_coh_on_irel = HierarchicalMetadata( - pending_coh.inner.onRelease(io.irel()), // Drop sharer - Mux(io.irel().hasData(), // Dirty writeback - pending_coh.outer.onHit(M_XWR), - pending_coh.outer)) + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + mergeDataInner(io.inner.release) - when(io.inner.release.fire() && irel_can_merge) { - xact_vol_ir_r_type := io.irel().r_type - xact_vol_ir_src := io.irel().client_id - xact_vol_ir_client_xact_id := io.irel().client_xact_id - pending_irel_beats := Mux(io.irel().hasMultibeatData(), - dropPendingBitWhenBeatHasData(io.inner.release), - UInt(0)) - } - pending_irel_beats := (pending_irel_beats & dropPendingBitWhenBeatHasData(io.inner.release)) - - // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: - // - // If we're allocating in this cache, we can use the current metadata - // to make an appropriate custom Acquire, otherwise we copy over the - // built-in Acquire from the inner TL to the outer TL - io.outer.acquire.valid := state === s_outer_acquire && - (xact_allocate || !pending_puts(oacq_data_idx)) - io.outer.acquire.bits := Mux(xact_allocate, - xact_old_meta.coh.outer.makeAcquire( - op_code = xact_op_code, - client_xact_id = UInt(0), - addr_block = xact_addr_block), - BuiltInAcquireBuilder( - a_type = xact.a_type, - client_xact_id = UInt(0), - addr_block = xact_addr_block, - addr_beat = oacq_data_idx, - data = data_buffer(oacq_data_idx), - addr_byte = xact_addr_byte, - operand_size = xact_op_size, - opcode = xact_op_code, - wmask = wmask_buffer(oacq_data_idx), - alloc = Bool(false)) - (p.alterPartial({ case TLId => p(OuterTLId)}))) + // Send outer request + outerAcquire( + caching = xact_allocate, + coh = xact_old_meta.coh.outer, // TODO outer_coh? + data = data_buffer(ognt_counter.up.idx), + wmask = wmask_buffer(ognt_counter.up.idx), + next = s_busy) + // Handle the response from outer memory - io.outer.grant.ready := state === s_busy - val pending_coh_on_ognt = HierarchicalMetadata( - ManagerMetadata.onReset, - pending_coh.outer.onGrant(io.outer.grant.bits, xact_op_code)) - updatePendingCohWhen(ognt_data_done, pending_coh_on_ognt) + updatePendingCohWhen(ognt_counter.down.done, pending_coh_on_ognt) mergeDataOuter(io.outer.grant) + // Send read request and get resp // Going back to the original inner transaction: // We read from the the cache at this level if data wasn't written back or refilled. // We may still merge further Gets, requiring further beats to be read. // If ECC requires a full writemask, we'll read out data on partial writes as well. - pending_reads := (pending_reads & - dropPendingBit(io.data.read) & - dropPendingBitWhenBeatHasData(io.inner.release) & - dropPendingBitWhenBeatHasData(io.outer.grant)) | - (~ignt_data_ready & ( - addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | - addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire))) - val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && pending_reads.orR && !pending_ognt - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact_way_en - io.data.read.bits.addr_idx := xact_addr_idx - io.data.read.bits.addr_beat := curr_read_beat - - pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | - addPendingBitInternal(io.data.read) - mergeDataInternal(io.data.resp) + readDataArray( + drop_pending_bit = (dropPendingBitWhenBeatHasData(io.inner.release) & + dropPendingBitWhenBeatHasData(io.outer.grant)), + add_pending_bit = addPendingBitWhenBeatNeedsRead(io.inner.acquire, Bool(alwaysWriteFullBeat)), + block_pending_read = ognt_counter.pending) + // Do write // We write data to the cache at this level if it was Put here with allocate flag, // written back dirty, or refilled from outer memory. - pending_writes := (pending_writes & - dropPendingBit(io.data.write) & - dropPendingBitsOnFirstBeat(io.inner.acquire)) | - addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | - addPendingBitWhenBeatHasData(io.inner.release) | - addPendingBitWhenBeatHasData(io.outer.grant, xact_allocate) - val curr_write_beat = PriorityEncoder(pending_writes) - io.data.write.valid := state === s_busy && - pending_writes.orR && - !pending_puts.orR && - !pending_ognt && - !pending_reads(curr_write_beat) && - !pending_resps(curr_write_beat) - io.data.write.bits.id := UInt(trackerId) - io.data.write.bits.way_en := xact_way_en - io.data.write.bits.addr_idx := xact_addr_idx - io.data.write.bits.addr_beat := curr_write_beat - io.data.write.bits.wmask := ~UInt(0, io.data.write.bits.wmask.getWidth) - io.data.write.bits.data := data_buffer(curr_write_beat) + writeDataArray( + add_pending_bit = (addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant, xact_allocate)), + block_pending_write = (ognt_counter.pending || + pending_put_data.orR || + pending_reads(curr_write_beat) || + pending_resps(curr_write_beat))) - // soon as the data is released, granted, put, or read from the cache - ignt_data_ready := ignt_data_ready | - addPendingBitWhenBeatHasData(io.inner.release) | - addPendingBitWhenBeatHasData(io.outer.grant) | - addPendingBitInternal(io.data.resp) - // We can issue a grant for a pending write once all data is - // received and committed to the data array or outer memory - val ignt_ack_ready = !(state === s_idle || - state === s_meta_read || - pending_puts.orR || - pending_writes.orR || - pending_ognt) + // Acknowledge or respond with data + innerGrant( + data = Mux(xact_iacq.isAtomic(), amo_result, data_buffer(ignt_data_idx)), + external_pending = pending_writes.orR || ognt_counter.pending, + add = addPendingBitInternal(io.data.resp)) - ignt_q.io.deq.ready := !pending_vol_ignt && ignt_data_done - io.inner.grant.valid := pending_vol_ignt || - (state === s_busy && - ignt_q.io.deq.valid && - Mux(io.ignt().hasData(), - ignt_data_ready(ignt_data_idx), - ignt_ack_ready)) - // Make the Grant message using the data stored in the secondary miss queue - val grant_from_acquire = pending_coh.inner.makeGrant( - sec = ignt_q.io.deq.bits, - manager_xact_id = UInt(trackerId), - data = Mux(xact.is(Acquire.putAtomicType), - amo_result, - data_buffer(ignt_data_idx))) - val grant_from_release = pending_coh.inner.makeGrant(xact_vol_irel) - io.inner.grant.bits := Mux(pending_vol_ignt, grant_from_release, grant_from_acquire) - io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter - - val pending_coh_on_ignt = HierarchicalMetadata( - pending_coh.inner.onGrant(io.ignt()), - Mux(ognt_data_done, - pending_coh_on_ognt.outer, - pending_coh.outer)) updatePendingCohWhen(io.inner.grant.fire() && io.ignt().last(), pending_coh_on_ignt) - // We must wait for as many Finishes as we sent Grants - io.inner.finish.ready := state === s_busy - // End a transaction by updating the block metadata - io.meta.write.valid := state === s_meta_write - io.meta.write.bits.id := UInt(trackerId) - io.meta.write.bits.idx := xact_addr_idx - io.meta.write.bits.way_en := xact_way_en - io.meta.write.bits.data.tag := xact_addr_tag - io.meta.write.bits.data.coh := pending_coh - - // State machine updates and transaction handler metadata intialization - when(state === s_idle && io.inner.acquire.valid && io.alloc.iacq) { - xact_addr_block := io.iacq().addr_block - xact_allocate := io.iacq().allocate() - xact_amo_shift_bytes := io.iacq().amo_shift_bytes() - xact_op_code := io.iacq().op_code() - xact_addr_byte := io.iacq().addr_byte() - xact_op_size := io.iacq().op_size() - amo_result := UInt(0) - pending_puts := Mux( // Make sure to collect all data from a PutBlock - io.iacq().isBuiltInType(Acquire.putBlockType), - dropPendingBitWhenBeatHasData(io.inner.acquire), - UInt(0)) - pending_reads := Mux( // GetBlocks and custom types read all beats - io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), - SInt(-1), - (addPendingBitWhenBeatIsGetOrAtomic(io.inner.acquire) | - addPendingBitWhenBeatHasPartialWritemask(io.inner.acquire)).toSInt).toUInt - pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) - pending_resps := UInt(0) - ignt_data_ready := UInt(0) - pending_meta_write := Bool(false) - state := s_meta_read - } - when(state === s_meta_read && io.meta.read.ready) { state := s_meta_resp } + metaWrite(io.meta, L2Metadata(xact_addr_tag, pending_coh), s_idle) + + // Initialization of some scoreboard logic based on the original + // Acquire message on on the results of the metadata read: when(state === s_meta_resp && io.meta.resp.valid) { - xact_tag_match := io.meta.resp.bits.tag_match - xact_old_meta := io.meta.resp.bits.meta - xact_way_en := io.meta.resp.bits.way_en - val coh = io.meta.resp.bits.meta.coh - val tag_match = io.meta.resp.bits.tag_match - val is_hit = (if(!isLastLevelCache) tag_match && coh.outer.isHit(xact_op_code) - else tag_match && coh.outer.isValid()) - val needs_writeback = !tag_match && - xact_allocate && - (coh.outer.requiresVoluntaryWriteback() || - coh.inner.requiresProbesOnVoluntaryWriteback()) - val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact) - val should_update_meta = !tag_match && xact_allocate || - is_hit && pending_coh_on_hit =/= coh - // Determine any changes to the coherence metadata - when (should_update_meta) { pending_meta_write := Bool(true) } - pending_coh := Mux(is_hit, pending_coh_on_hit, Mux(tag_match, coh, pending_coh_on_miss)) - // If we need to probe some clients, make a bitmask identifying them - when (needs_inner_probes) { - val full_sharers = coh.inner.full() - val mask_self = Mux( - xact.requiresSelfProbe(), - coh.inner.full() | UIntToOH(xact.client_id), - coh.inner.full() & ~UIntToOH(xact.client_id)) - val mask_incoherent = mask_self & ~io.incoherent.toBits - pending_iprbs := mask_incoherent - } // If some kind of Put is marked no-allocate but is already in the cache, // we need to write its data to the data array - when (is_hit && !xact_allocate && xact.hasData()) { - pending_writes := addPendingBitsFromAcquire(xact) + when(is_hit && !xact_allocate && xact_iacq.hasData()) { + pending_writes := addPendingBitsFromAcquire(xact_iacq) xact_allocate := Bool(true) } - // Next: request writeback, issue probes, query outer memory, or respond - state := Mux(needs_writeback, s_wb_req, - Mux(needs_inner_probes, s_inner_probe, - Mux(!is_hit, s_outer_acquire, s_busy))) + when (needs_inner_probes) { initializeProbes() } + pending_meta_write := should_update_meta //TODO what edge case was this covering? } - when(state === s_wb_req && io.wb.req.ready) { state := s_wb_resp } - when(state === s_wb_resp && io.wb.resp.valid) { state := s_outer_acquire } - when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels)) { - // Tag matches, so if this is the last level cache we can use the data without upgrading permissions - val skip_outer_acquire = - (if(!isLastLevelCache) xact_old_meta.coh.outer.isHit(xact_op_code) - else xact_old_meta.coh.outer.isValid()) - state := Mux(!skip_outer_acquire, s_outer_acquire, s_busy) - } - when(state === s_outer_acquire && oacq_data_done) { state := s_busy } - when(state === s_busy && all_pending_done) { + + // Initialize more transaction metadata. Pla + when(iacq_is_allocating) { wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle - state := Mux(pending_meta_write, s_meta_write, s_idle) + amo_result := UInt(0) + pending_meta_write := Bool(false) + pending_reads := Mux( // Pick out the specific beats of data that need to be read + io.iacq().isBuiltInType(Acquire.getBlockType) || !io.iacq().isBuiltInType(), + ~UInt(0, width = innerDataBeats), + addPendingBitWhenBeatNeedsRead(io.inner.acquire, Bool(alwaysWriteFullBeat))) + pending_writes := addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) + pending_resps := UInt(0) } - when(state === s_meta_write && io.meta.write.ready) { state := s_idle } + + initDataInner(io.inner.acquire) + + // Wait for everything to quiesce + quiesce(Mux(pending_meta_write, s_meta_write, s_idle)) } class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { @@ -1105,161 +986,89 @@ class L2WritebackIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) { val resp = Valid(new L2WritebackResp).flip } -class L2WritebackUnitIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) { - val wb = new L2WritebackIO().flip - val data = new L2DataRWIO +trait HasL2WritebackIO extends HasOuterCacheParameters { + val wb = new L2WritebackIO() } -class L2WritebackUnit(trackerId: Int)(implicit p: Parameters) extends L2XactTracker()(p) { +class L2WritebackUnitIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) + with HasL2DataRWIO { + val wb = new L2WritebackIO().flip() +} + +class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p) + with AcceptsVoluntaryReleases + with EmitsVoluntaryReleases + with EmitsInnerProbes + with ReadsFromOuterCacheDataArray + with RoutesInParent { val io = new L2WritebackUnitIO pinAllReadyValidLow(io) - val s_idle :: s_inner_probe :: s_busy :: s_outer_grant :: s_wb_resp :: Nil = Enum(UInt(), 5) - val state = Reg(init=s_idle) + val xact_id = Reg{ io.wb.req.bits.id } - val xact = Reg(new L2WritebackReq) - val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) - val xact_vol_ir_r_type = Reg{ io.irel().r_type } - val xact_vol_ir_src = Reg{ io.irel().client_id } - val xact_vol_ir_client_xact_id = Reg{ io.irel().client_xact_id } + val pending_coh_on_irel = HierarchicalMetadata( + inner_coh.onRelease(io.irel()), // Drop sharer + Mux(io.irel().hasData(), // Dirty writeback + outer_coh.onHit(M_XWR), + outer_coh)) - val xact_addr_block = if (cacheIdBits == 0) - Cat(xact.tag, xact.idx) - else - Cat(xact.tag, xact.idx, UInt(cacheId, cacheIdBits)) - val xact_vol_irel = Release( - src = xact_vol_ir_src, - voluntary = Bool(true), - r_type = xact_vol_ir_r_type, - client_xact_id = xact_vol_ir_client_xact_id, - addr_block = xact_addr_block) - - val pending_irels = connectTwoWayBeatCounter( - max = io.inner.tlNCachingClients, - up = io.inner.probe, - down = io.inner.release, - trackDown = (r: Release) => !r.isVoluntary())._1 - - val pending_vol_ignt = connectTwoWayBeatCounter( - max = 1, - up = io.inner.release, - down = io.inner.grant, - trackUp = (r: Release) => r.isVoluntary(), - trackDown = (g: Grant) => g.isVoluntary())._1 - - val (pending_ognt, orel_data_idx, orel_data_done, ognt_data_idx, ognt_data_done) = - connectTwoWayBeatCounter( - max = 1, - up = io.outer.release, - down = io.outer.grant) - - val pending_iprbs = Reg(init = Bits(0, width = io.inner.tlNCachingClients)) - val pending_reads = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_resps = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - val pending_orel_data = Reg(init=Bits(0, width = io.inner.tlDataBeats)) - - // These IOs are used for routing in the parent - io.matches.iacq := (state =/= s_idle) && io.iacq().conflicts(xact_addr_block) - io.matches.irel := (state =/= s_idle) && io.irel().conflicts(xact_addr_block) - io.matches.oprb := (state =/= s_idle) && io.oprb().conflicts(xact_addr_block) + routeInParent() // Start the writeback sub-transaction io.wb.req.ready := state === s_idle // Track which clients yet need to be probed and make Probe message - pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) - val curr_probe_dst = PriorityEncoder(pending_iprbs) - io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits := xact.coh.inner.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block) + innerProbe( + inner_coh.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block), + s_busy) // Handle incoming releases from clients, which may reduce sharer counts // and/or write back dirty data - val irel_can_merge = io.irel().conflicts(xact_addr_block) && + innerRelease() + + def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && - state === s_inner_probe && - !pending_vol_ignt - - val irel_same_xact = io.irel().conflicts(xact_addr_block) && - !io.irel().isVoluntary() && - state === s_inner_probe - - val irel_accepted = io.inner.release.fire() && - (io.alloc.irel || irel_can_merge || irel_same_xact) + (state =/= s_idle) && + !(state === s_busy && all_pending_done) && + !vol_ignt_counter.pending io.inner.release.ready := irel_can_merge || irel_same_xact - val pending_coh_on_irel = HierarchicalMetadata( - xact.coh.inner.onRelease(io.irel()), // Drop sharer - Mux(io.irel().hasData(), // Dirty writeback - xact.coh.outer.onHit(M_XWR), - xact.coh.outer)) - when(io.inner.release.fire()) { - xact.coh := pending_coh_on_irel - when(io.irel().hasData()) { data_buffer(io.irel().addr_beat) := io.irel().data } - when(irel_can_merge) { - xact_vol_ir_r_type := io.irel().r_type - xact_vol_ir_src := io.irel().client_id - xact_vol_ir_client_xact_id := io.irel().client_xact_id - } - } + + updatePendingCohWhen(io.inner.release.fire(), pending_coh_on_irel) + + mergeDataInner(io.inner.release) // If a release didn't write back data, have to read it from data array - pending_reads := (pending_reads & - dropPendingBit(io.data.read) & - dropPendingBitWhenBeatHasData(io.inner.release)) - val curr_read_beat = PriorityEncoder(pending_reads) - io.data.read.valid := state === s_busy && pending_reads.orR - io.data.read.bits.id := UInt(trackerId) - io.data.read.bits.way_en := xact.way_en - io.data.read.bits.addr_idx := xact.idx - io.data.read.bits.addr_beat := curr_read_beat - io.data.write.valid := Bool(false) - - pending_resps := (pending_resps & dropPendingBitInternal(io.data.resp)) | - addPendingBitInternal(io.data.read) - when(io.data.resp.valid) { - data_buffer(io.data.resp.bits.addr_beat) := io.data.resp.bits.data - } + readDataArray(drop_pending_bit = dropPendingBitWhenBeatHasData(io.inner.release)) // Once the data is buffered we can write it back to outer memory - pending_orel_data := pending_orel_data | - addPendingBitWhenBeatHasData(io.inner.release) | - addPendingBitInternal(io.data.resp) - io.outer.release.valid := state === s_busy && - (!io.orel().hasData() || pending_orel_data(orel_data_idx)) - io.outer.release.bits := xact.coh.outer.makeVoluntaryWriteback( - client_xact_id = UInt(trackerId), - addr_block = xact_addr_block, - addr_beat = orel_data_idx, - data = data_buffer(orel_data_idx)) - - // Ack a voluntary release if we got one - io.inner.grant.valid := pending_vol_ignt - io.inner.grant.bits := xact.coh.inner.makeGrant(xact_vol_irel) - - // Wait for an acknowledgement - io.outer.grant.ready := state === s_outer_grant + outerRelease( + coh = outer_coh, + data = data_buffer(vol_ognt_counter.up.idx), + add_pending_bit = addPendingBitInternal(io.data.resp)) // Respond to the initiating transaction handler signalling completion of the writeback - io.wb.resp.valid := state === s_wb_resp - io.wb.resp.bits.id := xact.id + io.wb.resp.valid := state === s_busy && all_pending_done + io.wb.resp.bits.id := xact_id + quiesce() + + def full_representation = io.wb.req.bits.coh.inner.full() // State machine updates and transaction handler metadata intialization when(state === s_idle && io.wb.req.valid) { - xact := io.wb.req.bits + xact_id := io.wb.req.bits.id + xact_way_en := io.wb.req.bits.way_en + xact_addr_block := (if (cacheIdBits == 0) Cat(io.wb.req.bits.tag, io.wb.req.bits.idx) + else Cat(io.wb.req.bits.tag, io.wb.req.bits.idx, UInt(cacheId, cacheIdBits))) val coh = io.wb.req.bits.coh val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() - when(needs_inner_probes) { pending_iprbs := coh.inner.full() & ~io.incoherent.toBits } - pending_reads := ~UInt(0, width = innerDataBeats) + val needs_outer_release = coh.outer.requiresVoluntaryWriteback() + when(needs_inner_probes) { initializeProbes() } + pending_reads := Mux(needs_outer_release, ~UInt(0, width = innerDataBeats), UInt(0)) pending_resps := UInt(0) - pending_orel_data := UInt(0) + pending_orel := needs_outer_release + //pending_orel_data := UInt(0) + pending_coh := coh state := Mux(needs_inner_probes, s_inner_probe, s_busy) } - when(state === s_inner_probe && !(pending_iprbs.orR || pending_irels || pending_vol_ignt)) { - state := Mux(xact.coh.outer.requiresVoluntaryWriteback(), s_busy, s_wb_resp) - } - when(state === s_busy && orel_data_done) { - state := Mux(io.orel().requiresAck(), s_outer_grant, s_wb_resp) - } - when(state === s_outer_grant && ognt_data_done) { state := s_wb_resp } - when(state === s_wb_resp ) { state := s_idle } } diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 569f4278..a76dc287 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -227,6 +227,17 @@ object ManagerTileLinkHeaderCreator { } } +class BeatCounterStatus extends Bundle { + val idx = UInt() + val done = Bool() +} + +class TwoWayBeatCounterStatus extends Bundle { + val pending = Bool() + val up = new BeatCounterStatus() + val down = new BeatCounterStatus() +} + /** Utility trait containing wiring functions to keep track of how many data beats have * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. * @@ -283,27 +294,31 @@ trait HasDataBeatCounters { /** Provides counters on two channels, as well a meta-counter that tracks how many * messages have been sent over the up channel but not yet responded to over the down channel * - * @param max max number of outstanding ups with no down + * @param status bundle of status of the counters * @param up outgoing channel * @param down incoming channel + * @param max max number of outstanding ups with no down * @param beat overrides cnts on single-beat messages * @param track whether up's message should be tracked * @return a tuple containing whether their are outstanding messages, up's count, * up's done, down's count, down's done */ - def connectTwoWayBeatCounter[T <: TileLinkChannel, S <: TileLinkChannel]( - max: Int, + def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel]( + status: TwoWayBeatCounterStatus, up: DecoupledIO[T], down: DecoupledIO[S], + max: Int = 1, beat: UInt = UInt(0), trackUp: T => Bool = (t: T) => Bool(true), - trackDown: S => Bool = (s: S) => Bool(true)): (Bool, UInt, Bool, UInt, Bool) = { - val (up_idx, up_done) = connectDataBeatCounter(up.fire(), up.bits, beat) - val (down_idx, down_done) = connectDataBeatCounter(down.fire(), down.bits, beat) - val do_inc = up_done && trackUp(up.bits) - val do_dec = down_done && trackDown(down.bits) - val cnt = TwoWayCounter(do_inc, do_dec, max) - (cnt > UInt(0), up_idx, up_done, down_idx, down_done) + trackDown: S => Bool = (s: S) => Bool(true)) { + val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat) + val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat) + val cnt = TwoWayCounter(up_done, dn_done, max) + status.pending := cnt > UInt(0) + status.up.idx := up_idx + status.up.done := up_done + status.down.idx := dn_idx + status.down.done := dn_done } } @@ -391,9 +406,10 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { addr_beat = ognt.addr_beat, data = ognt.data) + assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.") val rel_grant = Grant( is_builtin_type = Bool(true), - g_type = Mux(gnt_voluntary, Grant.voluntaryAckType, ognt.g_type), + g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, addr_beat = ognt.addr_beat, diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/metadata.scala index ede9e397..38c51428 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/metadata.scala @@ -47,13 +47,14 @@ class ClientMetadata(implicit p: Parameters) extends CoherenceMetadata()(p) { def makeAcquire( op_code: UInt, client_xact_id: UInt, - addr_block: UInt): Acquire = + addr_block: UInt): Acquire = { Acquire( is_builtin_type = Bool(false), a_type = co.getAcquireType(op_code, this), client_xact_id = client_xact_id, addr_block = addr_block, union = Cat(op_code, Bool(true)))(p) + } /** Constructs a Release message based on this metadata on cache control op * diff --git a/uncore/src/main/scala/sdq.scala b/uncore/src/main/scala/sdq.scala new file mode 100644 index 00000000..00790d56 --- /dev/null +++ b/uncore/src/main/scala/sdq.scala @@ -0,0 +1,118 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import cde.{Parameters, Field} + +case object L2StoreDataQueueDepth extends Field[Int] + +trait HasStoreDataQueueParameters extends HasCoherenceAgentParameters { + val sdqDepth = p(L2StoreDataQueueDepth)*innerDataBeats + val dqIdxBits = math.max(log2Up(nReleaseTransactors) + 1, log2Up(sdqDepth)) + val nDataQueueLocations = 3 //Stores, VoluntaryWBs, Releases +} + +class DataQueueLocation(implicit p: Parameters) extends CoherenceAgentBundle()(p) + with HasStoreDataQueueParameters { + val idx = UInt(width = dqIdxBits) + val loc = UInt(width = log2Ceil(nDataQueueLocations)) +} + +object DataQueueLocation { + def apply(idx: UInt, loc: UInt)(implicit p: Parameters) = { + val d = Wire(new DataQueueLocation) + d.idx := idx + d.loc := loc + d + } +} + +trait HasStoreDataQueue extends HasStoreDataQueueParameters { + val io: HierarchicalTLIO + val trackerIOsList: Seq[HierarchicalXactTrackerIO] + + val internalDataBits = new DataQueueLocation().getWidth + val inStoreQueue :: inVolWBQueue :: inClientReleaseQueue :: Nil = Enum(UInt(), nDataQueueLocations) + + val usingStoreDataQueue = p.alterPartial({ + case TLKey(`innerTLId`) => innerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) + case TLKey(`outerTLId`) => outerTLParams.copy(overrideDataBitsPerBeat = Some(internalDataBits)) + }) + + // Queue to store impending Put data + lazy val sdq = Reg(Vec(sdqDepth, io.iacq().data)) + lazy val sdq_val = Reg(init=Bits(0, sdqDepth)) + lazy val sdq_alloc_id = PriorityEncoder(~sdq_val) + lazy val sdq_rdy = !sdq_val.andR + lazy val sdq_enq = trackerIOsList.map( t => + (t.alloc_iacq.should || t.alloc_iacq.matches) && + t.inner.acquire.fire() && + t.iacq().hasData() + ).reduce(_||_) + + lazy val sdqLoc = List.fill(nTransactors) { + DataQueueLocation(sdq_alloc_id, inStoreQueue).toBits + } + + /* + doInputRoutingWithAllocation( + in = io.inner.acquire, + outs = trackerList.map(_.io.inner.acquire), + allocs = trackerList.map(_.io.alloc._iacq), + dataOverride = Some(sdqLoc), + allocOverride = Some(sdq_rdy && !irel_vs_iacq_conflict)) + */ + + // Queue to store impending Voluntary Release data + lazy val voluntary = io.irel().isVoluntary() + lazy val vwbdq_enq = io.inner.release.fire() && voluntary && io.irel().hasData() + lazy val (rel_data_cnt, rel_data_done) = Counter(vwbdq_enq, innerDataBeats) //TODO Zero width + lazy val vwbdq = Reg(Vec(innerDataBeats, io.irel().data)) //TODO Assumes nReleaseTransactors == 1 + + + lazy val vwbqLoc = (0 until nTransactors).map(i => + (DataQueueLocation(rel_data_cnt, + (if(i < nReleaseTransactors) inVolWBQueue + else inClientReleaseQueue)).toBits)) + /* + doInputRoutingWithAllocation( + io.inner.release, + trackerList.map(_.io.inner.release), + trackerList.map(_.io.matches.irel), + trackerList.map(_.io.alloc.irel), + Some(vwbqLoc)) + */ + + val outer_arb: ClientTileLinkIOArbiter + lazy val outer_data_ptr = new DataQueueLocation().fromBits(outer_arb.io.out.acquire.bits.data) + /* + val outer_arb = Module(new ClientTileLinkIOArbiter(trackerList.size) + (usingStoreDataQueue.alterPartial({ case TLId => p(OuterTLId) }))) + outer_arb.io.in <> trackerList + */ + // Get the pending data out of the store data queue + lazy val is_in_sdq = outer_data_ptr.loc === inStoreQueue + lazy val free_sdq = io.outer.acquire.fire() && + io.outer.acquire.bits.hasData() && + outer_data_ptr.loc === inStoreQueue + /* + io.outer <> outer_arb.io.out + io.outer.acquire.bits.data := MuxLookup(outer_data_ptr.loc, io.irel().data, Array( + inStoreQueue -> sdq(outer_data_ptr.idx), + inVolWBQueue -> vwbdq(outer_data_ptr.idx))) + */ + + // Enqueue SDQ data + def sdqEnqueue() { + when (sdq_enq) { sdq(sdq_alloc_id) := io.iacq().data } + when(vwbdq_enq) { vwbdq(rel_data_cnt) := io.irel().data } + } + + // Update SDQ valid bits + def sdqUpdate() { + when (io.outer.acquire.valid || sdq_enq) { + sdq_val := sdq_val & ~(UIntToOH(outer_data_ptr.idx) & Fill(sdqDepth, free_sdq)) | + PriorityEncoderOH(~sdq_val(sdqDepth-1,0)) & Fill(sdqDepth, sdq_enq) + } + } +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink.scala index 95e1ebf2..5599217b 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink.scala @@ -132,8 +132,8 @@ trait HasTileLinkData extends HasTileLinkBeatId { def hasData(dummy: Int = 0): Bool def hasMultibeatData(dummy: Int = 0): Bool - def first(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(0), Bool(true)) - def last(dummy: Int = 0): Bool = Mux(hasMultibeatData(), addr_beat === UInt(tlDataBeats-1), Bool(true)) + def first(dummy: Int = 0): Bool = !hasMultibeatData() || addr_beat === UInt(0) + def last(dummy: Int = 0): Bool = !hasMultibeatData() || addr_beat === UInt(tlDataBeats-1) } /** An entire cache block of data */ @@ -186,6 +186,10 @@ trait HasAcquireUnion extends HasTileLinkParameters { } /** Full, beat-sized writemask */ def full_wmask(dummy: Int = 0) = FillInterleaved(8, wmask()) + + /** Is this message a built-in read message */ + def hasPartialWritemask(dummy: Int = 0): Bool = wmask() =/= Acquire.fullWriteMask + } trait HasAcquireType extends HasTileLinkParameters { @@ -207,6 +211,12 @@ trait HasAcquireType extends HasTileLinkParameters { def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && (is(Acquire.getPrefetchType) || is(Acquire.putPrefetchType)) + /** Is this message a built-in atomic message */ + def isAtomic(dummy: Int = 0): Bool = isBuiltInType() && is(Acquire.putAtomicType) + + /** Is this message a built-in read message */ + def isGet(dummy: Int = 0): Bool = isBuiltInType() && (is(Acquire.getType) || is(Acquire.getBlockType)) + /** Does this message contain data? Assumes that no custom message types have data. */ def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) @@ -214,11 +224,6 @@ trait HasAcquireType extends HasTileLinkParameters { def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && Acquire.typesWithMultibeatData.contains(a_type) - /** Does this message require the manager to probe the client the very client that sent it? - * Needed if multiple caches are attached to the same port. - */ - def requiresSelfProbe(dummy: Int = 0) = Bool(false) - /** Mapping between each built-in Acquire type and a built-in Grant type. */ def getBuiltInGrantType(dummy: Int = 0): UInt = Acquire.getBuiltInGrantType(this.a_type) } diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala new file mode 100644 index 00000000..87d7b0b9 --- /dev/null +++ b/uncore/src/main/scala/trackers.scala @@ -0,0 +1,546 @@ +// See LICENSE for license details. + +package uncore +import Chisel._ +import cde.{Parameters, Field} + +class TrackerAllocation extends Bundle { + val matches = Bool(OUTPUT) + val can = Bool(OUTPUT) + val should = Bool(INPUT) +} + +trait HasTrackerAllocationIO extends Bundle { + val alloc_iacq = new TrackerAllocation + val alloc_irel = new TrackerAllocation + val alloc_oprb = new TrackerAllocation +} + +class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) + with HasTrackerAllocationIO + +class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p) + with HasTrackerAllocationIO + +abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p) + with HasXactTrackerStates + with HasPendingBitHelpers { + override val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) + val state = Reg(init=s_idle) + + def quiesce(next: UInt = s_idle) { + all_pending_done := !scoreboard.foldLeft(Bool(false))(_||_) + when(state === s_busy && all_pending_done) { state := next } + } + + def pinAllReadyValidLow[T <: Data](b: Bundle) { + b.elements.foreach { + _._2 match { + case d: DecoupledIO[_] => + if(d.ready.dir == OUTPUT) d.ready := Bool(false) + else if(d.valid.dir == OUTPUT) d.valid := Bool(false) + case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) + case b: Bundle => pinAllReadyValidLow(b) + case _ => + } + } + } +} + +trait HasXactTrackerStates { + def state: UInt + def s_idle: UInt = UInt(0) + def s_meta_read: UInt + def s_meta_resp: UInt + def s_wb_req: UInt + def s_wb_resp: UInt + def s_inner_probe: UInt + def s_outer_acquire: UInt + def s_busy: UInt + def s_meta_write: UInt +} + +trait HasPendingBitHelpers extends HasDataBeatCounters { + val scoreboard = scala.collection.mutable.ListBuffer.empty[Bool] + val all_pending_done = Wire(Bool()) + + def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = + Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) + + def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = + ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) + + def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt = + Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id) + + def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt = + ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id) + + def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) + + def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = + addPendingBitWhenBeatHasData(in, in.bits.allocate()) + + def addPendingBitWhenBeatNeedsRead(in: DecoupledIO[AcquireFromSrc], inc: Bool = Bool(true)): UInt = { + val a = in.bits + val needs_read = (a.isGet() || a.isAtomic() || a.hasPartialWritemask()) || inc + addPendingBitWhenBeat(in.fire() && needs_read, a) + } + + def addPendingBitWhenBeatHasPartialWritemask(in: DecoupledIO[AcquireFromSrc]): UInt = + addPendingBitWhenBeat(in.fire() && in.bits.hasPartialWritemask(), in.bits) + + def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt = + Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat)) + + def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = + dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) + + def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt = + dropPendingBitWhenId(in.fire(), in.bits) + + def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = + dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) + + def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt = + addPendingBitWhenId(in.fire(), in.bits) + + def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = + addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) + + def addOtherBits(en: Bool, nBits: Int): UInt = + Mux(en, Cat(Fill(nBits - 1, UInt(1, 1)), UInt(0, 1)), UInt(0, nBits)) + + def addPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + addOtherBits(in.fire() && + in.bits.hasMultibeatData() && + in.bits.addr_beat === UInt(0), + in.bits.tlDataBeats) + + def dropPendingBitsOnFirstBeat(in: DecoupledIO[Acquire]): UInt = + ~addPendingBitsOnFirstBeat(in) +} + +trait HasDataBuffer extends HasCoherenceAgentParameters { + val data_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerDataBits))) + + type TLDataBundle = TLBundle with HasTileLinkData with HasTileLinkBeatId + + def initDataInner[T <: Acquire](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + data_buffer(in.bits.addr_beat) := in.bits.data + } + } + + // TODO: provide func for accessing when innerDataBeats =/= outerDataBeats or internalDataBeats + def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { + data_buffer(beat) := incoming + } + + def mergeDataInner[T <: TLDataBundle](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(innerDataBits)(in.bits.addr_beat, in.bits.data) + } + } + + def mergeDataOuter[T <: TLDataBundle](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + mergeData(outerDataBits)(in.bits.addr_beat, in.bits.data) + } + } +} + +trait HasByteWriteMaskBuffer extends HasDataBuffer { + val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerWriteMaskBits))) + + override def initDataInner[T <: Acquire](in: DecoupledIO[T]) { + when(in.fire() && in.bits.hasData()) { + val beat = in.bits.addr_beat + val full = FillInterleaved(8, in.bits.wmask()) + data_buffer(beat) := (~full & data_buffer(beat)) | (full & in.bits.data) + wmask_buffer(beat) := in.bits.wmask() | wmask_buffer(beat) // assumes wmask_buffer is zeroed + } + } + + override def mergeData(dataBits: Int)(beat: UInt, incoming: UInt) { + val old_data = incoming // Refilled, written back, or de-cached data + val new_data = data_buffer(beat) // Newly Put data is already in the buffer + val wmask = FillInterleaved(8, wmask_buffer(beat)) + data_buffer(beat) := ~wmask & old_data | wmask & new_data + } +} + +trait HasBlockAddressBuffer extends HasCoherenceAgentParameters { + val xact_addr_block = Reg(init = UInt(0, width = blockAddrBits)) +} + + +trait HasAcquireMetadataBuffer extends HasBlockAddressBuffer { + val xact_allocate = Reg{ Bool() } + val xact_amo_shift_bytes = Reg{ UInt() } + val xact_op_code = Reg{ UInt() } + val xact_addr_byte = Reg{ UInt() } + val xact_op_size = Reg{ UInt() } + val xact_addr_beat = Wire(UInt()) + val xact_iacq = Wire(new SecondaryMissInfo) +} + +trait HasVoluntaryReleaseMetadataBuffer extends HasBlockAddressBuffer + with HasPendingBitHelpers + with HasXactTrackerStates { + def io: HierarchicalXactTrackerIO + + val xact_vol_ir_r_type = Reg{ UInt() } + val xact_vol_ir_src = Reg{ UInt() } + val xact_vol_ir_client_xact_id = Reg{ UInt() } + + def xact_vol_irel = Release( + src = xact_vol_ir_src, + voluntary = Bool(true), + r_type = xact_vol_ir_r_type, + client_xact_id = xact_vol_ir_client_xact_id, + addr_block = xact_addr_block) + (p.alterPartial({ case TLId => p(InnerTLId) })) +} + +trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { + def inner_coh: ManagerMetadata + + val pending_irel_data = Reg(init=Bits(0, width = innerDataBeats)) + val vol_ignt_counter = Wire(new TwoWayBeatCounterStatus) + + def irel_can_merge: Bool + def irel_same_xact: Bool + def irel_is_allocating: Bool = state === s_idle && io.alloc_irel.should && io.inner.release.valid + def irel_is_merging: Bool = (irel_can_merge || irel_same_xact) && io.inner.release.valid + + def innerRelease(block_vol_ignt: Bool = Bool(false), next: UInt = s_busy) { + connectTwoWayBeatCounters( + status = vol_ignt_counter, + up = io.inner.release, + down = io.inner.grant, + trackUp = (r: Release) => { + Mux(state === s_idle, io.alloc_irel.should, io.alloc_irel.matches) && r.isVoluntary() && r.requiresAck() + }, + trackDown = (g: Grant) => (state =/= s_idle) && g.isVoluntary()) + + pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) + + when(irel_is_allocating) { + xact_addr_block := io.irel().addr_block + state := next + } + + when(io.inner.release.fire()) { + when(io.alloc_irel.should || (irel_can_merge && io.irel().first())) { + xact_vol_ir_r_type := io.irel().r_type + xact_vol_ir_src := io.irel().client_id + xact_vol_ir_client_xact_id := io.irel().client_xact_id + pending_irel_data := Mux(io.irel().hasMultibeatData(), + dropPendingBitWhenBeatHasData(io.inner.release), + UInt(0)) + } + } + + io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) && + vol_ignt_counter.pending && + !(pending_irel_data.orR || block_vol_ignt) + + io.inner.grant.bits := inner_coh.makeGrant(xact_vol_irel) + + scoreboard += (pending_irel_data.orR, vol_ignt_counter.pending) + } + +} + +trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { + val pending_orel = Reg(init=Bool(false)) + val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats)) + val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus) + + def outerRelease( + coh: ClientMetadata, + buffering: Bool = Bool(true), + data: UInt = io.irel().data, + add_pending_bit: UInt = UInt(0)) { + pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) | + addPendingBitWhenBeatHasData(io.inner.release) | + add_pending_bit + + connectTwoWayBeatCounters( + status = vol_ognt_counter, + up = io.outer.release, + down = io.outer.grant, + trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(), + trackDown = (g: Grant) => g.isVoluntary()) + + io.outer.release.valid := state === s_busy && + Mux(io.orel().hasData(), + Mux(buffering, + pending_orel_data(vol_ognt_counter.up.idx), + io.inner.release.valid), + pending_orel) + + + io.outer.release.bits := coh.makeVoluntaryWriteback( + client_xact_id = UInt(0), // TODO was tracker id, but not needed? + addr_block = xact_addr_block, + addr_beat = vol_ognt_counter.up.idx, + data = data) + + when(pending_orel_data.orR) { pending_orel := Bool(true) } + when(vol_ognt_counter.up.done) { pending_orel := Bool(false) } + + io.outer.grant.ready := state === s_busy + + scoreboard += (pending_orel, vol_ognt_counter.pending) + } +} + +trait EmitsInnerProbes extends HasBlockAddressBuffer + with HasXactTrackerStates + with HasPendingBitHelpers { + def io: HierarchicalXactTrackerIO + + val pending_iprbs = Reg(UInt(width = innerNCachingClients)) + val curr_probe_dst = PriorityEncoder(pending_iprbs) + val irel_counter = Wire(new TwoWayBeatCounterStatus) + + def full_representation: UInt + def initializeProbes() { pending_iprbs := full_representation & ~io.incoherent.toBits } + def irel_same_xact = io.irel().conflicts(xact_addr_block) && + !io.irel().isVoluntary() && + state === s_inner_probe + + def innerProbe(prb: Probe, next: UInt) { + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := prb + + connectTwoWayBeatCounters( + status = irel_counter, + up = io.inner.probe, + down = io.inner.release, + max = innerNCachingClients, + trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary()) + + when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) { + state := next + } + + //N.B. no pending bits added to scoreboard because all handled in s_inner_probe + } +} + +trait RoutesInParent extends HasBlockAddressBuffer + with HasXactTrackerStates { + def io: HierarchicalXactTrackerIO + type AddrComparison = HasCacheBlockAddress => Bool + def exactAddrMatch(a: HasCacheBlockAddress): Bool = a.conflicts(xact_addr_block) + def routeInParent(iacqMatches: AddrComparison = exactAddrMatch, + irelMatches: AddrComparison = exactAddrMatch, + oprbMatches: AddrComparison = exactAddrMatch) { + io.alloc_iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq()) + io.alloc_irel.matches := (state =/= s_idle) && irelMatches(io.irel()) + io.alloc_oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb()) + io.alloc_iacq.can := state === s_idle + io.alloc_irel.can := state === s_idle + io.alloc_oprb.can := Bool(false) + } +} + +trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer + with AcceptsVoluntaryReleases + with HasXactTrackerStates + with HasPendingBitHelpers { + def io: HierarchicalXactTrackerIO + def nSecondaryMisses: Int + def alwaysWriteFullBeat: Boolean + def inner_coh: ManagerMetadata + def trackerId: Int + + // Secondary miss queue holds transaction metadata used to make grants + lazy val ignt_q = Module(new Queue( + new SecondaryMissInfo()(p.alterPartial({ case TLId => p(InnerTLId) })), + 1 + nSecondaryMisses)) + + val pending_ignt = Wire(Bool()) + val ignt_data_idx = Wire(UInt()) + val ignt_data_done = Wire(Bool()) + val ifin_counter = Wire(new TwoWayBeatCounterStatus) + val pending_put_data = Reg(init=Bits(0, width = innerDataBeats)) + val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) + + def iacq_same_xact: Bool = { + (xact_iacq.client_xact_id === io.iacq().client_xact_id) && + xact_iacq.hasMultibeatData() && + pending_ignt && + pending_put_data(io.iacq().addr_beat) + } + def iacq_can_merge: Bool + def iacq_is_allocating: Bool = state === s_idle && io.alloc_iacq.should && io.inner.acquire.valid + def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid + + def innerAcquire(can_alloc: Bool, next: UInt) { + + // Enqueue some metadata information that we'll use to make coherence updates with later + ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && !iacq_same_xact) + ignt_q.io.enq.bits := io.iacq() + + // Use the outputs of the queue to make further messages + xact_iacq := Mux(ignt_q.io.deq.valid, ignt_q.io.deq.bits, ignt_q.io.enq.bits) + xact_addr_beat := xact_iacq.addr_beat + pending_ignt := ignt_q.io.count > UInt(0) + + // Track whether any beats are missing from a PutBlock + pending_put_data := (pending_put_data & + dropPendingBitWhenBeatHasData(io.inner.acquire)) | + addPendingBitsOnFirstBeat(io.inner.acquire) + + // Intialize transaction metadata for accepted Acquire + when(iacq_is_allocating) { + xact_addr_block := io.iacq().addr_block + xact_allocate := io.iacq().allocate() && can_alloc + xact_amo_shift_bytes := io.iacq().amo_shift_bytes() + xact_op_code := io.iacq().op_code() + xact_addr_byte := io.iacq().addr_byte() + xact_op_size := io.iacq().op_size() + // Make sure to collect all data from a PutBlock + pending_put_data := Mux( + io.iacq().isBuiltInType(Acquire.putBlockType), + dropPendingBitWhenBeatHasData(io.inner.acquire), + UInt(0)) + pending_ignt_data := UInt(0) + state := next + } + + scoreboard += (pending_put_data.orR) + } + + def innerGrant( + data: UInt = io.ognt().data, + external_pending: Bool = Bool(false), + add: UInt = UInt(0)) { + // Track the number of outstanding inner.finishes + connectTwoWayBeatCounters( + status = ifin_counter, + up = io.inner.grant, + down = io.inner.finish, + max = nSecondaryMisses, + trackUp = (g: Grant) => g.requiresAck()) + + // Track which beats are ready for response + when(!iacq_is_allocating) { + pending_ignt_data := (pending_ignt_data & dropPendingBitWhenBeatHasData(io.inner.grant)) | + addPendingBitWhenBeatHasData(io.inner.release) | + addPendingBitWhenBeatHasData(io.outer.grant) | + add + } + + // We can issue a grant for a pending write once all data is + // received and committed to the data array or outer memory + val ignt_ack_ready = !(state === s_idle || + state === s_meta_read || + pending_put_data.orR) + + val ignt_from_iacq = inner_coh.makeGrant( + sec = ignt_q.io.deq.bits, + manager_xact_id = UInt(trackerId), + data = data) + + // Make the Grant message using the data stored in the secondary miss queue + val (cnt, done) = connectOutgoingDataBeatCounter(io.inner.grant, ignt_q.io.deq.bits.addr_beat) + ignt_data_idx := cnt + ignt_data_done := done + ignt_q.io.deq.ready := Bool(false) + when(!vol_ignt_counter.pending) { + ignt_q.io.deq.ready := ignt_data_done + io.inner.grant.bits := ignt_from_iacq + io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter + when (state === s_busy && pending_ignt) { + io.inner.grant.valid := !external_pending && + Mux(io.ignt().hasData(), pending_ignt_data(ignt_data_idx), ignt_ack_ready) + } + } + + // We must wait for as many Finishes as we sent Grants + io.inner.finish.ready := state === s_busy + + scoreboard += (pending_ignt, ifin_counter.pending) + } + +} + +trait EmitsOuterAcquires extends AcceptsInnerAcquires { + val ognt_counter = Wire(new TwoWayBeatCounterStatus) + + // Handle misses or coherence permission upgrades by initiating a new transaction in the outer memory: + // + // If we're allocating in this cache, we can use the current metadata + // to make an appropriate custom Acquire, otherwise we copy over the + // built-in Acquire from the inner TL to the outer TL + def outerAcquire( + caching: Bool, + coh: ClientMetadata, + buffering: Bool = Bool(true), + data: UInt = io.iacq().data, + wmask: UInt = io.iacq().wmask(), + next: UInt = s_busy) { + + // Tracks outstanding Acquires, waiting for their matching Grant. + connectTwoWayBeatCounters( + status = ognt_counter, + up = io.outer.acquire, + down = io.outer.grant, + beat = xact_addr_beat, + trackDown = (g: Grant) => !g.isVoluntary()) + + io.outer.acquire.valid := state === s_outer_acquire && + (xact_allocate || + Mux(buffering, + !pending_put_data(ognt_counter.up.idx), + io.inner.acquire.valid)) + + io.outer.acquire.bits := + Mux(caching, + coh.makeAcquire( + op_code = xact_op_code, + client_xact_id = UInt(0), + addr_block = xact_addr_block), + BuiltInAcquireBuilder( + a_type = xact_iacq.a_type, + client_xact_id = UInt(0), + addr_block = xact_addr_block, + addr_beat = ognt_counter.up.idx, + data = data, + addr_byte = xact_addr_byte, + operand_size = xact_op_size, + opcode = xact_op_code, + wmask = wmask, + alloc = Bool(false)) + (p.alterPartial({ case TLId => p(OuterTLId)}))) + + when(state === s_outer_acquire && ognt_counter.up.done) { state := next } + + io.outer.grant.ready := state === s_busy + + scoreboard += ognt_counter.pending + } +} + +abstract class VoluntaryReleaseTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p) + with AcceptsVoluntaryReleases + with RoutesInParent { + def irel_can_merge = Bool(false) + def irel_same_xact = io.irel().conflicts(xact_addr_block) && + io.irel().isVoluntary() && + pending_irel_data.orR +} + +abstract class AcquireTracker(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p) + with AcceptsInnerAcquires + with EmitsOuterAcquires + with EmitsInnerProbes + with RoutesInParent { +} diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/uncore.scala index 8def2a1c..be20394d 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/uncore.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import cde.{Parameters, Field} +import junctions._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] @@ -18,6 +19,7 @@ trait HasCoherenceAgentParameters { val nReleaseTransactors = 1 val nAcquireTransactors = p(NAcquireTransactors) val nTransactors = nReleaseTransactors + nAcquireTransactors + val blockAddrBits = p(PAddrBits) - p(CacheBlockOffsetBits) val outerTLId = p(OuterTLId) val outerTLParams = p(TLKey(outerTLId)) val outerDataBeats = outerTLParams.dataBeats @@ -32,6 +34,7 @@ trait HasCoherenceAgentParameters { val innerWriteMaskBits = innerTLParams.writeMaskBits val innerBeatAddrBits = log2Up(innerDataBeats) val innerByteAddrBits = log2Up(innerDataBits/8) + val innerNCachingClients = innerTLParams.nCachingClients val maxManagerXacts = innerTLParams.maxManagerXacts require(outerDataBeats == innerDataBeats) //TODO: fix all xact_data Vecs to remove this requirement } @@ -66,31 +69,33 @@ trait HasCoherenceAgentWiringHelpers { * * When a match is reported, if ready is high the new transaction * is merged; when ready is low the transaction is being blocked. - * When no match is reported, any high readys are presumed to be + * When no match is reported, any high idles are presumed to be * from trackers that are available for allocation, and one is - * assigned via alloc based on priority; f no readys are high then - * all trackers are busy with other transactions. + * assigned via alloc based on priority; if no idles are high then + * all trackers are busy with other transactions. If idle is high + * but ready is low, the tracker will be allocated but does not + * have sufficient buffering for the data. */ def doInputRoutingWithAllocation[T <: TileLinkChannel with HasTileLinkData]( in: DecoupledIO[T], outs: Seq[DecoupledIO[T]], - matches: Seq[Bool], - allocs: Seq[Bool], + allocs: Seq[TrackerAllocation], dataOverrides: Option[Seq[UInt]] = None, allocOverride: Option[Bool] = None, matchOverride: Option[Bool] = None) { val ready_bits = Vec(outs.map(_.ready)).toBits - val alloc_bits = PriorityEncoderOH(ready_bits) - val match_bits = Vec(matches).toBits + val can_alloc_bits = Vec(allocs.map(_.can)).toBits + val should_alloc_bits = PriorityEncoderOH(can_alloc_bits) + val match_bits = Vec(allocs.map(_.matches)).toBits val no_matches = !match_bits.orR val alloc_ok = allocOverride.getOrElse(Bool(true)) val match_ok = matchOverride.getOrElse(Bool(true)) - in.ready := Mux(no_matches, ready_bits.orR, (match_bits & ready_bits).orR) && alloc_ok && match_ok - outs.zip(allocs).zipWithIndex.foreach { case((out, a), i) => + in.ready := (Mux(no_matches, can_alloc_bits, match_bits) & ready_bits).orR && alloc_ok && match_ok + outs.zip(allocs).zipWithIndex.foreach { case((out, alloc), i) => out.valid := in.valid && match_ok && alloc_ok out.bits := in.bits dataOverrides foreach { d => out.bits.data := d(i) } - a := alloc_bits(i) & no_matches & alloc_ok + alloc.should := should_alloc_bits(i) && no_matches && alloc_ok } } } @@ -141,87 +146,10 @@ class HierarchicalTLIO(implicit p: Parameters) extends CoherenceAgentBundle()(p) with HasInnerTLIO with HasCachedOuterTLIO -abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) { +abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends CoherenceAgent()(p) + with HasCoherenceAgentWiringHelpers { val io = new HierarchicalTLIO def innerTL = io.inner def outerTL = io.outer def incoherent = io.incoherent } - -trait HasTrackerAllocationIO extends Bundle { - val matches = new Bundle { - val iacq = Bool(OUTPUT) - val irel = Bool(OUTPUT) - val oprb = Bool(OUTPUT) - } - val alloc = new Bundle { - val iacq = Bool(INPUT) - val irel = Bool(INPUT) - val oprb = Bool(INPUT) - } -} - -class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) - with HasTrackerAllocationIO - -class HierarchicalXactTrackerIO(implicit p: Parameters) extends HierarchicalTLIO()(p) - with HasTrackerAllocationIO - -abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule()(p) - with HasDataBeatCounters { - def addPendingBitWhenBeat[T <: HasBeat](inc: Bool, in: T): UInt = - Fill(in.tlDataBeats, inc) & UIntToOH(in.addr_beat) - - def dropPendingBitWhenBeat[T <: HasBeat](dec: Bool, in: T): UInt = - ~Fill(in.tlDataBeats, dec) | ~UIntToOH(in.addr_beat) - - def addPendingBitWhenId[T <: HasClientId](inc: Bool, in: T): UInt = - Fill(in.tlNCachingClients, inc) & UIntToOH(in.client_id) - - def dropPendingBitWhenId[T <: HasClientId](dec: Bool, in: T): UInt = - ~Fill(in.tlNCachingClients, dec) | ~UIntToOH(in.client_id) - - def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = - addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) - - def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = - addPendingBitWhenBeatHasData(in, in.bits.allocate()) - - def addPendingBitWhenBeatIsGetOrAtomic(in: DecoupledIO[AcquireFromSrc]): UInt = { - val a = in.bits - val isGetOrAtomic = a.isBuiltInType() && - (Vec(Acquire.getType, Acquire.getBlockType, Acquire.putAtomicType).contains(a.a_type)) - addPendingBitWhenBeat(in.fire() && isGetOrAtomic, a) - } - - def addPendingBitsFromAcquire(a: SecondaryMissInfo): UInt = - Mux(a.hasMultibeatData(), Fill(a.tlDataBeats, UInt(1, 1)), UIntToOH(a.addr_beat)) - - def dropPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T]): UInt = - dropPendingBitWhenBeat(in.fire() && in.bits.hasData(), in.bits) - - def dropPendingBitAtDest[T <: HasId](in: DecoupledIO[T]): UInt = - dropPendingBitWhenId(in.fire(), in.bits) - - def dropPendingBitAtDestWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = - dropPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) - - def addPendingBitAtSrc[T <: HasId](in: DecoupledIO[T]): UInt = - addPendingBitWhenId(in.fire(), in.bits) - - def addPendingBitAtSrcWhenVoluntary[T <: HasId with MightBeVoluntary](in: DecoupledIO[T]): UInt = - addPendingBitWhenId(in.fire() && in.bits.isVoluntary(), in.bits) - - def pinAllReadyValidLow[T <: Data](b: Bundle) { - b.elements.foreach { - _._2 match { - case d: DecoupledIO[_] => - if(d.ready.dir == OUTPUT) d.ready := Bool(false) - else if(d.valid.dir == OUTPUT) d.valid := Bool(false) - case v: ValidIO[_] => if(v.valid.dir == OUTPUT) v.valid := Bool(false) - case b: Bundle => pinAllReadyValidLow(b) - case _ => - } - } - } -} From 2789e60b6b5854b7fabbe531410776b5f642174a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 14 Jun 2016 16:07:02 -0700 Subject: [PATCH 628/688] fix ignt_q logic --- uncore/src/main/scala/trackers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index 87d7b0b9..252a0162 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -385,7 +385,7 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer def innerAcquire(can_alloc: Bool, next: UInt) { // Enqueue some metadata information that we'll use to make coherence updates with later - ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && !iacq_same_xact) + ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && io.iacq().first()) ignt_q.io.enq.bits := io.iacq() // Use the outputs of the queue to make further messages From 7e43b1d8896c152f5ed962f000853354ec624c6b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 9 Jun 2016 16:03:29 -0700 Subject: [PATCH 629/688] fix mistaken dequeueing from roq in TileLink unwrapper --- uncore/src/main/scala/converters.scala | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index a76dc287..9be72d61 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -389,12 +389,17 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { io.out.acquire <> acqArb.io.out - acqRoq.io.deq.valid := io.out.grant.fire() && ognt.last() + val grant_deq_roq = io.out.grant.fire() && ognt.last() + + acqRoq.io.deq.valid := acqRoq.io.deq.matches && grant_deq_roq acqRoq.io.deq.tag := ognt.client_xact_id - relRoq.io.deq.valid := io.out.grant.fire() && ognt.last() + relRoq.io.deq.valid := !acqRoq.io.deq.matches && grant_deq_roq relRoq.io.deq.tag := ognt.client_xact_id + assert(!grant_deq_roq || acqRoq.io.deq.matches || relRoq.io.deq.matches, + "TileLink Unwrapper: client_xact_id mismatch") + val gnt_builtin = acqRoq.io.deq.data val gnt_voluntary = relRoq.io.deq.data @@ -622,8 +627,11 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), data = io.nasti.r.bits.data) - assert(!gnt_arb.io.in(0).valid || roq.io.deq.matches, "NASTI tag error") - assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, "NASTI tag error") + + assert(!roq.io.deq.valid || roq.io.deq.matches, + "TL -> NASTI converter ReorderQueue: NASTI tag error") + assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, + "TL -> NASTI ID Mapper: NASTI tag error") gnt_arb.io.in(1).valid := io.nasti.b.valid io.nasti.b.ready := gnt_arb.io.in(1).ready From e716661637583c052b23c08d914c0dfeefd6584f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 14 Jun 2016 17:33:12 -0700 Subject: [PATCH 630/688] make sure merged no-alloc put still allocs if original put allocs --- uncore/src/main/scala/cache.scala | 9 +++++++-- uncore/src/main/scala/trackers.scala | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 2ea72f0d..55226168 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -824,7 +824,9 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) // First, take care of accpeting new acquires or secondary misses // Handling of primary and secondary misses' data and write mask merging def iacq_can_merge = acquiresAreMergeable(io.iacq()) && - state =/= s_idle && state =/= s_meta_write && + state =/= s_idle && + state =/= s_meta_resp && + state =/= s_meta_write && !all_pending_done && !io.inner.release.fire() && !io.outer.grant.fire() && @@ -919,11 +921,14 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) add_pending_bit = addPendingBitWhenBeatNeedsRead(io.inner.acquire, Bool(alwaysWriteFullBeat)), block_pending_read = ognt_counter.pending) + // No override for first accepted acquire + val alloc_override = xact_allocate && (state =/= s_idle) + // Do write // We write data to the cache at this level if it was Put here with allocate flag, // written back dirty, or refilled from outer memory. writeDataArray( - add_pending_bit = (addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire) | + add_pending_bit = (addPendingBitWhenBeatHasDataAndAllocs(io.inner.acquire, alloc_override) | addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant, xact_allocate)), block_pending_write = (ognt_counter.pending || diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index 252a0162..d37ab60e 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -79,8 +79,10 @@ trait HasPendingBitHelpers extends HasDataBeatCounters { def addPendingBitWhenBeatHasData[T <: HasBeat](in: DecoupledIO[T], inc: Bool = Bool(true)): UInt = addPendingBitWhenBeat(in.fire() && in.bits.hasData() && inc, in.bits) - def addPendingBitWhenBeatHasDataAndAllocs(in: DecoupledIO[AcquireFromSrc]): UInt = - addPendingBitWhenBeatHasData(in, in.bits.allocate()) + def addPendingBitWhenBeatHasDataAndAllocs( + in: DecoupledIO[AcquireFromSrc], + alloc_override: Bool = Bool(false)): UInt = + addPendingBitWhenBeatHasData(in, in.bits.allocate() || alloc_override) def addPendingBitWhenBeatNeedsRead(in: DecoupledIO[AcquireFromSrc], inc: Bool = Bool(true)): UInt = { val a = in.bits From aba13cee7fee3191f2d4276ad0915df0f4245952 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 15 Jun 2016 20:06:13 -0700 Subject: [PATCH 631/688] fix BRAM slave so that it can correctly take all TileLink requests --- uncore/src/main/scala/bram.scala | 104 +++++++++++++++++-------------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/bram.scala index 8a822744..379dfa0d 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/bram.scala @@ -9,62 +9,74 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters { val io = new ClientUncachedTileLinkIO().flip - val bram = SeqMem(depth, Bits(width = tlDataBits)) + val bram = SeqMem(depth, Vec(tlDataBytes, UInt(width = 8))) - val (s0_get :: s0_getblk :: s0_put :: s0_putblk :: Nil) = Seq( + val s_idle :: s_getblk :: s_putblk :: s_resp :: Nil = Enum(Bits(), 4) + val state = Reg(init = s_idle) + + val acq = io.acquire.bits + val r_acq = Reg(new AcquireMetadata) + + val (is_get :: is_getblk :: is_put :: is_putblk :: Nil) = Seq( Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType - ).map(io.acquire.bits.isBuiltInType _) + ).map(acq.isBuiltInType _) - val fire_acq = io.acquire.fire() - val fire_gnt = io.grant.fire() + val beats = Reg(UInt(width = tlBeatAddrBits)) - val multibeat = Reg(init = Bool(false)) - when (fire_acq) { - multibeat := s0_getblk - } - - - val last = Wire(Bool()) - val s0_valid = io.acquire.valid || (multibeat && !last) - val s1_valid = Reg(next = s0_valid, init = Bool(false)) - val s1_acq = RegEnable(io.acquire.bits, fire_acq) - - val s0_addr = Cat(io.acquire.bits.addr_block, io.acquire.bits.addr_beat) - val s1_beat = s1_acq.addr_beat + Mux(io.grant.ready, UInt(1), UInt(0)) - val s1_addr = Cat(s1_acq.addr_block, s1_beat) - val raddr = Mux(multibeat, s1_addr, s0_addr) - - last := (s1_acq.addr_beat === UInt(tlDataBeats-1)) - val ren = (io.acquire.valid && (s0_get || s0_getblk)) || (multibeat && !last) - val wen = (io.acquire.valid && (s0_put || s0_putblk)) - - val rdata = bram.read(raddr, ren) - val wdata = io.acquire.bits.data - val wmask = io.acquire.bits.wmask() - assert(!wen || (wmask === Fill(tlDataBytes, Bool(true))), - "bram: subblock writes not supported") - when (wen) { - bram.write(s0_addr, wdata) - } - - when (multibeat && fire_gnt) { - s1_acq.addr_beat := s1_beat - when (last) { - multibeat := Bool(false) + when (io.acquire.fire()) { + r_acq := acq + when (is_get || is_put || acq.isPrefetch()) { + state := s_resp + } + when (is_getblk) { + beats := UInt(tlDataBeats - 1) + state := s_getblk + } + /** Need to collect the rest of the beats. + * Beat 0 has already been accepted. */ + when (is_putblk) { + beats := UInt(tlDataBeats - 2) + state := s_putblk } } - io.grant.valid := s1_valid + when (state === s_getblk && io.grant.ready) { + r_acq.addr_beat := r_acq.addr_beat + UInt(1) + beats := beats - UInt(1) + when (beats === UInt(0)) { state := s_idle } + } + + when (state === s_putblk && io.acquire.valid) { + beats := beats - UInt(1) + when (beats === UInt(0)) { state := s_resp } + } + + when (state === s_resp && io.grant.ready) { state := s_idle } + + val acq_addr = Cat(acq.addr_block, acq.addr_beat) + val r_acq_addr = Cat(r_acq.addr_block, r_acq.addr_beat) + + val ren = (io.acquire.fire() && (is_get || is_getblk)) || + (state === s_getblk && io.grant.ready) + val raddr = Mux(state === s_idle, acq_addr, + Mux(io.grant.ready, r_acq_addr + UInt(1), r_acq_addr)) + val rdata = bram.read(raddr, ren) + + val wen = (io.acquire.fire() && (is_put || is_putblk)) + val wdata = Vec.tabulate(tlDataBytes) { i => acq.data((i+1)*8-1, i*8) } + val wmask = Vec.tabulate(tlWriteMaskBits) { i => acq.wmask()(i) } + + when (wen) { bram.write(acq_addr, wdata, wmask) } + + io.grant.valid := (state === s_resp) || (state === s_getblk) io.grant.bits := Grant( is_builtin_type = Bool(true), - g_type = s1_acq.getBuiltInGrantType(), - client_xact_id = s1_acq.client_xact_id, + g_type = r_acq.getBuiltInGrantType(), + client_xact_id = r_acq.client_xact_id, manager_xact_id = UInt(0), - addr_beat = s1_acq.addr_beat, - data = rdata) - - val stall = multibeat || (io.grant.valid && !io.grant.ready) - io.acquire.ready := !stall + addr_beat = r_acq.addr_beat, + data = rdata.toBits) + io.acquire.ready := (state === s_idle) || (state === s_putblk) } class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { From ebe95fa8279a7c0041b2536b7a98b4f07cebb12a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 16 Jun 2016 15:15:36 -0700 Subject: [PATCH 632/688] fix wmask buffer clearing in L2 agents --- uncore/src/main/scala/broadcast.scala | 8 +++----- uncore/src/main/scala/bufferless.scala | 4 ++-- uncore/src/main/scala/cache.scala | 9 ++++----- uncore/src/main/scala/trackers.scala | 19 +++++++++++++------ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 1c51287f..9f996c23 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -122,7 +122,7 @@ class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Param coh = outer_coh.onHit(M_XWR), data = data_buffer(vol_ognt_counter.up.idx)) - quiesce() + quiesce() {} } class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) @@ -189,13 +189,11 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending) when(iacq_is_allocating) { - wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle initializeProbes() } - initDataInner(io.inner.acquire) + initDataInner(io.inner.acquire, iacq_is_allocating || iacq_is_merging) // Wait for everything to quiesce - quiesce() - + quiesce() { clearWmaskBuffer() } } diff --git a/uncore/src/main/scala/bufferless.scala b/uncore/src/main/scala/bufferless.scala index e5ddcb98..2927c928 100644 --- a/uncore/src/main/scala/bufferless.scala +++ b/uncore/src/main/scala/bufferless.scala @@ -78,7 +78,7 @@ class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Par outerRelease(coh = outer_coh.onHit(M_XWR)) io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data - quiesce() + quiesce() {} } class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) @@ -137,5 +137,5 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) when(iacq_is_allocating) { initializeProbes() } // Wait for everything to quiesce - quiesce() + quiesce() {} } diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index 55226168..b67f2621 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -752,7 +752,7 @@ class CacheVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) pending_writes := addPendingBitWhenBeatHasData(io.inner.release) } - quiesce(s_meta_write) + quiesce(s_meta_write) {} // Checks for illegal behavior assert(!(state === s_meta_resp && io.meta.resp.valid && !io.meta.resp.bits.tag_match), @@ -962,7 +962,6 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) // Initialize more transaction metadata. Pla when(iacq_is_allocating) { - wmask_buffer.foreach { w => w := UInt(0) } // This is the only reg that must be clear in s_idle amo_result := UInt(0) pending_meta_write := Bool(false) pending_reads := Mux( // Pick out the specific beats of data that need to be read @@ -973,10 +972,10 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) pending_resps := UInt(0) } - initDataInner(io.inner.acquire) + initDataInner(io.inner.acquire, iacq_is_allocating || iacq_is_merging) // Wait for everything to quiesce - quiesce(Mux(pending_meta_write, s_meta_write, s_idle)) + quiesce(Mux(pending_meta_write, s_meta_write, s_idle)) { clearWmaskBuffer() } } class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { @@ -1056,7 +1055,7 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr io.wb.resp.valid := state === s_busy && all_pending_done io.wb.resp.bits.id := xact_id - quiesce() + quiesce() {} def full_representation = io.wb.req.bits.coh.inner.full() // State machine updates and transaction handler metadata intialization diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index d37ab60e..5852b070 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -28,9 +28,12 @@ abstract class XactTracker(implicit p: Parameters) extends CoherenceAgentModule( override val s_idle :: s_meta_read :: s_meta_resp :: s_wb_req :: s_wb_resp :: s_inner_probe :: s_outer_acquire :: s_busy :: s_meta_write :: Nil = Enum(UInt(), 9) val state = Reg(init=s_idle) - def quiesce(next: UInt = s_idle) { + def quiesce(next: UInt = s_idle)(restore: => Unit) { all_pending_done := !scoreboard.foldLeft(Bool(false))(_||_) - when(state === s_busy && all_pending_done) { state := next } + when(state === s_busy && all_pending_done) { + state := next + restore + } } def pinAllReadyValidLow[T <: Data](b: Bundle) { @@ -129,8 +132,8 @@ trait HasDataBuffer extends HasCoherenceAgentParameters { type TLDataBundle = TLBundle with HasTileLinkData with HasTileLinkBeatId - def initDataInner[T <: Acquire](in: DecoupledIO[T]) { - when(in.fire() && in.bits.hasData()) { + def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) { + when(in.fire() && in.bits.hasData() && alloc) { data_buffer(in.bits.addr_beat) := in.bits.data } } @@ -156,8 +159,8 @@ trait HasDataBuffer extends HasCoherenceAgentParameters { trait HasByteWriteMaskBuffer extends HasDataBuffer { val wmask_buffer = Reg(init=Vec.fill(innerDataBeats)(UInt(0, width = innerWriteMaskBits))) - override def initDataInner[T <: Acquire](in: DecoupledIO[T]) { - when(in.fire() && in.bits.hasData()) { + override def initDataInner[T <: Acquire](in: DecoupledIO[T], alloc: Bool) { + when(in.fire() && in.bits.hasData() && alloc) { val beat = in.bits.addr_beat val full = FillInterleaved(8, in.bits.wmask()) data_buffer(beat) := (~full & data_buffer(beat)) | (full & in.bits.data) @@ -171,6 +174,10 @@ trait HasByteWriteMaskBuffer extends HasDataBuffer { val wmask = FillInterleaved(8, wmask_buffer(beat)) data_buffer(beat) := ~wmask & old_data | wmask & new_data } + + def clearWmaskBuffer() { + wmask_buffer.foreach { w => w := UInt(0) } + } } trait HasBlockAddressBuffer extends HasCoherenceAgentParameters { From b75b6fdcda7486f1299ce31d17eea676792b4532 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 17 Jun 2016 15:31:40 -0700 Subject: [PATCH 633/688] make sure no-data voluntary releases get tracked --- uncore/src/main/scala/broadcast.scala | 5 ++--- uncore/src/main/scala/cache.scala | 12 ++++++------ uncore/src/main/scala/trackers.scala | 16 +++++++++------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index 9f996c23..a0ed179f 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -113,14 +113,13 @@ class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Param // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat io.inner.release.ready := state === s_idle || irel_can_merge || irel_same_xact - when(irel_is_allocating) { pending_orel := io.irel().hasData() } - when(io.inner.release.fire()) { data_buffer(io.irel().addr_beat) := io.irel().data } // Dispatch outer release outerRelease( coh = outer_coh.onHit(M_XWR), - data = data_buffer(vol_ognt_counter.up.idx)) + data = data_buffer(vol_ognt_counter.up.idx), + add_pending_send_bit = irel_is_allocating) quiesce() {} } diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index b67f2621..ecaa3dc7 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -1045,11 +1045,16 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr // If a release didn't write back data, have to read it from data array readDataArray(drop_pending_bit = dropPendingBitWhenBeatHasData(io.inner.release)) + val coh = io.wb.req.bits.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + val needs_outer_release = coh.outer.requiresVoluntaryWriteback() + // Once the data is buffered we can write it back to outer memory outerRelease( coh = outer_coh, data = data_buffer(vol_ognt_counter.up.idx), - add_pending_bit = addPendingBitInternal(io.data.resp)) + add_pending_data_bits = addPendingBitInternal(io.data.resp), + add_pending_send_bit = io.wb.req.fire() && needs_outer_release) // Respond to the initiating transaction handler signalling completion of the writeback io.wb.resp.valid := state === s_busy && all_pending_done @@ -1064,14 +1069,9 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr xact_way_en := io.wb.req.bits.way_en xact_addr_block := (if (cacheIdBits == 0) Cat(io.wb.req.bits.tag, io.wb.req.bits.idx) else Cat(io.wb.req.bits.tag, io.wb.req.bits.idx, UInt(cacheId, cacheIdBits))) - val coh = io.wb.req.bits.coh - val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() - val needs_outer_release = coh.outer.requiresVoluntaryWriteback() when(needs_inner_probes) { initializeProbes() } pending_reads := Mux(needs_outer_release, ~UInt(0, width = innerDataBeats), UInt(0)) pending_resps := UInt(0) - pending_orel := needs_outer_release - //pending_orel_data := UInt(0) pending_coh := coh state := Mux(needs_inner_probes, s_inner_probe, s_busy) } diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index 5852b070..25f2f57e 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -264,18 +264,23 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { } trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { - val pending_orel = Reg(init=Bool(false)) + val pending_orel_send = Reg(init=Bool(false)) val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats)) val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus) + val pending_orel = pending_orel_send || pending_orel_data.orR || vol_ognt_counter.pending def outerRelease( coh: ClientMetadata, buffering: Bool = Bool(true), data: UInt = io.irel().data, - add_pending_bit: UInt = UInt(0)) { + add_pending_data_bits: UInt = UInt(0), + add_pending_send_bit: Bool = Bool(false)) { + pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) | addPendingBitWhenBeatHasData(io.inner.release) | - add_pending_bit + add_pending_data_bits + when (add_pending_send_bit) { pending_orel_send := Bool(true) } + when (io.outer.release.fire()) { pending_orel_send := Bool(false) } connectTwoWayBeatCounters( status = vol_ognt_counter, @@ -290,7 +295,7 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { pending_orel_data(vol_ognt_counter.up.idx), io.inner.release.valid), pending_orel) - + io.outer.release.bits := coh.makeVoluntaryWriteback( client_xact_id = UInt(0), // TODO was tracker id, but not needed? @@ -298,9 +303,6 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { addr_beat = vol_ognt_counter.up.idx, data = data) - when(pending_orel_data.orR) { pending_orel := Bool(true) } - when(vol_ognt_counter.up.done) { pending_orel := Bool(false) } - io.outer.grant.ready := state === s_busy scoreboard += (pending_orel, vol_ognt_counter.pending) From 719fffff4054921d90006546710172918efbe826 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 17 Jun 2016 16:07:47 -0700 Subject: [PATCH 634/688] make sure updates from irel and iacq gated by tracker allocation --- uncore/src/main/scala/broadcast.scala | 12 +++---- uncore/src/main/scala/bufferless.scala | 8 ++--- uncore/src/main/scala/cache.scala | 38 +++++++++++++------- uncore/src/main/scala/sdq.scala | 2 +- uncore/src/main/scala/trackers.scala | 49 +++++++++++++++----------- 5 files changed, 65 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/broadcast.scala index a0ed179f..3665ceaa 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/broadcast.scala @@ -34,14 +34,14 @@ class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent( doInputRoutingWithAllocation( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), - allocs = trackerList.map(_.io.alloc_iacq), + allocs = trackerList.map(_.io.alloc.iacq), allocOverride = !irel_vs_iacq_conflict) // Handle releases, which might be voluntary and might have data doInputRoutingWithAllocation( in = io.inner.release, outs = trackerList.map(_.io.inner.release), - allocs = trackerList.map(_.io.alloc_irel)) + allocs = trackerList.map(_.io.alloc.irel)) // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) @@ -71,7 +71,7 @@ abstract class BroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Para pinAllReadyValidLow(io) // Checks for illegal behavior - assert(!(state === s_idle && io.inner.release.fire() && io.alloc_irel.should && !io.irel().isVoluntary()), + assert(!(state === s_idle && io.inner.release.fire() && io.alloc.irel.should && !io.irel().isVoluntary()), "VoluntaryReleaseTracker accepted Release that wasn't voluntary!") } @@ -88,7 +88,7 @@ abstract class BroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // Checks for illegal behavior // TODO: this could be allowed, but is a useful check against allocation gone wild - assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc_iacq.should && + assert(!(state === s_idle && io.inner.acquire.fire() && io.alloc.iacq.should && io.iacq().hasMultibeatData() && !io.iacq().first()), "AcquireTracker initialized with a tail data beat.") @@ -105,7 +105,7 @@ class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Param // Tell the parent if any incoming messages conflict with the ongoing transaction routeInParent() - io.alloc_iacq.can := Bool(false) + io.alloc.iacq.can := Bool(false) // Start transaction by accepting inner release innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) @@ -130,7 +130,7 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // Setup IOs used for routing in the parent routeInParent() - io.alloc_irel.can := Bool(false) + io.alloc.irel.can := Bool(false) // First, take care of accpeting new acquires or secondary misses // Handling of primary and secondary misses' data and write mask merging diff --git a/uncore/src/main/scala/bufferless.scala b/uncore/src/main/scala/bufferless.scala index 2927c928..eca709e4 100644 --- a/uncore/src/main/scala/bufferless.scala +++ b/uncore/src/main/scala/bufferless.scala @@ -35,7 +35,7 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren doInputRoutingWithAllocation( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), - allocs = trackerList.map(_.io.alloc_iacq), + allocs = trackerList.map(_.io.alloc.iacq), allocOverride = !irel_vs_iacq_conflict) io.outer.acquire.bits.data := io.inner.acquire.bits.data io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat @@ -44,7 +44,7 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren doInputRoutingWithAllocation( in = io.inner.release, outs = trackerList.map(_.io.inner.release), - allocs = trackerList.map(_.io.alloc_irel)) + allocs = trackerList.map(_.io.alloc.irel)) io.outer.release.bits.data := io.inner.release.bits.data io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat @@ -63,7 +63,7 @@ class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Par // Tell the parent if any incoming messages conflict with the ongoing transaction routeInParent() - io.alloc_iacq.can := Bool(false) + io.alloc.iacq.can := Bool(false) // Start transaction by accepting inner release innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) @@ -86,7 +86,7 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // Setup IOs used for routing in the parent routeInParent() - io.alloc_irel.can := Bool(false) + io.alloc.irel.can := Bool(false) // First, take care of accpeting new acquires or secondary misses // Handling of primary and secondary misses' data and write mask merging diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/cache.scala index ecaa3dc7..5464dd1c 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/cache.scala @@ -476,19 +476,19 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) doInputRoutingWithAllocation( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), - allocs = trackerList.map(_.io.alloc_iacq), + allocs = trackerList.map(_.io.alloc.iacq), allocOverride = !irel_vs_iacq_conflict) - assert(PopCount(trackerList.map(_.io.alloc_iacq.should)) <= UInt(1), + assert(PopCount(trackerList.map(_.io.alloc.iacq.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Acquire") // Wire releases from clients doInputRoutingWithAllocation( in = io.inner.release, outs = trackerAndWbIOs.map(_.inner.release), - allocs = trackerAndWbIOs.map(_.alloc_irel)) + allocs = trackerAndWbIOs.map(_.alloc.irel)) - assert(PopCount(trackerAndWbIOs.map(_.alloc_irel.should)) <= UInt(1), + assert(PopCount(trackerAndWbIOs.map(_.alloc.irel.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Release") // Wire probe requests and grant reply to clients, finish acks from clients @@ -570,9 +570,14 @@ trait ReadsFromOuterCacheDataArray extends HasCoherenceMetadataBuffer def readDataArray(drop_pending_bit: UInt, add_pending_bit: UInt = UInt(0), - block_pending_read: Bool = Bool(false)) { + block_pending_read: Bool = Bool(false), + can_update_pending: Bool = Bool(true)) { val port = io.data - pending_reads := (pending_reads & dropPendingBit(port.read) & drop_pending_bit) | add_pending_bit + when (can_update_pending) { + pending_reads := (pending_reads & + dropPendingBit(port.read) & drop_pending_bit) | + add_pending_bit + } port.read.valid := state === s_busy && pending_reads.orR && !block_pending_read port.read.bits := L2DataReadReq( id = UInt(trackerId), @@ -598,9 +603,13 @@ trait WritesToOuterCacheDataArray extends HasCoherenceMetadataBuffer val curr_write_beat = PriorityEncoder(pending_writes) def writeDataArray(add_pending_bit: UInt = UInt(0), - block_pending_write: Bool = Bool(false)) { + block_pending_write: Bool = Bool(false), + can_update_pending: Bool = Bool(true)) { val port = io.data - pending_writes := (pending_writes & dropPendingBit(port.write)) | add_pending_bit + when (can_update_pending) { + pending_writes := (pending_writes & dropPendingBit(port.write)) | + add_pending_bit + } port.write.valid := state === s_busy && pending_writes.orR && !block_pending_write port.write.bits := L2DataWriteReq( id = UInt(trackerId), @@ -720,7 +729,7 @@ class CacheVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) // Avoid metatdata races with writebacks routeInParent(iacqMatches = inSameSet(_, xact_addr_block)) - io.alloc_iacq.can := Bool(false) + io.alloc.iacq.can := Bool(false) // Initialize and accept pending Release beats innerRelease( @@ -733,7 +742,8 @@ class CacheVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) metaRead(io.meta, s_busy) // Write the voluntarily written back data to this cache - writeDataArray(add_pending_bit = addPendingBitWhenBeatHasData(io.inner.release)) + writeDataArray(add_pending_bit = addPendingBitWhenBeatHasData(io.inner.release), + can_update_pending = state =/= s_idle || io.alloc.irel.should) // End a transaction by updating the block metadata metaWrite( @@ -801,7 +811,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) iacqMatches = inSameSet(_, xact_addr_block), irelMatches = (irel: HasCacheBlockAddress) => Mux(before_wb_alloc, inSameSet(irel, xact_addr_block), exactAddrMatch(irel))) - io.alloc_irel.can := Bool(false) + io.alloc.irel.can := Bool(false) // TileLink allows for Gets-under-Get // and Puts-under-Put, and either may also merge with a preceding prefetch @@ -919,7 +929,8 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) drop_pending_bit = (dropPendingBitWhenBeatHasData(io.inner.release) & dropPendingBitWhenBeatHasData(io.outer.grant)), add_pending_bit = addPendingBitWhenBeatNeedsRead(io.inner.acquire, Bool(alwaysWriteFullBeat)), - block_pending_read = ognt_counter.pending) + block_pending_read = ognt_counter.pending, + can_update_pending = state =/= s_idle || io.alloc.irel.should) // No override for first accepted acquire val alloc_override = xact_allocate && (state =/= s_idle) @@ -934,7 +945,8 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) block_pending_write = (ognt_counter.pending || pending_put_data.orR || pending_reads(curr_write_beat) || - pending_resps(curr_write_beat))) + pending_resps(curr_write_beat)), + can_update_pending = state =/= s_idle || io.alloc.iacq.should || io.alloc.irel.should) // Acknowledge or respond with data innerGrant( diff --git a/uncore/src/main/scala/sdq.scala b/uncore/src/main/scala/sdq.scala index 00790d56..6b2cc092 100644 --- a/uncore/src/main/scala/sdq.scala +++ b/uncore/src/main/scala/sdq.scala @@ -45,7 +45,7 @@ trait HasStoreDataQueue extends HasStoreDataQueueParameters { lazy val sdq_alloc_id = PriorityEncoder(~sdq_val) lazy val sdq_rdy = !sdq_val.andR lazy val sdq_enq = trackerIOsList.map( t => - (t.alloc_iacq.should || t.alloc_iacq.matches) && + (t.alloc.iacq.should || t.alloc.iacq.matches) && t.inner.acquire.fire() && t.iacq().hasData() ).reduce(_||_) diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index 25f2f57e..cdf5b6d4 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -11,9 +11,11 @@ class TrackerAllocation extends Bundle { } trait HasTrackerAllocationIO extends Bundle { - val alloc_iacq = new TrackerAllocation - val alloc_irel = new TrackerAllocation - val alloc_oprb = new TrackerAllocation + val alloc = new Bundle { + val iacq = new TrackerAllocation + val irel = new TrackerAllocation + val oprb = new TrackerAllocation + } } class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) @@ -221,7 +223,7 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { def irel_can_merge: Bool def irel_same_xact: Bool - def irel_is_allocating: Bool = state === s_idle && io.alloc_irel.should && io.inner.release.valid + def irel_is_allocating: Bool = state === s_idle && io.alloc.irel.should && io.inner.release.valid def irel_is_merging: Bool = (irel_can_merge || irel_same_xact) && io.inner.release.valid def innerRelease(block_vol_ignt: Bool = Bool(false), next: UInt = s_busy) { @@ -230,11 +232,10 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { up = io.inner.release, down = io.inner.grant, trackUp = (r: Release) => { - Mux(state === s_idle, io.alloc_irel.should, io.alloc_irel.matches) && r.isVoluntary() && r.requiresAck() + Mux(state === s_idle, io.alloc.irel.should, io.alloc.irel.matches) && r.isVoluntary() && r.requiresAck() }, trackDown = (g: Grant) => (state =/= s_idle) && g.isVoluntary()) - pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) when(irel_is_allocating) { xact_addr_block := io.irel().addr_block @@ -242,7 +243,7 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { } when(io.inner.release.fire()) { - when(io.alloc_irel.should || (irel_can_merge && io.irel().first())) { + when(io.alloc.irel.should || (irel_can_merge && io.irel().first())) { xact_vol_ir_r_type := io.irel().r_type xact_vol_ir_src := io.irel().client_id xact_vol_ir_client_xact_id := io.irel().client_xact_id @@ -252,6 +253,10 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { } } + when (irel_is_merging) { + pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) + } + io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) && vol_ignt_counter.pending && !(pending_irel_data.orR || block_vol_ignt) @@ -276,9 +281,11 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { add_pending_data_bits: UInt = UInt(0), add_pending_send_bit: Bool = Bool(false)) { - pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) | - addPendingBitWhenBeatHasData(io.inner.release) | - add_pending_data_bits + when (state =/= s_idle || io.alloc.irel.should) { + pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) | + addPendingBitWhenBeatHasData(io.inner.release) | + add_pending_data_bits + } when (add_pending_send_bit) { pending_orel_send := Bool(true) } when (io.outer.release.fire()) { pending_orel_send := Bool(false) } @@ -352,12 +359,12 @@ trait RoutesInParent extends HasBlockAddressBuffer def routeInParent(iacqMatches: AddrComparison = exactAddrMatch, irelMatches: AddrComparison = exactAddrMatch, oprbMatches: AddrComparison = exactAddrMatch) { - io.alloc_iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq()) - io.alloc_irel.matches := (state =/= s_idle) && irelMatches(io.irel()) - io.alloc_oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb()) - io.alloc_iacq.can := state === s_idle - io.alloc_irel.can := state === s_idle - io.alloc_oprb.can := Bool(false) + io.alloc.iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq()) + io.alloc.irel.matches := (state =/= s_idle) && irelMatches(io.irel()) + io.alloc.oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb()) + io.alloc.iacq.can := state === s_idle + io.alloc.irel.can := state === s_idle + io.alloc.oprb.can := Bool(false) } } @@ -390,7 +397,7 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer pending_put_data(io.iacq().addr_beat) } def iacq_can_merge: Bool - def iacq_is_allocating: Bool = state === s_idle && io.alloc_iacq.should && io.inner.acquire.valid + def iacq_is_allocating: Bool = state === s_idle && io.alloc.iacq.should && io.inner.acquire.valid def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid def innerAcquire(can_alloc: Bool, next: UInt) { @@ -405,9 +412,11 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer pending_ignt := ignt_q.io.count > UInt(0) // Track whether any beats are missing from a PutBlock - pending_put_data := (pending_put_data & - dropPendingBitWhenBeatHasData(io.inner.acquire)) | - addPendingBitsOnFirstBeat(io.inner.acquire) + when (state =/= s_idle || io.alloc.iacq.should) { + pending_put_data := (pending_put_data & + dropPendingBitWhenBeatHasData(io.inner.acquire)) | + addPendingBitsOnFirstBeat(io.inner.acquire) + } // Intialize transaction metadata for accepted Acquire when(iacq_is_allocating) { From e3391b36b2768584bee7092014357f80fc264716 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 21 Jun 2016 10:43:44 -0700 Subject: [PATCH 635/688] get rid of MuxBundle now that MuxCase and MuxLookup are fixed --- uncore/src/main/scala/coherence.scala | 10 +++++----- uncore/src/main/scala/converters.scala | 8 ++++---- uncore/src/main/scala/dma.scala | 2 +- uncore/src/main/scala/util.scala | 10 ---------- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence.scala index f7eb5992..3bf224e9 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence.scala @@ -197,7 +197,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) - MuxBundle(meta, Array( + MuxCase(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) } @@ -299,7 +299,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) - MuxBundle(meta, Array( + MuxCase(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) } @@ -417,7 +417,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) - MuxBundle(meta, Array( + MuxCase(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) } @@ -537,7 +537,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) - MuxBundle(meta, Array( + MuxCase(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped)) } @@ -680,7 +680,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d def managerMetadataOnRelease(incoming: HasReleaseType, src: UInt, meta: ManagerMetadata) = { val popped = ManagerMetadata(sharers=dir.pop(meta.sharers, src))(meta.p) - MuxBundle(meta, Array( + MuxCase(meta, Array( incoming.is(releaseInvalidateData) -> popped, incoming.is(releaseInvalidateAck) -> popped, incoming.is(releaseInvalidateDataMigratory) -> popped, diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala index 9be72d61..93b1372c 100644 --- a/uncore/src/main/scala/converters.scala +++ b/uncore/src/main/scala/converters.scala @@ -887,7 +887,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) wmask = put_wmask.toBits) io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) - io.out.acquire.bits := MuxBundle(get_block_acquire, Seq( + io.out.acquire.bits := MuxCase(get_block_acquire, Seq( sending_put -> put_block_acquire, smallget -> get_acquire, smallput -> put_acquire)) @@ -961,7 +961,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) data = ognt.data) io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) - io.in.grant.bits := MuxBundle(default_grant, Seq( + io.in.grant.bits := MuxCase(default_grant, Seq( returning_data -> get_block_grant, smallgnt -> get_grant)) io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) @@ -1085,7 +1085,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) val pass_valid = io.in.acquire.valid && !stretch - io.out.acquire.bits := MuxBundle(Wire(io.out.acquire.bits, init=iacq), Seq( + io.out.acquire.bits := MuxCase(Wire(io.out.acquire.bits, init=iacq), Seq( (sending_put, put_block_acquire), (shrink, get_block_acquire), (smallput, put_acquire), @@ -1138,7 +1138,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - io.in.grant.bits := MuxBundle(Wire(io.in.grant.bits, init=ognt), Seq( + io.in.grant.bits := MuxCase(Wire(io.in.grant.bits, init=ognt), Seq( sending_get -> get_block_grant, smallget_grant -> get_grant)) diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala index 16fa01b0..79c88be3 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/dma.scala @@ -369,7 +369,7 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) io.mem.acquire.valid := (state === s_get) || (state === s_put && get_done) || (state === s_prefetch && !prefetch_busy(prefetch_id)) - io.mem.acquire.bits := MuxBundle( + io.mem.acquire.bits := MuxLookup( state, prefetch_acquire, Seq( s_get -> get_acquire, s_put -> put_acquire)) diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util.scala index d0afa758..b6d11fd5 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util.scala @@ -4,16 +4,6 @@ package uncore import Chisel._ -object MuxBundle { - def apply[T <: Data] (default: T, mapping: Seq[(Bool, T)]): T = { - mapping.reverse.foldLeft(default)((b, a) => Mux(a._1, a._2, b)) - } - - def apply[T <: Data] (key: UInt, default: T, mapping: Seq[(UInt, T)]): T = { - apply(default, mapping.map{ case (a, b) => (a === key, b) }) - } -} - // Produces 0-width value when counting to 1 class ZCounter(val n: Int) { val value = Reg(init=UInt(0, log2Ceil(n))) From f57524e0c14311322a69aa507c59819db1404c2e Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 23 Jun 2016 00:01:26 -0700 Subject: [PATCH 636/688] Remove FENCE.I from debug ROM; specialize for RV64 --- uncore/src/main/scala/debug.scala | 61 ++++++++++++++++++------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/debug.scala index daa86b82..a065d442 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/debug.scala @@ -64,46 +64,56 @@ object DsbBusConsts { def sbAddrWidth = 12 def sbIdWidth = 10 - //These are the default ROM contents, which support - //RV32 and RV64. RV128 are implemented as NOP. + //These are the default ROM contents, which support RV32 and RV64. // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h/S - // The code assumes 4*0xF bytes of Debug RAM. + // The code assumes 64 bytes of Debug RAM. def defaultRomContents : Array[Byte] = Array( - 0x6f, 0x00, 0x00, 0x06, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + 0x6f, 0x00, 0xc0, 0x04, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0x0f, 0x00, 0xf0, 0x0f, 0xf3, 0x24, 0x00, 0xf1, 0x63, 0xc6, 0x04, 0x00, 0x83, 0x24, 0xc0, 0x43, - 0x6f, 0x00, 0x80, 0x01, 0x93, 0x94, 0x14, 0x00, 0x63, 0xc6, 0x04, 0x00, - 0x83, 0x34, 0x80, 0x43, 0x6f, 0x00, 0x80, 0x00, 0x13, 0x00, 0x00, 0x00, - 0x23, 0x2e, 0x80, 0x42, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x20, 0x80, 0x10, - 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, 0x63, 0x04, 0x04, 0x00, - 0x6f, 0x00, 0x80, 0x05, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, - 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, - 0x13, 0x04, 0x04, 0xf4, 0x63, 0x18, 0x04, 0x02, 0x0f, 0x10, 0x00, 0x00, + 0x6f, 0x00, 0x80, 0x00, 0x83, 0x34, 0x80, 0x43, 0x23, 0x2e, 0x80, 0x42, + 0x73, 0x24, 0x40, 0xf1, 0x23, 0x20, 0x80, 0x10, 0x73, 0x24, 0x00, 0x7b, + 0x13, 0x74, 0x84, 0x00, 0x63, 0x12, 0x04, 0x04, 0x73, 0x24, 0x20, 0x7b, + 0x73, 0x00, 0x20, 0x7b, 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, + 0x13, 0x74, 0x04, 0x1c, 0x13, 0x04, 0x04, 0xf4, 0x63, 0x1e, 0x04, 0x00, 0x73, 0x24, 0x00, 0xf1, 0x63, 0x46, 0x04, 0x00, 0x23, 0x2e, 0x90, 0x42, - 0x67, 0x00, 0x00, 0x40, 0x13, 0x14, 0x14, 0x00, 0x63, 0x46, 0x04, 0x00, - 0x23, 0x3c, 0x90, 0x42, 0x67, 0x00, 0x00, 0x40, 0x13, 0x00, 0x00, 0x00, - 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, - 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, - 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0xdf, 0xfb).map(x => x.toByte) + 0x67, 0x00, 0x00, 0x40, 0x23, 0x3c, 0x90, 0x42, 0x67, 0x00, 0x00, 0x40, + 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, 0x73, 0x60, 0x04, 0x7b, + 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, 0xe3, 0x0c, 0x04, 0xfe, + 0x6f, 0xf0, 0x1f, 0xfd).map(_.toByte) // These ROM contents support only RV32 // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h/S // The code assumes only 28 bytes of Debug RAM. def xlen32OnlyRomContents : Array[Byte] = Array( - 0x6f, 0x00, 0x00, 0x04, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + 0x6f, 0x00, 0xc0, 0x03, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0x0f, 0x00, 0xf0, 0x0f, 0x83, 0x24, 0x80, 0x41, 0x23, 0x2c, 0x80, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x20, 0x80, 0x10, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, - 0x63, 0x04, 0x04, 0x00, 0x6f, 0x00, 0x80, 0x03, 0x73, 0x24, 0x20, 0x7b, - 0x73, 0x00, 0x20, 0x7b, 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, - 0x13, 0x74, 0x04, 0x1c, 0x13, 0x04, 0x04, 0xf4, 0x63, 0x18, 0x04, 0x00, - 0x0f, 0x10, 0x00, 0x00, 0x23, 0x2c, 0x90, 0x40, 0x67, 0x00, 0x00, 0x40, - 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, 0x73, 0x60, 0x04, 0x7b, - 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, 0xe3, 0x0c, 0x04, 0xfe, - 0x6f, 0xf0, 0xdf, 0xfd).map(x => x.toByte) + 0x63, 0x1a, 0x04, 0x02, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, + 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, + 0x13, 0x04, 0x04, 0xf4, 0x63, 0x16, 0x04, 0x00, 0x23, 0x2c, 0x90, 0x40, + 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, + 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, + 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0x1f, 0xfe).map(_.toByte) + // These ROM contents support only RV64 + // See $RISCV/riscv-tools/riscv-isa-sim/debug_rom/debug_rom.h/S + // The code assumes 64 bytes of Debug RAM. + + def xlen64OnlyRomContents : Array[Byte] = Array( + 0x6f, 0x00, 0xc0, 0x03, 0x6f, 0x00, 0xc0, 0x00, 0x13, 0x04, 0xf0, 0xff, + 0x6f, 0x00, 0x80, 0x00, 0x13, 0x04, 0x00, 0x00, 0x0f, 0x00, 0xf0, 0x0f, + 0x83, 0x34, 0x80, 0x43, 0x23, 0x2e, 0x80, 0x42, 0x73, 0x24, 0x40, 0xf1, + 0x23, 0x20, 0x80, 0x10, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x84, 0x00, + 0x63, 0x1a, 0x04, 0x02, 0x73, 0x24, 0x20, 0x7b, 0x73, 0x00, 0x20, 0x7b, + 0x73, 0x10, 0x24, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x1c, + 0x13, 0x04, 0x04, 0xf4, 0x63, 0x16, 0x04, 0x00, 0x23, 0x3c, 0x90, 0x42, + 0x67, 0x00, 0x00, 0x40, 0x73, 0x24, 0x40, 0xf1, 0x23, 0x26, 0x80, 0x10, + 0x73, 0x60, 0x04, 0x7b, 0x73, 0x24, 0x00, 0x7b, 0x13, 0x74, 0x04, 0x02, + 0xe3, 0x0c, 0x04, 0xfe, 0x6f, 0xf0, 0x1f, 0xfe).map(_.toByte) } @@ -243,8 +253,7 @@ class DefaultDebugModuleConfig (val ncomponents : Int, val xlen:Int) }, debugRomContents = xlen match { case 32 => Some(DsbBusConsts.xlen32OnlyRomContents) - case 64 => Some(DsbBusConsts.defaultRomContents) - case 128 => Some(DsbBusConsts.defaultRomContents) + case 64 => Some(DsbBusConsts.xlen64OnlyRomContents) }, hasBusMaster = false, hasAccess128 = false, From 354b81c8fe2dc383becbfb13478921a16d1fbd10 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 23 Jun 2016 13:17:11 -0700 Subject: [PATCH 637/688] Remove legacy HTIF things The SCR file is gone, too, because it is tightly coupled. The general concept could be revived as a module that somehow connects to (or is contained by) the debug module. --- uncore/src/main/scala/dma.scala | 9 +- uncore/src/main/scala/htif.scala | 326 ------------------------------- uncore/src/main/scala/scr.scala | 102 ---------- 3 files changed, 5 insertions(+), 432 deletions(-) delete mode 100644 uncore/src/main/scala/htif.scala delete mode 100644 uncore/src/main/scala/scr.scala diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/dma.scala index 79c88be3..fb157e07 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/dma.scala @@ -83,8 +83,7 @@ class DmaTrackerIO(implicit p: Parameters) extends DmaBundle()(p) { class DmaManager(outstandingCSR: Int)(implicit p: Parameters) extends DmaModule()(p) with HasNastiParameters - with HasAddrMapParameters - with HasHtifParameters { + with HasAddrMapParameters { val io = new Bundle { val ctrl = (new NastiIO).flip @@ -168,7 +167,9 @@ class DmaManager(outstandingCSR: Int)(implicit p: Parameters) } val addrTable = Vec.tabulate(nDmaClients) { i => - UInt(addrMap(s"conf:csr$i").start + outstandingCSR * csrDataBytes) + //UInt(addrMap(s"conf:csr$i").start + outstandingCSR * csrDataBytes) + require(false, "CSR MMIO ports no longer exist") + UInt(0) } io.mmio.ar.valid := Bool(false) @@ -176,7 +177,7 @@ class DmaManager(outstandingCSR: Int)(implicit p: Parameters) io.mmio.aw.bits := NastiWriteAddressChannel( id = UInt(0), addr = addrTable(resp_client_id), - size = UInt(log2Up(csrDataBytes))) + size = { require(false, "CSR MMIO ports no longer exist"); UInt(0) }) io.mmio.w.valid := resp_wdata_pending io.mmio.w.bits := NastiWriteDataChannel(data = resp_status) io.mmio.b.ready := resp_wresp_pending diff --git a/uncore/src/main/scala/htif.scala b/uncore/src/main/scala/htif.scala deleted file mode 100644 index 82c75a30..00000000 --- a/uncore/src/main/scala/htif.scala +++ /dev/null @@ -1,326 +0,0 @@ -// See LICENSE for license details. - -package uncore - -import Chisel._ -import Chisel.ImplicitConversions._ -import junctions._ -import junctions.NastiConstants._ -import cde.{Parameters, Field} - -case object HtifKey extends Field[HtifParameters] - -case class HtifParameters(width: Int, nCores: Int, offsetBits: Int, csrDataBits: Int, nSCR: Int = 64) - -trait HasHtifParameters { - implicit val p: Parameters - val htifExternal = p(HtifKey) - val dataBits = p(TLKey(p(TLId))).dataBitsPerBeat - val dataBeats = p(TLKey(p(TLId))).dataBeats - val w = htifExternal.width - val nSCR = htifExternal.nSCR - val scrAddrBits = log2Up(nSCR) - val scrDataBits = 64 - val scrDataBytes = scrDataBits / 8 - val csrDataBits = htifExternal.csrDataBits - val csrDataBytes = csrDataBits / 8 - val offsetBits = htifExternal.offsetBits - val nCores = htifExternal.nCores -} - -abstract class HtifModule(implicit val p: Parameters) extends Module with HasHtifParameters -abstract class HtifBundle(implicit val p: Parameters) extends ParameterizedBundle()(p) - with HasHtifParameters - -class HostIO(w: Int) extends Bundle { - val clk = Bool(OUTPUT) - val clk_edge = Bool(OUTPUT) - val in = Decoupled(Bits(width = w)).flip - val out = Decoupled(Bits(width = w)) - - override def cloneType = new HostIO(w).asInstanceOf[this.type] -} - -class HtifIO(implicit p: Parameters) extends HtifBundle()(p) { - val reset = Bool(INPUT) - val id = UInt(INPUT, log2Up(nCores)) - val csr = new SmiIO(csrDataBits, 12).flip -} - -class Htif(csr_RESET: Int)(implicit val p: Parameters) extends Module with HasHtifParameters { - val io = new Bundle { - val host = new HostIO(w) - val cpu = Vec(nCores, new HtifIO).flip - val mem = new ClientUncachedTileLinkIO - val scr = new SmiIO(scrDataBits, scrAddrBits) - } - - val short_request_bits = 64 - val long_request_bits = short_request_bits + dataBits*dataBeats - require(short_request_bits % w == 0) - - val rx_count_w = 13 + log2Up(64) - log2Up(w) // data size field is 12 bits - val rx_count = Reg(init=UInt(0,rx_count_w)) - val rx_shifter = Reg(Bits(width = short_request_bits)) - val rx_shifter_in = Cat(io.host.in.bits, rx_shifter(short_request_bits-1,w)) - val next_cmd = rx_shifter_in(3,0) - val cmd = Reg(Bits()) - val size = Reg(Bits()) - val pos = Reg(Bits()) - val seqno = Reg(Bits()) - val addr = Reg(Bits()) - when (io.host.in.valid && io.host.in.ready) { - rx_shifter := rx_shifter_in - rx_count := rx_count + UInt(1) - when (rx_count === UInt(short_request_bits/w-1)) { - cmd := next_cmd - size := rx_shifter_in(15,4) - pos := rx_shifter_in(15,4+offsetBits-3) - seqno := rx_shifter_in(23,16) - addr := rx_shifter_in(63,24) - } - } - - val rx_word_count = (rx_count >> log2Up(short_request_bits/w)) - val rx_word_done = io.host.in.valid && rx_count(log2Up(short_request_bits/w)-1,0).andR - val packet_ram_depth = long_request_bits/short_request_bits-1 - val packet_ram = Mem(packet_ram_depth, Bits(width = short_request_bits)) - when (rx_word_done && io.host.in.ready) { - packet_ram(rx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1)) := rx_shifter_in - } - - val cmd_readmem :: cmd_writemem :: cmd_readcr :: cmd_writecr :: cmd_ack :: cmd_nack :: Nil = Enum(UInt(), 6) - - val csr_addr = addr(io.cpu(0).csr.req.bits.addr.getWidth-1, 0) - val csr_coreid = addr(log2Up(nCores)-1+20+1,20) - val csr_wdata = packet_ram(0) - - val bad_mem_packet = size(offsetBits-1-3,0).orR || addr(offsetBits-1-3,0).orR - val nack = Mux(cmd === cmd_readmem || cmd === cmd_writemem, bad_mem_packet, - Mux(cmd === cmd_readcr || cmd === cmd_writecr, size =/= UInt(1), - Bool(true))) - - val tx_count = Reg(init=UInt(0, rx_count_w)) - val tx_subword_count = tx_count(log2Up(short_request_bits/w)-1,0) - val tx_word_count = tx_count(rx_count_w-1, log2Up(short_request_bits/w)) - val packet_ram_raddr = tx_word_count(log2Up(packet_ram_depth)-1,0) - UInt(1) - when (io.host.out.valid && io.host.out.ready) { - tx_count := tx_count + UInt(1) - } - - val rx_done = rx_word_done && Mux(rx_word_count === UInt(0), next_cmd =/= cmd_writemem && next_cmd =/= cmd_writecr, rx_word_count === size || rx_word_count(log2Up(packet_ram_depth)-1,0) === UInt(0)) - val tx_size = Mux(!nack && (cmd === cmd_readmem || cmd === cmd_readcr || cmd === cmd_writecr), size, UInt(0)) - val tx_done = io.host.out.ready && tx_subword_count.andR && (tx_word_count === tx_size || tx_word_count > UInt(0) && packet_ram_raddr.andR) - - val state_rx :: state_csr_req :: state_csr_resp :: state_mem_rreq :: state_mem_wreq :: state_mem_rresp :: state_mem_wresp :: state_tx :: Nil = Enum(UInt(), 8) - val state = Reg(init=state_rx) - - val (cnt, cnt_done) = Counter((state === state_mem_wreq && io.mem.acquire.ready) || - (state === state_mem_rresp && io.mem.grant.valid), dataBeats) - val rx_cmd = Mux(rx_word_count === UInt(0), next_cmd, cmd) - when (state === state_rx && rx_done) { - state := Mux(rx_cmd === cmd_readmem, state_mem_rreq, - Mux(rx_cmd === cmd_writemem, state_mem_wreq, - Mux(rx_cmd === cmd_readcr || rx_cmd === cmd_writecr, state_csr_req, - state_tx))) - } - when (state === state_mem_wreq) { - when (cnt_done) { state := state_mem_wresp } - } - when (state === state_mem_rreq) { - when(io.mem.acquire.ready) { state := state_mem_rresp } - } - when (state === state_mem_wresp && io.mem.grant.valid) { - state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) - pos := pos - UInt(1) - addr := addr + UInt(1 << offsetBits-3) - } - when (state === state_mem_rresp && cnt_done) { - state := Mux(cmd === cmd_readmem || pos === UInt(1), state_tx, state_rx) - pos := pos - UInt(1) - addr := addr + UInt(1 << offsetBits-3) - } - when (state === state_tx && tx_done) { - when (tx_word_count === tx_size) { - rx_count := UInt(0) - tx_count := UInt(0) - } - state := Mux(cmd === cmd_readmem && pos =/= UInt(0), state_mem_rreq, state_rx) - } - - val n = dataBits/short_request_bits - val mem_req_data = (0 until n).map { i => - def addr(offset: UInt) = - if (dataBits == short_request_bits) offset - else Cat(offset, UInt(i, log2Up(n))) - when (state === state_mem_rresp && io.mem.grant.valid) { - packet_ram(addr(io.mem.grant.bits.addr_beat)) := - io.mem.grant.bits.data((i+1)*short_request_bits-1, i*short_request_bits) - } - packet_ram(addr(cnt)) - }.reverse.reduce(_##_) - - val init_addr = addr.toUInt >> (offsetBits-3) - io.mem.acquire.valid := state === state_mem_rreq || state === state_mem_wreq - io.mem.acquire.bits := Mux(cmd === cmd_writemem, - PutBlock( - addr_block = init_addr, - addr_beat = cnt, - client_xact_id = UInt(0), - data = mem_req_data), - GetBlock(addr_block = init_addr)) - io.mem.grant.ready := Bool(true) - - val csrReadData = Reg(Bits(width = io.cpu(0).csr.resp.bits.getWidth)) - for (i <- 0 until nCores) { - val my_reset = Reg(init=Bool(true)) - - val cpu = io.cpu(i) - val me = csr_coreid === UInt(i) - cpu.csr.req.valid := state === state_csr_req && me && csr_addr =/= UInt(csr_RESET) - cpu.csr.req.bits.rw := cmd === cmd_writecr - cpu.csr.req.bits.addr := csr_addr - cpu.csr.req.bits.data := csr_wdata - cpu.reset := my_reset - - when (cpu.csr.req.fire()) { state := state_csr_resp } - - when (state === state_csr_req && me && csr_addr === UInt(csr_RESET)) { - when (cmd === cmd_writecr) { - my_reset := csr_wdata(0) - } - csrReadData := my_reset.toBits - state := state_tx - } - - cpu.csr.resp.ready := Bool(true) - when (state === state_csr_resp && cpu.csr.resp.valid) { - csrReadData := cpu.csr.resp.bits - state := state_tx - } - } - - io.scr.req.valid := (state === state_csr_req && csr_coreid.andR) - io.scr.req.bits.addr := addr(scrAddrBits - 1, 0).toUInt - io.scr.req.bits.data := csr_wdata - io.scr.req.bits.rw := (cmd === cmd_writecr) - io.scr.resp.ready := Bool(true) - - when (io.scr.req.fire()) { state := state_csr_resp } - when (state === state_csr_resp && io.scr.resp.valid) { - csrReadData := io.scr.resp.bits - state := state_tx - } - - val tx_cmd = Mux(nack, cmd_nack, cmd_ack) - val tx_cmd_ext = Cat(Bits(0, 4-tx_cmd.getWidth), tx_cmd) - val tx_header = Cat(addr, seqno, tx_size, tx_cmd_ext) - val tx_data = Mux(tx_word_count === UInt(0), tx_header, - Mux(cmd === cmd_readcr || cmd === cmd_writecr, csrReadData, - packet_ram(packet_ram_raddr))) - - io.host.in.ready := state === state_rx - io.host.out.valid := state === state_tx - io.host.out.bits := tx_data >> Cat(tx_count(log2Up(short_request_bits/w)-1,0), Bits(0, log2Up(w))) -} - -class NastiIOHostIOConverter(htifW: Int)(implicit val p: Parameters) - extends Module with HasNastiParameters { - val io = new Bundle { - val nasti = (new NastiIO).flip - val host = new HostIO(htifW).flip - val reset = Bool(OUTPUT) - } - - def cloneType = new NastiIOHostIOConverter(htifW).asInstanceOf[this.type] - - val raddr = io.nasti.ar.bits.addr(6, 2) - val waddr = io.nasti.aw.bits.addr(6, 2) - - - val DCOUNT_ADDR = 0x00 - val RFIFO_ADDR = 0x01 - - val WFIFO_ADDR = 0x00 - val RESET_ADDR = 0x1f - - val FIFO_DEPTH = 32 - - val fifo_ren = Reg(init = Bool(false)) - val fifo_wen = Reg(init = Bool(false)) - - val fifo_rd_len = Reg(UInt(width = nastiXLenBits)) - val fifo_rd_id = Reg(UInt(width = nastiXIdBits)) - val fifo_wr_id = Reg(UInt(width = nastiXIdBits)) - val fifo_wr_ack = Reg(init = Bool(false)) - - val rd_count = Reg(init = Bool(false)) - val wr_reset = Reg(init = Bool(false)) - - when (io.nasti.ar.fire()) { - fifo_rd_len := io.nasti.ar.bits.len - fifo_rd_id := io.nasti.ar.bits.id - when (raddr === UInt(RFIFO_ADDR)) { - fifo_ren := Bool(true) - } .elsewhen (raddr === UInt(DCOUNT_ADDR)) { - rd_count := Bool(true) - } - } - - when (io.nasti.r.fire()) { - when (io.nasti.r.bits.last) { - fifo_ren := Bool(false) - rd_count := Bool(false) - } .otherwise { fifo_rd_len := fifo_rd_len - UInt(1) } - } - - when (io.nasti.aw.fire()) { - fifo_wr_id := io.nasti.aw.bits.id - when (waddr === UInt(WFIFO_ADDR)) { - fifo_wen := Bool(true) - } .elsewhen (waddr === UInt(RESET_ADDR)) { - wr_reset := Bool(true) - } - } - - when (io.nasti.w.fire() && io.nasti.w.bits.last) { - fifo_wen := Bool(false) - wr_reset := Bool(false) - fifo_wr_ack := Bool(true) - } - - when (io.nasti.b.fire()) { fifo_wr_ack := Bool(false) } - - io.nasti.ar.ready := !fifo_ren - io.nasti.aw.ready := !fifo_wen && !fifo_wr_ack - io.nasti.b.valid := fifo_wr_ack - io.nasti.b.bits := NastiWriteResponseChannel(id = fifo_wr_id) - - io.reset := io.nasti.w.valid && wr_reset - - val hn_fifo = Module(new MultiWidthFifo(htifW, nastiXDataBits, FIFO_DEPTH)) - hn_fifo.io.in <> io.host.out - hn_fifo.io.out.ready := fifo_ren && io.nasti.r.ready - io.nasti.r.valid := (fifo_ren && hn_fifo.io.out.valid) || rd_count - io.nasti.r.bits := NastiReadDataChannel( - id = fifo_rd_id, - data = Mux(fifo_ren, hn_fifo.io.out.bits, hn_fifo.io.count), - last = (fifo_rd_len === UInt(0))) - - val nh_fifo = Module(new MultiWidthFifo(nastiXDataBits, htifW, FIFO_DEPTH)) - io.host.in <> nh_fifo.io.out - nh_fifo.io.in.valid := fifo_wen && io.nasti.w.valid - nh_fifo.io.in.bits := io.nasti.w.bits.data - io.nasti.w.ready := (fifo_wen && nh_fifo.io.in.ready) || wr_reset - - assert(!io.nasti.w.valid || io.nasti.w.bits.strb.andR, - "Nasti to HostIO converter cannot take partial writes") - assert(!io.nasti.ar.valid || - io.nasti.ar.bits.len === UInt(0) || - io.nasti.ar.bits.burst === BURST_FIXED, - "Nasti to HostIO converter can only take fixed bursts") - assert(!io.nasti.aw.valid || - io.nasti.aw.bits.len === UInt(0) || - io.nasti.aw.bits.burst === BURST_FIXED, - "Nasti to HostIO converter can only take fixed bursts") -} diff --git a/uncore/src/main/scala/scr.scala b/uncore/src/main/scala/scr.scala deleted file mode 100644 index 29a591af..00000000 --- a/uncore/src/main/scala/scr.scala +++ /dev/null @@ -1,102 +0,0 @@ -package uncore - -import Chisel._ -import junctions.SmiIO -import cde.Parameters -import scala.collection.mutable.HashMap -import scala.collection.mutable.ArrayBuffer - -/** Stores a map between SCR file names and address in the SCR file, which can - * later be dumped to a header file for the test bench. */ -class SCRFileMap(prefix: String, maxAddress: Int, baseAddress: BigInt) { - private val addr2name = HashMap.empty[Int, String] - private val name2addr = HashMap.empty[String, Int] - - def allocate(address: Int, name: String): Int = { - Predef.assert(!addr2name.contains(address), "address already allocated") - Predef.assert(!name2addr.contains(name), "name already allocated") - Predef.assert(address < maxAddress, "address too large") - addr2name += (address -> name) - name2addr += (name -> address) - println(prefix + ": %x -> ".format(baseAddress + address) + name) - address - } - - def allocate(name: String): Int = { - val addr = (0 until maxAddress).filter{ addr => !addr2name.contains(addr) }(0) - allocate(addr, name) - } - - def as_c_header(): String = { - addr2name.map{ case(address, name) => - List( - "#define " + prefix + "__" + name + "__PADDR 0x%x".format(baseAddress + address), - "#define " + prefix + "__" + name + "__OFFSET 0x%x".format(address) - ) - }.flatten.mkString("\n") + "\n" - } -} - -class SCRIO(map: SCRFileMap)(implicit p: Parameters) extends HtifBundle()(p) { - val rdata = Vec(nSCR, Bits(INPUT, scrDataBits)) - val wen = Bool(OUTPUT) - val waddr = UInt(OUTPUT, log2Up(nSCR)) - val wdata = Bits(OUTPUT, scrDataBits) - - def attach(regs: Seq[Data], name_base: String): Seq[Data] = { - regs.zipWithIndex.map{ case(reg, i) => attach(reg, name_base + "__" + i) } - } - - def attach(reg: Data, name: String): Data = { - val addr = map.allocate(name) - when (wen && (waddr === UInt(addr))) { - reg := wdata - } - rdata(addr) := reg - reg - } - - def allocate(address: Int, name: String): Unit = { - map.allocate(address, name) - } -} - -class SCRFile(prefix: String, baseAddress: BigInt)(implicit p: Parameters) extends HtifModule()(p) { - val map = new SCRFileMap(prefix, 64, baseAddress) - AllSCRFiles += map - - val io = new Bundle { - val smi = new SmiIO(scrDataBits, scrAddrBits).flip - val scr = new SCRIO(map) - } - - val scr_rdata = Wire(Vec(io.scr.rdata.size, Bits(width=scrDataBits))) - for (i <- 0 until scr_rdata.size) - scr_rdata(i) := io.scr.rdata(i) - - val read_addr = Reg(init = UInt(0, scrAddrBits)) - val resp_valid = Reg(init = Bool(false)) - - io.smi.req.ready := !resp_valid - io.smi.resp.valid := resp_valid - io.smi.resp.bits := scr_rdata(read_addr) - - io.scr.wen := io.smi.req.fire() && io.smi.req.bits.rw - io.scr.wdata := io.smi.req.bits.data - io.scr.waddr := io.smi.req.bits.addr - - when (io.smi.req.fire()) { - read_addr := io.smi.req.bits.addr - resp_valid := Bool(true) - } - when (io.smi.resp.fire()) { resp_valid := Bool(false) } -} - -/** Every elaborated SCR file ends up in this global arry so it can be printed - * out later. */ -object AllSCRFiles { - private var maps = ArrayBuffer.empty[SCRFileMap] - - def +=(map: SCRFileMap): Unit = { maps += map } - def foreach( f: (SCRFileMap => Unit) ): Unit = { maps.foreach{ m => f(m) } } -} From a93a70c8ec1caeb3f34a529c6da60bf574a68561 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 27 Jun 2016 11:40:32 -0700 Subject: [PATCH 638/688] make sure merged voluntary releases are handled properly --- uncore/src/main/scala/trackers.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index cdf5b6d4..ec14730e 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -247,16 +247,16 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { xact_vol_ir_r_type := io.irel().r_type xact_vol_ir_src := io.irel().client_id xact_vol_ir_client_xact_id := io.irel().client_xact_id + // If this release has data, set all the pending bits except the first. + // Otherwise, clear all the pending bits pending_irel_data := Mux(io.irel().hasMultibeatData(), dropPendingBitWhenBeatHasData(io.inner.release), UInt(0)) + } .elsewhen (irel_same_xact) { + pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) } } - when (irel_is_merging) { - pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) - } - io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) && vol_ignt_counter.pending && !(pending_irel_data.orR || block_vol_ignt) From ec5b9dfc8617feb87baad301070efae5bc6c2373 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 27 Jun 2016 16:29:51 -0700 Subject: [PATCH 639/688] make sure trackers can handle case where there are no caching clients --- uncore/src/main/scala/trackers.scala | 42 ++++++++++++++++++---------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/trackers.scala index ec14730e..25323d3c 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/trackers.scala @@ -3,6 +3,7 @@ package uncore import Chisel._ import cde.{Parameters, Field} +import scala.math.max class TrackerAllocation extends Bundle { val matches = Bool(OUTPUT) @@ -321,30 +322,41 @@ trait EmitsInnerProbes extends HasBlockAddressBuffer with HasPendingBitHelpers { def io: HierarchicalXactTrackerIO - val pending_iprbs = Reg(UInt(width = innerNCachingClients)) + val needs_probes = (innerNCachingClients > 0) + val pending_iprbs = Reg(UInt(width = max(innerNCachingClients, 1))) val curr_probe_dst = PriorityEncoder(pending_iprbs) - val irel_counter = Wire(new TwoWayBeatCounterStatus) def full_representation: UInt - def initializeProbes() { pending_iprbs := full_representation & ~io.incoherent.toBits } + def initializeProbes() { + if (needs_probes) + pending_iprbs := full_representation & ~io.incoherent.toBits + else + pending_iprbs := UInt(0) + } def irel_same_xact = io.irel().conflicts(xact_addr_block) && !io.irel().isVoluntary() && state === s_inner_probe def innerProbe(prb: Probe, next: UInt) { - pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) - io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR - io.inner.probe.bits := prb - - connectTwoWayBeatCounters( - status = irel_counter, - up = io.inner.probe, - down = io.inner.release, - max = innerNCachingClients, - trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary()) + if (needs_probes) { + val irel_counter = Wire(new TwoWayBeatCounterStatus) - when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) { - state := next + pending_iprbs := pending_iprbs & dropPendingBitAtDest(io.inner.probe) + io.inner.probe.valid := state === s_inner_probe && pending_iprbs.orR + io.inner.probe.bits := prb + + connectTwoWayBeatCounters( + status = irel_counter, + up = io.inner.probe, + down = io.inner.release, + max = innerNCachingClients, + trackDown = (r: Release) => (state =/= s_idle) && !r.isVoluntary()) + + when(state === s_inner_probe && !(pending_iprbs.orR || irel_counter.pending)) { + state := next + } + } else { + when (state === s_inner_probe) { state := next } } //N.B. no pending bits added to scoreboard because all handled in s_inner_probe From 97e74aec3abda968d7f29cf20430136445032684 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Mon, 27 Jun 2016 23:05:48 -0700 Subject: [PATCH 640/688] Merge RTC and PRCI --- uncore/src/main/scala/prci.scala | 86 ++++++++++++++++++++++++-------- uncore/src/main/scala/rtc.scala | 50 ------------------- 2 files changed, 66 insertions(+), 70 deletions(-) delete mode 100644 uncore/src/main/scala/rtc.scala diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/prci.scala index 13064c44..7f7db90d 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/prci.scala @@ -12,7 +12,6 @@ import cde.{Parameters, Field} case object NTiles extends Field[Int] class PRCIInterrupts extends Bundle { - val mtip = Bool() val meip = Bool() val seip = Bool() val debug = Bool() @@ -22,12 +21,22 @@ class PRCITileIO(implicit p: Parameters) extends Bundle { val reset = Bool(OUTPUT) val id = UInt(OUTPUT, log2Up(p(NTiles))) val interrupts = new PRCIInterrupts { + val mtip = Bool() val msip = Bool() }.asOutput override def cloneType: this.type = new PRCITileIO().asInstanceOf[this.type] } +object PRCI { + def msip(hart: Int) = hart * msipBytes + def timecmp(hart: Int) = 0x4000 + hart * timecmpBytes + def time = 0xbff8 + def msipBytes = 4 + def timecmpBytes = 8 + def size = 0xc000 +} + /** Power, Reset, Clock, Interrupt */ class PRCI(implicit val p: Parameters) extends Module with HasTileLinkParameters @@ -36,16 +45,20 @@ class PRCI(implicit val p: Parameters) extends Module val interrupts = Vec(p(NTiles), new PRCIInterrupts).asInput val tl = new ClientUncachedTileLinkIO().flip val tiles = Vec(p(NTiles), new PRCITileIO) + val rtcTick = Bool(INPUT) } - val ipi = Reg(init=Vec.fill(p(NTiles))(Bool(false))) + val timeWidth = 64 + val timecmp = Reg(Vec(p(NTiles), UInt(width = timeWidth))) + val time = Reg(init=UInt(0, timeWidth)) + when (io.rtcTick) { time := time + UInt(1) } + + val ipi = Reg(init=Vec.fill(p(NTiles))(UInt(0, 32))) val acq = Queue(io.tl.acquire, 1) - val addr = acq.bits.full_addr() + val addr = acq.bits.full_addr()(log2Ceil(PRCI.size)-1,0) val read = acq.bits.isBuiltInType(Acquire.getType) - val write = acq.bits.isBuiltInType(Acquire.putType) val rdata = Wire(init=UInt(0)) - val masked_wdata = (acq.bits.data & acq.bits.full_wmask()) | (rdata & ~acq.bits.full_wmask()) io.tl.grant.valid := acq.valid acq.ready := io.tl.grant.ready io.tl.grant.bits := Grant( @@ -56,25 +69,58 @@ class PRCI(implicit val p: Parameters) extends Module addr_beat = UInt(0), data = rdata) - when (write) { - val ipiRegBytes = 4 - val regsPerBeat = tlDataBytes/ipiRegBytes - val word = - if (regsPerBeat >= ipi.size) UInt(0) - else addr(log2Up(ipi.size*ipiRegBytes)-1,log2Up(tlDataBytes)) - for (i <- 0 until ipi.size by regsPerBeat) { - when (word === i/regsPerBeat) { - rdata := Cat(ipi.slice(i, i + regsPerBeat).map(p => Cat(UInt(0, 8*ipiRegBytes-1), p)).reverse) - for (j <- 0 until (regsPerBeat min (ipi.size - i))) { - when (write) { ipi(i+j) := masked_wdata(j*8*ipiRegBytes) } - } - } - } + when (addr(log2Floor(PRCI.time))) { + require(log2Floor(PRCI.timecmp(p(NTiles)-1)) < log2Floor(PRCI.time)) + rdata := load(Vec(time + UInt(0)), acq.bits) + }.elsewhen (addr >= PRCI.timecmp(0)) { + rdata := store(timecmp, acq.bits) + }.otherwise { + rdata := store(ipi, acq.bits) & Fill(tlDataBits/32, UInt(1, 32)) } for ((tile, i) <- io.tiles zipWithIndex) { tile.interrupts := io.interrupts(i) - tile.interrupts.msip := ipi(i) + tile.interrupts.msip := ipi(i)(0) + tile.interrupts.mtip := time >= timecmp(i) tile.id := UInt(i) } + + // TODO generalize these to help other TL slaves + def load(v: Vec[UInt], acq: Acquire): UInt = { + val w = v.head.getWidth + val a = acq.full_addr() + require(isPow2(w) && w >= 8) + if (w > tlDataBits) { + (v(a(log2Up(w/8*v.size)-1,log2Up(w/8))) >> a(log2Up(w/8)-1,log2Up(tlDataBytes)))(tlDataBits-1,0) + } else { + val row = for (i <- 0 until v.size by tlDataBits/w) + yield Cat(v.slice(i, i + tlDataBits/w).reverse) + if (row.size == 1) row.head + else Vec(row)(a(log2Up(w/8*v.size)-1,log2Up(tlDataBytes))) + } + } + + def store(v: Vec[UInt], acq: Acquire): UInt = { + val w = v.head.getWidth + require(isPow2(w) && w >= 8) + val a = acq.full_addr() + val rdata = load(v, acq) + val wdata = (acq.data & acq.full_wmask()) | (rdata & ~acq.full_wmask()) + if (w <= tlDataBits) { + val word = + if (tlDataBits/w >= v.size) UInt(0) + else a(log2Up(w/8*v.size)-1,log2Up(tlDataBytes)) + for (i <- 0 until v.size) { + when (acq.isBuiltInType(Acquire.putType) && word === i/(tlDataBits/w)) { + val base = i % (tlDataBits/w) + v(i) := wdata >> (w * (i % (tlDataBits/w))) + } + } + } else { + val i = a(log2Up(w/8*v.size)-1,log2Up(w/8)) + val mask = FillInterleaved(tlDataBits, UIntToOH(a(log2Up(w/8)-1,log2Up(tlDataBytes)))) + v(i) := (wdata & mask) | (v(i) & ~mask) + } + rdata + } } diff --git a/uncore/src/main/scala/rtc.scala b/uncore/src/main/scala/rtc.scala deleted file mode 100644 index d2940578..00000000 --- a/uncore/src/main/scala/rtc.scala +++ /dev/null @@ -1,50 +0,0 @@ -package uncore - -import Chisel._ -import junctions._ -import cde.{Parameters, Field} - -case object RTCPeriod extends Field[Int] - -class RTC(nHarts: Int)(implicit val p: Parameters) extends Module - with HasTileLinkParameters - with HasAddrMapParameters { - val io = new Bundle { - val tl = new ClientUncachedTileLinkIO().flip - val irqs = Vec(nHarts, Bool()).asOutput - } - - val w = 64 - val regs = Reg(Vec(nHarts+1, UInt(width = w))) - require(w == tlDataBits) // TODO relax this constraint for narrower TL - - val acq = Queue(io.tl.acquire, 1) - - val full_addr = acq.bits.full_addr() - val byte_addr = full_addr(log2Up(w/8)-1,0) - val size = w/8*(nHarts+1) - val addr = full_addr(log2Up(size)-1,log2Up(w/8)) - val rdata = regs(addr) - val wdata = acq.bits.data - val read = acq.bits.isBuiltInType(Acquire.getType) - val write = acq.bits.isBuiltInType(Acquire.putType) - val wmask = acq.bits.full_wmask() - assert(!acq.valid || read || write, "unsupported RTC operation") - - io.tl.grant.valid := acq.valid - acq.ready := io.tl.grant.ready - io.tl.grant.bits := Grant( - is_builtin_type = Bool(true), - g_type = acq.bits.getBuiltInGrantType(), - client_xact_id = acq.bits.client_xact_id, - manager_xact_id = UInt(0), - addr_beat = UInt(0), - data = rdata) - - for ((irq, cmp) <- io.irqs zip regs.tail) - irq := (regs(0) >= cmp) - - when (Counter(p(RTCPeriod)).inc()) { regs(0) := regs(0) + UInt(1) } - when (acq.valid && write) { regs(addr) := wdata & wmask | rdata & ~wmask } - when (reset) { regs(0) := UInt(0) } -} From b936aa98264bd9c3ef112cbc5fcaff1573c761c8 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 28 Jun 2016 11:21:38 -0700 Subject: [PATCH 641/688] refactor uncore files into separate packages --- .../scala/{builder.scala => Builder.scala} | 3 + .../main/scala/{consts.scala => Consts.scala} | 1 + uncore/src/main/scala/Package.scala | 4 + .../{uncore.scala => agents/Agents.scala} | 13 +- .../Broadcast.scala} | 10 +- .../Bufferless.scala} | 10 +- .../scala/{cache.scala => agents/Cache.scala} | 11 +- .../scala/{ecc.scala => agents/Ecc.scala} | 2 +- uncore/src/main/scala/agents/Mmio.scala | 73 + .../StoreDataQueue.scala} | 3 +- .../{trackers.scala => agents/Trackers.scala} | 8 +- .../Directory.scala} | 2 +- .../Metadata.scala} | 10 +- .../Policies.scala} | 5 +- uncore/src/main/scala/converters.scala | 1226 ----------------- .../scala/{ahb.scala => converters/Ahb.scala} | 5 +- uncore/src/main/scala/converters/Nasti.scala | 353 +++++ .../scala/{smi.scala => converters/Smi.scala} | 3 +- .../src/main/scala/converters/Tilelink.scala | 538 ++++++++ .../scala/{bram.scala => devices/Bram.scala} | 3 +- .../{debug.scala => devices/Debug.scala} | 4 +- .../scala/{dma.scala => devices/Dma.scala} | 3 +- .../scala/{plic.scala => devices/Plic.scala} | 3 +- .../scala/{prci.scala => devices/Prci.scala} | 3 +- .../scala/{rom.scala => devices/Rom.scala} | 4 +- uncore/src/main/scala/network.scala | 122 -- uncore/src/main/scala/package.scala | 6 - uncore/src/main/scala/tilelink/Arbiters.scala | 196 +++ .../Definitions.scala} | 250 +--- .../Interconnect.scala} | 3 +- uncore/src/main/scala/tilelink/Network.scala | 308 +++++ .../scala/{amoalu.scala => util/AmoAlu.scala} | 7 +- uncore/src/main/scala/util/Counters.scala | 134 ++ uncore/src/main/scala/util/Enqueuer.scala | 56 + .../{util.scala => util/Serializer.scala} | 35 +- 35 files changed, 1750 insertions(+), 1667 deletions(-) rename uncore/src/main/scala/{builder.scala => Builder.scala} (98%) rename uncore/src/main/scala/{consts.scala => Consts.scala} (99%) create mode 100644 uncore/src/main/scala/Package.scala rename uncore/src/main/scala/{uncore.scala => agents/Agents.scala} (95%) rename uncore/src/main/scala/{broadcast.scala => agents/Broadcast.scala} (97%) rename uncore/src/main/scala/{bufferless.scala => agents/Bufferless.scala} (96%) rename uncore/src/main/scala/{cache.scala => agents/Cache.scala} (99%) rename uncore/src/main/scala/{ecc.scala => agents/Ecc.scala} (99%) create mode 100644 uncore/src/main/scala/agents/Mmio.scala rename uncore/src/main/scala/{sdq.scala => agents/StoreDataQueue.scala} (98%) rename uncore/src/main/scala/{trackers.scala => agents/Trackers.scala} (99%) rename uncore/src/main/scala/{directory.scala => coherence/Directory.scala} (98%) rename uncore/src/main/scala/{metadata.scala => coherence/Metadata.scala} (97%) rename uncore/src/main/scala/{coherence.scala => coherence/Policies.scala} (99%) delete mode 100644 uncore/src/main/scala/converters.scala rename uncore/src/main/scala/{ahb.scala => converters/Ahb.scala} (99%) create mode 100644 uncore/src/main/scala/converters/Nasti.scala rename uncore/src/main/scala/{smi.scala => converters/Smi.scala} (94%) create mode 100644 uncore/src/main/scala/converters/Tilelink.scala rename uncore/src/main/scala/{bram.scala => devices/Bram.scala} (98%) rename uncore/src/main/scala/{debug.scala => devices/Debug.scala} (99%) rename uncore/src/main/scala/{dma.scala => devices/Dma.scala} (99%) rename uncore/src/main/scala/{plic.scala => devices/Plic.scala} (99%) rename uncore/src/main/scala/{prci.scala => devices/Prci.scala} (98%) rename uncore/src/main/scala/{rom.scala => devices/Rom.scala} (97%) delete mode 100644 uncore/src/main/scala/network.scala delete mode 100644 uncore/src/main/scala/package.scala create mode 100644 uncore/src/main/scala/tilelink/Arbiters.scala rename uncore/src/main/scala/{tilelink.scala => tilelink/Definitions.scala} (77%) rename uncore/src/main/scala/{interconnect.scala => tilelink/Interconnect.scala} (99%) create mode 100644 uncore/src/main/scala/tilelink/Network.scala rename uncore/src/main/scala/{amoalu.scala => util/AmoAlu.scala} (97%) create mode 100644 uncore/src/main/scala/util/Counters.scala create mode 100644 uncore/src/main/scala/util/Enqueuer.scala rename uncore/src/main/scala/{util.scala => util/Serializer.scala} (69%) diff --git a/uncore/src/main/scala/builder.scala b/uncore/src/main/scala/Builder.scala similarity index 98% rename from uncore/src/main/scala/builder.scala rename to uncore/src/main/scala/Builder.scala index 74cb7fa1..ba2c296a 100644 --- a/uncore/src/main/scala/builder.scala +++ b/uncore/src/main/scala/Builder.scala @@ -3,6 +3,9 @@ package uncore import Chisel._ import cde.{Config, Parameters, ParameterDump, Knob, Dump} import junctions.PAddrBits +import uncore.tilelink._ +import uncore.agents._ +import uncore.coherence._ object UncoreBuilder extends App with FileSystemUtilities { val topModuleName = args(0) diff --git a/uncore/src/main/scala/consts.scala b/uncore/src/main/scala/Consts.scala similarity index 99% rename from uncore/src/main/scala/consts.scala rename to uncore/src/main/scala/Consts.scala index 14904463..a4a4e93b 100644 --- a/uncore/src/main/scala/consts.scala +++ b/uncore/src/main/scala/Consts.scala @@ -47,3 +47,4 @@ trait MemoryOpConstants { def isWrite(cmd: UInt) = cmd === M_XWR || cmd === M_XSC || isAMO(cmd) def isWriteIntent(cmd: UInt) = isWrite(cmd) || cmd === M_PFW || cmd === M_XLR } + diff --git a/uncore/src/main/scala/Package.scala b/uncore/src/main/scala/Package.scala new file mode 100644 index 00000000..c9a35dbb --- /dev/null +++ b/uncore/src/main/scala/Package.scala @@ -0,0 +1,4 @@ +// See LICENSE for license details. +package uncore + +package object constants extends uncore.constants.MemoryOpConstants diff --git a/uncore/src/main/scala/uncore.scala b/uncore/src/main/scala/agents/Agents.scala similarity index 95% rename from uncore/src/main/scala/uncore.scala rename to uncore/src/main/scala/agents/Agents.scala index be20394d..15bf0f7b 100644 --- a/uncore/src/main/scala/uncore.scala +++ b/uncore/src/main/scala/agents/Agents.scala @@ -1,19 +1,18 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ import cde.{Parameters, Field} import junctions._ +import uncore.tilelink._ +import uncore.converters._ +import uncore.coherence._ case object NReleaseTransactors extends Field[Int] case object NProbeTransactors extends Field[Int] case object NAcquireTransactors extends Field[Int] -/** Identifies the TLId of the inner network in a hierarchical cache controller */ -case object InnerTLId extends Field[String] -/** Identifies the TLId of the outer network in a hierarchical cache controller */ -case object OuterTLId extends Field[String] - trait HasCoherenceAgentParameters { implicit val p: Parameters val nReleaseTransactors = 1 @@ -49,7 +48,7 @@ trait HasCoherenceAgentWiringHelpers { out: DecoupledIO[T], ins: Seq[DecoupledIO[T]]) { def lock(o: T) = o.hasMultibeatData() - val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, lock _)) + val arb = Module(new LockingRRArbiter(out.bits, ins.size, out.bits.tlDataBeats, Some(lock _))) out <> arb.io.out arb.io.in <> ins } diff --git a/uncore/src/main/scala/broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala similarity index 97% rename from uncore/src/main/scala/broadcast.scala rename to uncore/src/main/scala/agents/Broadcast.scala index 3665ceaa..cedda9b9 100644 --- a/uncore/src/main/scala/broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { @@ -35,7 +39,7 @@ class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) // Handle releases, which might be voluntary and might have data doInputRoutingWithAllocation( diff --git a/uncore/src/main/scala/bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala similarity index 96% rename from uncore/src/main/scala/bufferless.scala rename to uncore/src/main/scala/agents/Bufferless.scala index eca709e4..959367f8 100644 --- a/uncore/src/main/scala/bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { @@ -36,7 +40,7 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) io.outer.acquire.bits.data := io.inner.acquire.bits.data io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat diff --git a/uncore/src/main/scala/cache.scala b/uncore/src/main/scala/agents/Cache.scala similarity index 99% rename from uncore/src/main/scala/cache.scala rename to uncore/src/main/scala/agents/Cache.scala index 5464dd1c..6f7e1cae 100644 --- a/uncore/src/main/scala/cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -1,9 +1,14 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ import scala.reflect.ClassTag import junctions._ +import uncore.util.AMOALU +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ import cde.{Parameters, Field} case object CacheName extends Field[String] @@ -12,11 +17,9 @@ case object NWays extends Field[Int] case object RowBits extends Field[Int] case object Replacer extends Field[() => ReplacementPolicy] case object L2Replacer extends Field[() => SeqReplacementPolicy] -case object AmoAluOperandBits extends Field[Int] case object NPrimaryMisses extends Field[Int] case object NSecondaryMisses extends Field[Int] case object CacheBlockBytes extends Field[Int] -case object CacheBlockOffsetBits extends Field[Int] case object ECCCode extends Field[Option[Code]] case object CacheIdBits extends Field[Int] case object CacheId extends Field[Int] @@ -477,7 +480,7 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), - allocOverride = !irel_vs_iacq_conflict) + allocOverride = Some(!irel_vs_iacq_conflict)) assert(PopCount(trackerList.map(_.io.alloc.iacq.should)) <= UInt(1), "At most a single tracker should now be allocated for any given Acquire") diff --git a/uncore/src/main/scala/ecc.scala b/uncore/src/main/scala/agents/Ecc.scala similarity index 99% rename from uncore/src/main/scala/ecc.scala rename to uncore/src/main/scala/agents/Ecc.scala index 4a582fc1..6e5fdba6 100644 --- a/uncore/src/main/scala/ecc.scala +++ b/uncore/src/main/scala/agents/Ecc.scala @@ -1,6 +1,6 @@ // See LICENSE for license details. -package uncore +package uncore.agents import Chisel._ diff --git a/uncore/src/main/scala/agents/Mmio.scala b/uncore/src/main/scala/agents/Mmio.scala new file mode 100644 index 00000000..a3b2ab03 --- /dev/null +++ b/uncore/src/main/scala/agents/Mmio.scala @@ -0,0 +1,73 @@ +package uncore.agents + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +class MMIOTileLinkManagerData(implicit p: Parameters) + extends TLBundle()(p) + with HasClientId + with HasClientTransactionId + +class MMIOTileLinkManager(implicit p: Parameters) + extends CoherenceAgentModule()(p) { + val io = new ManagerTLIO + + // MMIO requests should never need probe or release + io.inner.probe.valid := Bool(false) + io.inner.release.ready := Bool(false) + + val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData() + val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0) + val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1) + + // Acquire and Grant are basically passthru, + // except client_id and client_xact_id need to be converted. + // Associate the inner client_id and client_xact_id + // with the outer client_xact_id. + val xact_pending = Reg(init = UInt(0, maxManagerXacts)) + val xact_id_sel = PriorityEncoder(~xact_pending) + val xact_id_reg = RegEnable(xact_id_sel, multibeat_start) + val xact_multibeat = Reg(init = Bool(false)) + val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel) + val xact_free = !xact_pending.andR + val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData)) + + io.inner.acquire.ready := io.outer.acquire.ready && xact_free + io.outer.acquire.valid := io.inner.acquire.valid && xact_free + io.outer.acquire.bits := io.inner.acquire.bits + io.outer.acquire.bits.client_xact_id := outer_xact_id + + def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool = + !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1) + + def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt = + Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0)) + + def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt = + ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(), + UIntToOH(in.bits.manager_xact_id), UInt(0)) + + def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt = + ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0)) + + xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) & + clearPendingBitOnFin(io.inner.finish) & + clearPendingBitOnGnt(io.inner.grant) + + when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) { + xact_buffer(outer_xact_id) := io.iacq() + } + + when (multibeat_start) { xact_multibeat := Bool(true) } + when (multibeat_end) { xact_multibeat := Bool(false) } + + val gnt_xact = xact_buffer(io.ognt().client_xact_id) + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.valid := io.outer.grant.valid + io.inner.grant.bits := io.outer.grant.bits + io.inner.grant.bits.client_id := gnt_xact.client_id + io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id + io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id + io.inner.finish.ready := Bool(true) +} diff --git a/uncore/src/main/scala/sdq.scala b/uncore/src/main/scala/agents/StoreDataQueue.scala similarity index 98% rename from uncore/src/main/scala/sdq.scala rename to uncore/src/main/scala/agents/StoreDataQueue.scala index 6b2cc092..e2079772 100644 --- a/uncore/src/main/scala/sdq.scala +++ b/uncore/src/main/scala/agents/StoreDataQueue.scala @@ -1,7 +1,8 @@ // See LICENSE for license details. -package uncore +package uncore.agents import Chisel._ +import uncore.tilelink._ import cde.{Parameters, Field} case object L2StoreDataQueueDepth extends Field[Int] diff --git a/uncore/src/main/scala/trackers.scala b/uncore/src/main/scala/agents/Trackers.scala similarity index 99% rename from uncore/src/main/scala/trackers.scala rename to uncore/src/main/scala/agents/Trackers.scala index 25323d3c..13bcb70d 100644 --- a/uncore/src/main/scala/trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -1,8 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.agents + import Chisel._ -import cde.{Parameters, Field} +import uncore.coherence._ +import uncore.tilelink._ +import uncore.util._ +import cde.Parameters import scala.math.max class TrackerAllocation extends Bundle { diff --git a/uncore/src/main/scala/directory.scala b/uncore/src/main/scala/coherence/Directory.scala similarity index 98% rename from uncore/src/main/scala/directory.scala rename to uncore/src/main/scala/coherence/Directory.scala index db555ad3..86e4fde5 100644 --- a/uncore/src/main/scala/directory.scala +++ b/uncore/src/main/scala/coherence/Directory.scala @@ -1,6 +1,6 @@ // See LICENSE for license details. -package uncore +package uncore.coherence import Chisel._ // This class encapsulates transformations on different directory information diff --git a/uncore/src/main/scala/metadata.scala b/uncore/src/main/scala/coherence/Metadata.scala similarity index 97% rename from uncore/src/main/scala/metadata.scala rename to uncore/src/main/scala/coherence/Metadata.scala index 38c51428..c0d7a6bf 100644 --- a/uncore/src/main/scala/metadata.scala +++ b/uncore/src/main/scala/coherence/Metadata.scala @@ -1,9 +1,17 @@ // See LICENSE for license details. -package uncore +package uncore.coherence + import Chisel._ +import uncore.tilelink._ +import uncore.constants._ import cde.{Parameters, Field} +/** Identifies the TLId of the inner network in a hierarchical cache controller */ +case object InnerTLId extends Field[String] +/** Identifies the TLId of the outer network in a hierarchical cache controller */ +case object OuterTLId extends Field[String] + /** Base class to represent coherence information in clients and managers */ abstract class CoherenceMetadata(implicit p: Parameters) extends TLBundle()(p) { val co = tlCoh diff --git a/uncore/src/main/scala/coherence.scala b/uncore/src/main/scala/coherence/Policies.scala similarity index 99% rename from uncore/src/main/scala/coherence.scala rename to uncore/src/main/scala/coherence/Policies.scala index 3bf224e9..a28e4177 100644 --- a/uncore/src/main/scala/coherence.scala +++ b/uncore/src/main/scala/coherence/Policies.scala @@ -1,7 +1,10 @@ // See LICENSE for license details. -package uncore +package uncore.coherence + import Chisel._ +import uncore.tilelink._ +import uncore.constants._ /** The entire CoherencePolicy API consists of the following three traits: * HasCustomTileLinkMessageTypes, used to define custom messages diff --git a/uncore/src/main/scala/converters.scala b/uncore/src/main/scala/converters.scala deleted file mode 100644 index 93b1372c..00000000 --- a/uncore/src/main/scala/converters.scala +++ /dev/null @@ -1,1226 +0,0 @@ -package uncore - -import Chisel._ -import junctions._ -import cde.Parameters - -/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ -object TileLinkIOWrapper { - def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { - val conv = Module(new ClientTileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { - val conv = Module(new TileLinkIOWrapper) - conv.io.in <> tl - conv.io.out - } - def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl - def apply(tl: TileLinkIO): TileLinkIO = tl -} - -class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new UncachedTileLinkIO().flip - val out = new TileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.finish <> io.in.finish - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientUncachedTileLinkIO().flip - val out = new ClientTileLinkIO - } - io.out.acquire <> io.in.acquire - io.in.grant <> io.out.grant - io.out.probe.ready := Bool(true) - io.out.release.valid := Bool(false) -} - -/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse - * to [[uncore.Grant]] that it receives from a manager and forwards to a client - */ -class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) - with HasDataBeatCounters { - val io = new Bundle { - val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip - val refill = Decoupled(new Grant) - val finish = Decoupled(new LogicalNetworkIO(new Finish)) - val ready = Bool(OUTPUT) - } - - val g = io.grant.bits.payload - - if(tlNetworkPreservesPointToPointOrdering) { - io.finish.valid := Bool(false) - io.refill.valid := io.grant.valid - io.refill.bits := g - io.grant.ready := io.refill.ready - io.ready := Bool(true) - } else { - // We only want to send Finishes after we have collected all beats of - // a multibeat Grant. But Grants from multiple managers or transactions may - // get interleaved, so we could need a counter for each. - val done = if(tlNetworkDoesNotInterleaveBeats) { - connectIncomingDataBeatCounterWithHeader(io.grant) - } else { - val entries = 1 << tlClientXactIdBits - def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id - assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") - connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) - } - val q = Module(new FinishQueue(outstanding)) - q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) - q.io.enq.bits := g.makeFinish() - q.io.enq.bits.manager_id := io.grant.bits.header.src - - io.finish.bits.header.src := UInt(srcId) - io.finish.bits.header.dst := q.io.deq.bits.manager_id - io.finish.bits.payload := q.io.deq.bits - io.finish.valid := q.io.deq.valid - q.io.deq.ready := io.finish.ready - - io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid - io.refill.bits := g - io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready - io.ready := q.io.enq.ready - } -} - -class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries) - -/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] - * - * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Probe Probes]]. - * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response, - * setting up the headers for each. - * - * @param clientId network port id of this agent - * @param addrConvert how a physical address maps to a destination manager port id - */ -class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val client = new ClientTileLinkIO().flip - val network = new TileLinkIO - } - - val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) - val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId) - val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) - val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant) - - io.network.acquire <> acq_with_header - io.network.release <> rel_with_header - io.network.finish <> fin_with_header - io.client.probe <> prb_without_header - io.client.grant.bits.manager_id := io.network.grant.bits.header.src - io.client.grant <> gnt_without_header -} - -/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]] - * - * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, - * calculating header.dst and filling in header.src. - * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. - * - * @param clientId network port id of this agent - * @param addrConvert how a physical address maps to a destination manager port id - */ -class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val client = new ClientUncachedTileLinkIO().flip - val network = new TileLinkIO - } - - val finisher = Module(new FinishUnit(clientId)) - finisher.io.grant <> io.network.grant - io.network.finish <> finisher.io.finish - - val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) - val gnt_without_header = finisher.io.refill - - io.network.acquire.bits := acq_with_header.bits - io.network.acquire.valid := acq_with_header.valid && finisher.io.ready - acq_with_header.ready := io.network.acquire.ready && finisher.io.ready - io.client.grant <> gnt_without_header - io.network.probe.ready := Bool(false) - io.network.release.valid := Bool(false) -} - -object ClientTileLinkHeaderCreator { - def apply[T <: ClientToManagerChannel with HasManagerId]( - in: DecoupledIO[T], - clientId: Int) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := in.bits.manager_id - out.valid := in.valid - in.ready := out.ready - out - } - def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( - in: DecoupledIO[T], - clientId: Int, - addrConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(clientId) - out.bits.header.dst := addrConvert(in.bits.addr_block) - out.valid := in.valid - in.ready := out.ready - out - } -} - -/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip - * - * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, - * calculating header.dst and filling in header.src. - * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], - * but supplies client_id instead. - * - * @param managerId the network port id of this agent - * @param idConvert how a sharer id maps to a destination client port id - */ -class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) - (implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val manager = new ManagerTileLinkIO().flip - val network = new TileLinkIO().flip - } - io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) - io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) - io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) - io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src - io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) - io.manager.release.bits.client_id := io.network.release.bits.header.src - io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) -} - -object ManagerTileLinkHeaderCreator { - def apply[T <: ManagerToClientChannel with HasClientId]( - in: DecoupledIO[T], - managerId: Int, - idConvert: UInt => UInt) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) - out.bits.payload := in.bits - out.bits.header.src := UInt(managerId) - out.bits.header.dst := idConvert(in.bits.client_id) - out.valid := in.valid - in.ready := out.ready - out - } -} - -class BeatCounterStatus extends Bundle { - val idx = UInt() - val done = Bool() -} - -class TwoWayBeatCounterStatus extends Bundle { - val pending = Bool() - val up = new BeatCounterStatus() - val down = new BeatCounterStatus() -} - -/** Utility trait containing wiring functions to keep track of how many data beats have - * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. - * - * Won't count message types that don't have data. - * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. - */ -trait HasDataBeatCounters { - type HasBeat = TileLinkChannel with HasTileLinkBeatId - type HasId = TileLinkChannel with HasClientId - - /** Returns the current count on this channel and when a message is done - * @param inc increment the counter (usually .valid or .fire()) - * @param data the actual channel data - * @param beat count to return for single-beat messages - */ - def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { - val multi = data.hasMultibeatData() - val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) - val cnt = Mux(multi, multi_cnt, beat) - val done = Mux(multi, multi_done, inc) - (cnt, done) - } - - /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ - def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( - out: DecoupledIO[T], - beat: UInt = UInt(0)): (UInt, Bool) = - connectDataBeatCounter(out.fire(), out.bits, beat) - - /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on - * incoming channels in case of network reordering. - */ - def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = - connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 - - /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ - def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = - connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 - - /** If the network might interleave beats from different messages, we need a Vec of counters, - * one for every outstanding message id that might be interleaved. - * - * @param getId mapping from Message to counter id - */ - def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( - in: DecoupledIO[LogicalNetworkIO[T]], - entries: Int, - getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { - Vec((0 until entries).map { i => - connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 - }) - } - - /** Provides counters on two channels, as well a meta-counter that tracks how many - * messages have been sent over the up channel but not yet responded to over the down channel - * - * @param status bundle of status of the counters - * @param up outgoing channel - * @param down incoming channel - * @param max max number of outstanding ups with no down - * @param beat overrides cnts on single-beat messages - * @param track whether up's message should be tracked - * @return a tuple containing whether their are outstanding messages, up's count, - * up's done, down's count, down's done - */ - def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel]( - status: TwoWayBeatCounterStatus, - up: DecoupledIO[T], - down: DecoupledIO[S], - max: Int = 1, - beat: UInt = UInt(0), - trackUp: T => Bool = (t: T) => Bool(true), - trackDown: S => Bool = (s: S) => Bool(true)) { - val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat) - val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat) - val cnt = TwoWayCounter(up_done, dn_done, max) - status.pending := cnt > UInt(0) - status.up.idx := up_idx - status.up.done := up_done - status.down.idx := dn_idx - status.down.done := dn_done - } -} - -class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { - val io = new Bundle { - val in = new ClientTileLinkIO().flip - val out = new ClientUncachedTileLinkIO - } - - val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, - Some((acq: Acquire) => acq.hasMultibeatData()))) - - val acqRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val relRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val iacq = io.in.acquire.bits - val irel = io.in.release.bits - val ognt = io.out.grant.bits - - val acq_roq_enq = iacq.first() - val rel_roq_enq = irel.first() - - val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready - val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready - - val acq_helper = DecoupledHelper( - io.in.acquire.valid, - acq_roq_ready, - acqArb.io.in(0).ready) - - val rel_helper = DecoupledHelper( - io.in.release.valid, - rel_roq_ready, - acqArb.io.in(1).ready) - - acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) - acqRoq.io.enq.bits.data := iacq.isBuiltInType() - acqRoq.io.enq.bits.tag := iacq.client_xact_id - - acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) - acqArb.io.in(0).bits := Acquire( - is_builtin_type = Bool(true), - a_type = Mux(iacq.isBuiltInType(), - iacq.a_type, Acquire.getBlockType), - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = iacq.addr_beat, - data = iacq.data, - union = Mux(iacq.isBuiltInType(), - iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) - io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) - - relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) - relRoq.io.enq.bits.data := irel.isVoluntary() - relRoq.io.enq.bits.tag := irel.client_xact_id - - acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) - acqArb.io.in(1).bits := PutBlock( - client_xact_id = irel.client_xact_id, - addr_block = irel.addr_block, - addr_beat = irel.addr_beat, - data = irel.data, - wmask = Acquire.fullWriteMask) - io.in.release.ready := rel_helper.fire(io.in.release.valid) - - io.out.acquire <> acqArb.io.out - - val grant_deq_roq = io.out.grant.fire() && ognt.last() - - acqRoq.io.deq.valid := acqRoq.io.deq.matches && grant_deq_roq - acqRoq.io.deq.tag := ognt.client_xact_id - - relRoq.io.deq.valid := !acqRoq.io.deq.matches && grant_deq_roq - relRoq.io.deq.tag := ognt.client_xact_id - - assert(!grant_deq_roq || acqRoq.io.deq.matches || relRoq.io.deq.matches, - "TileLink Unwrapper: client_xact_id mismatch") - - val gnt_builtin = acqRoq.io.deq.data - val gnt_voluntary = relRoq.io.deq.data - - val acq_grant = Grant( - is_builtin_type = gnt_builtin, - g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.") - val rel_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - io.in.grant.valid := io.out.grant.valid - io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) - io.out.grant.ready := io.in.grant.ready - - io.in.probe.valid := Bool(false) -} - -class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module - with HasTileLinkParameters with HasNastiParameters { - val io = new Bundle { - val req = new Bundle { - val valid = Bool(INPUT) - val ready = Bool(OUTPUT) - val tl_id = UInt(INPUT, tlClientXactIdBits) - val nasti_id = UInt(OUTPUT, nastiXIdBits) - } - val resp = new Bundle { - val valid = Bool(INPUT) - val matches = Bool(OUTPUT) - val nasti_id = UInt(INPUT, nastiXIdBits) - val tl_id = UInt(OUTPUT, tlClientXactIdBits) - } - } - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort - - if (tlClientXactIdBits <= nastiXIdBits) { - io.req.ready := Bool(true) - io.req.nasti_id := io.req.tl_id - io.resp.matches := Bool(true) - io.resp.tl_id := io.resp.nasti_id - } else if (nastiXIdBits <= 2) { - val nQueues = 1 << nastiXIdBits - val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) - io.req.ready := Bool(false) - io.resp.matches := Bool(false) - io.resp.tl_id := UInt(0) - io.req.nasti_id := req_nasti_id - for (i <- 0 until nQueues) { - val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) - queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) - queue.io.enq.bits := io.req.tl_id - when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } - - queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) - when (io.resp.nasti_id === UInt(i)) { - io.resp.matches := queue.io.deq.valid - io.resp.tl_id := queue.io.deq.bits - } - } - } else { - val maxNastiId = 1 << nastiXIdBits - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) - val roq = Module(new ReorderQueue( - UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) - roq.io.enq.valid := io.req.valid - roq.io.enq.bits.data := io.req.tl_id - roq.io.enq.bits.tag := req_nasti_id - io.req.ready := roq.io.enq.ready - io.req.nasti_id := req_nasti_id - roq.io.deq.valid := io.resp.valid - roq.io.deq.tag := io.resp.nasti_id - io.resp.tl_id := roq.io.deq.data - io.resp.matches := roq.io.deq.matches - } -} - -class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { - val addr_beat = UInt(width = tlBeatAddrBits) - val subblock = Bool() -} - -class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) - with HasNastiParameters { - val io = new Bundle { - val tl = new ClientUncachedTileLinkIO().flip - val nasti = new NastiIO - } - - private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( - MT_B -> UInt(0), - MT_BU -> UInt(0), - MT_H -> UInt(1), - MT_HU -> UInt(1), - MT_W -> UInt(2), - MT_WU -> UInt(2), - MT_D -> UInt(3), - MT_Q -> UInt(log2Up(tlDataBytes)))) - - val dataBits = tlDataBits*tlDataBeats - require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction - require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") - - val has_data = io.tl.acquire.bits.hasData() - - val is_subblock = io.tl.acquire.bits.isSubBlockType() - val is_multibeat = io.tl.acquire.bits.hasMultibeatData() - val (tl_cnt_out, tl_wrap_out) = Counter( - io.tl.acquire.fire() && is_multibeat, tlDataBeats) - - val get_valid = io.tl.acquire.valid && !has_data - val put_valid = io.tl.acquire.valid && has_data - - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort - - // Reorder queue saves extra information needed to send correct - // grant back to TL client - val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) - - val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) - val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) - - val get_id_ready = get_id_mapper.io.req.ready - val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) - val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask - - // For Get/GetBlock, make sure Reorder queue can accept new entry - val get_helper = DecoupledHelper( - get_valid, - roq.io.enq.ready, - io.nasti.ar.ready, - get_id_ready) - - val w_inflight = Reg(init = Bool(false)) - - // For Put/PutBlock, make sure aw and w channel are both ready before - // we send the first beat - val aw_ready = w_inflight || io.nasti.aw.ready - val put_helper = DecoupledHelper( - put_valid, - aw_ready, - io.nasti.w.ready, - put_id_ready) - - val (nasti_cnt_out, nasti_wrap_out) = Counter( - io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - - roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) - roq.io.enq.bits.tag := io.nasti.ar.bits.id - roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat - roq.io.enq.bits.data.subblock := is_subblock - roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) - roq.io.deq.tag := io.nasti.r.bits.id - - get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) - get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id - get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last - get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id - - put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) - put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id - put_id_mapper.io.resp.valid := io.nasti.b.fire() - put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id - - // Decompose outgoing TL Acquires into Nasti address and data channels - io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) - io.nasti.ar.bits := NastiReadAddressChannel( - id = get_id_mapper.io.req.nasti_id, - addr = io.tl.acquire.bits.full_addr(), - size = Mux(is_subblock, - opSizeToXSize(io.tl.acquire.bits.op_size()), - UInt(log2Ceil(tlDataBytes))), - len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) - - io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) - io.nasti.aw.bits := NastiWriteAddressChannel( - id = put_id_mapper.io.req.nasti_id, - addr = io.tl.acquire.bits.full_addr(), - size = UInt(log2Ceil(tlDataBytes)), - len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) - - io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) - io.nasti.w.bits := NastiWriteDataChannel( - id = put_id_mapper.io.req.nasti_id, - data = io.tl.acquire.bits.data, - strb = io.tl.acquire.bits.wmask(), - last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) - - io.tl.acquire.ready := Mux(has_data, - put_helper.fire(put_valid), - get_helper.fire(get_valid)) - - when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { - w_inflight := Bool(true) - } - - when (w_inflight) { - when (tl_wrap_out) { w_inflight := Bool(false) } - } - - // Aggregate incoming NASTI responses into TL Grants - val (tl_cnt_in, tl_wrap_in) = Counter( - io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) - io.tl.grant <> gnt_arb.io.out - - gnt_arb.io.in(0).valid := io.nasti.r.valid - io.nasti.r.ready := gnt_arb.io.in(0).ready - gnt_arb.io.in(0).bits := Grant( - is_builtin_type = Bool(true), - g_type = Mux(roq.io.deq.data.subblock, - Grant.getDataBeatType, Grant.getDataBlockType), - client_xact_id = get_id_mapper.io.resp.tl_id, - manager_xact_id = UInt(0), - addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = io.nasti.r.bits.data) - - assert(!roq.io.deq.valid || roq.io.deq.matches, - "TL -> NASTI converter ReorderQueue: NASTI tag error") - assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, - "TL -> NASTI ID Mapper: NASTI tag error") - - gnt_arb.io.in(1).valid := io.nasti.b.valid - io.nasti.b.ready := gnt_arb.io.in(1).ready - gnt_arb.io.in(1).bits := Grant( - is_builtin_type = Bool(true), - g_type = Grant.putAckType, - client_xact_id = put_id_mapper.io.resp.tl_id, - manager_xact_id = UInt(0), - addr_beat = UInt(0), - data = Bits(0)) - assert(!gnt_arb.io.in(1).valid || put_id_mapper.io.resp.matches, "NASTI tag error") - - assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") - assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") -} - -class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) - with HasNastiParameters { - val io = new Bundle { - val nasti = (new NastiIO).flip - val tl = new ClientUncachedTileLinkIO - } - - val (s_idle :: s_put :: Nil) = Enum(Bits(), 2) - val state = Reg(init = s_idle) - - private val blockOffset = tlByteAddrBits + tlBeatAddrBits - - val aw_req = Reg(new NastiWriteAddressChannel) - - def is_singlebeat(chan: NastiAddressChannel): Bool = - chan.len === UInt(0) - - def is_multibeat(chan: NastiAddressChannel): Bool = - chan.len === UInt(tlDataBeats - 1) && chan.size === UInt(log2Up(tlDataBytes)) - - def nasti_addr_block(chan: NastiAddressChannel): UInt = - chan.addr(nastiXAddrBits - 1, blockOffset) - - def nasti_addr_beat(chan: NastiAddressChannel): UInt = - chan.addr(blockOffset - 1, tlByteAddrBits) - - def nasti_addr_byte(chan: NastiAddressChannel): UInt = - chan.addr(tlByteAddrBits - 1, 0) - - def nasti_operand_size(chan: NastiAddressChannel): UInt = - MuxLookup(chan.size, MT_Q, Seq( - UInt(0) -> MT_BU, - UInt(1) -> MT_HU, - UInt(2) -> MT_WU, - UInt(3) -> MT_D)) - - def size_mask(size: UInt): UInt = - (UInt(1) << (UInt(1) << size)) - UInt(1) - - def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { - val base = w.strb & size_mask(aw.size) - val addr_byte = nasti_addr_byte(aw) - w.strb & (size_mask(aw.size) << addr_byte) - } - - def tl_last(gnt: GrantMetadata): Bool = - !gnt.hasMultibeatData() || gnt.addr_beat === UInt(tlDataBeats - 1) - - def tl_b_grant(gnt: GrantMetadata): Bool = - gnt.g_type === Grant.putAckType - - assert(!io.nasti.ar.valid || - is_singlebeat(io.nasti.ar.bits) || is_multibeat(io.nasti.ar.bits), - "NASTI read transaction cannot convert to TileLInk") - - assert(!io.nasti.aw.valid || - is_singlebeat(io.nasti.aw.bits) || is_multibeat(io.nasti.aw.bits), - "NASTI write transaction cannot convert to TileLInk") - - val put_count = Reg(init = UInt(0, tlBeatAddrBits)) - - when (io.nasti.aw.fire()) { - aw_req := io.nasti.aw.bits - state := s_put - } - - when (io.nasti.w.fire()) { - put_count := put_count + UInt(1) - when (io.nasti.w.bits.last) { - put_count := UInt(0) - state := s_idle - } - } - - val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), - GetBlock( - client_xact_id = io.nasti.ar.bits.id, - addr_block = nasti_addr_block(io.nasti.ar.bits)), - Get( - client_xact_id = io.nasti.ar.bits.id, - addr_block = nasti_addr_block(io.nasti.ar.bits), - addr_beat = nasti_addr_beat(io.nasti.ar.bits), - addr_byte = nasti_addr_byte(io.nasti.ar.bits), - operand_size = nasti_operand_size(io.nasti.ar.bits), - alloc = Bool(false))) - - val put_acquire = Mux(is_multibeat(aw_req), - PutBlock( - client_xact_id = aw_req.id, - addr_block = nasti_addr_block(aw_req), - addr_beat = put_count, - data = io.nasti.w.bits.data, - wmask = io.nasti.w.bits.strb), - Put( - client_xact_id = aw_req.id, - addr_block = nasti_addr_block(aw_req), - addr_beat = nasti_addr_beat(aw_req), - data = io.nasti.w.bits.data, - wmask = nasti_wmask(aw_req, io.nasti.w.bits))) - - io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) - io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || - (state === s_put && io.nasti.w.valid) - io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) - io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) - io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) - - val nXacts = tlMaxClientXacts * tlMaxClientsPerPort - - io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) - io.nasti.b.bits := NastiWriteResponseChannel( - id = io.tl.grant.bits.client_xact_id) - - io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) - io.nasti.r.bits := NastiReadDataChannel( - id = io.tl.grant.bits.client_xact_id, - data = io.tl.grant.bits.data, - last = tl_last(io.tl.grant.bits)) - - io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), - io.nasti.b.ready, io.nasti.r.ready) -} - -object TileLinkWidthAdapter { - def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { - require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) - - if (out.tlDataBits > in.tlDataBits) { - val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) - widener.io.in <> in - out <> widener.io.out - } else if (out.tlDataBits < in.tlDataBits) { - val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) - narrower.io.in <> in - out <> narrower.io.out - } else { - out <> in - } - } -} - -class TileLinkIOWidener(innerTLId: String, outerTLId: String) - (implicit p: Parameters) extends TLModule()(p) { - - val paddrBits = p(PAddrBits) - val innerParams = p(TLKey(innerTLId)) - val outerParams = p(TLKey(outerTLId)) - val innerDataBeats = innerParams.dataBeats - val innerDataBits = innerParams.dataBitsPerBeat - val innerWriteMaskBits = innerParams.writeMaskBits - val innerByteAddrBits = log2Up(innerWriteMaskBits) - val innerMaxXacts = innerParams.maxClientXacts * innerParams.maxClientsPerPort - val innerXactIdBits = log2Up(innerMaxXacts) - val outerDataBeats = outerParams.dataBeats - val outerDataBits = outerParams.dataBitsPerBeat - val outerWriteMaskBits = outerParams.writeMaskBits - val outerByteAddrBits = log2Up(outerWriteMaskBits) - val outerBeatAddrBits = log2Up(outerDataBeats) - val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits - val outerMaxClients = outerParams.maxClientsPerPort - val outerClientIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - val outerManagerIdBits = log2Up(outerParams.maxManagerXacts) - val outerBlockAddrBits = paddrBits - outerBlockOffset - - require(outerDataBeats <= innerDataBeats) - require(outerDataBits >= innerDataBits) - require(outerDataBits % innerDataBits == 0) - require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) - - val factor = innerDataBeats / outerDataBeats - - val io = new Bundle { - val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip - val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) - } - - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits - val ignt = io.in.grant.bits - - val shrink = iacq.a_type === Acquire.putBlockType - val stretch = ognt.g_type === Grant.getDataBlockType - val smallget = iacq.a_type === Acquire.getType - val smallput = iacq.a_type === Acquire.putType - val smallgnt = ognt.g_type === Grant.getDataBeatType - - val sending_put = Reg(init = Bool(false)) - val collecting = Reg(init = Bool(false)) - val put_block = Reg(UInt(width = outerBlockAddrBits)) - val put_id = Reg(UInt(width = outerClientIdBits)) - val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) - val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) - val put_allocate = Reg(Bool()) - val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) - val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) - - val in_addr = iacq.full_addr() - val out_addr_block = in_addr(paddrBits - 1, outerBlockOffset) - val out_addr_beat = in_addr(outerBlockOffset - 1, outerByteAddrBits) - val out_addr_byte = in_addr(outerByteAddrBits - 1, 0) - - val switch_addr = in_addr(outerByteAddrBits - 1, innerByteAddrBits) - val smallget_switch = Reg(Vec(innerMaxXacts, switch_addr)) - - def align_data(addr: UInt, data: UInt): UInt = - data << Cat(addr, UInt(0, log2Up(innerDataBits))) - - def align_wmask(addr: UInt, wmask: UInt): UInt = - wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - addr_beat = out_addr_beat, - addr_byte = out_addr_byte, - operand_size = iacq.op_size(), - alloc = iacq.allocate()) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - alloc = iacq.allocate()) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = out_addr_block, - addr_beat = out_addr_beat, - data = align_data(switch_addr, iacq.data), - wmask = align_wmask(switch_addr, iacq.wmask()), - alloc = iacq.allocate()) - - val put_block_acquire = PutBlock( - client_xact_id = put_id, - addr_block = put_block, - addr_beat = put_beat, - data = put_data.toBits, - wmask = put_wmask.toBits) - - io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) - io.out.acquire.bits := MuxCase(get_block_acquire, Seq( - sending_put -> put_block_acquire, - smallget -> get_acquire, - smallput -> put_acquire)) - io.in.acquire.ready := !sending_put && (shrink || io.out.acquire.ready) - - when (io.in.acquire.fire() && shrink) { - when (!collecting) { - put_block := out_addr_block - put_id := iacq.client_xact_id - put_allocate := iacq.allocate() - collecting := Bool(true) - } - put_data(recv_idx) := iacq.data - put_wmask(recv_idx) := iacq.wmask() - } - - when (io.in.acquire.fire() && smallget) { - smallget_switch(iacq.client_xact_id) := switch_addr - } - - when (recv_done) { sending_put := Bool(true) } - when (sending_put && io.out.acquire.ready) { sending_put := Bool(false) } - when (put_done) { collecting := Bool(false) } - - val returning_data = Reg(init = Bool(false)) - val (send_idx, send_done) = Counter( - io.in.grant.ready && returning_data, factor) - - val gnt_beat = Reg(UInt(width = outerBeatAddrBits)) - val gnt_client_id = Reg(UInt(width = outerClientIdBits)) - val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) - val gnt_data = Reg(UInt(width = outerDataBits)) - - when (io.out.grant.fire() && stretch) { - gnt_data := ognt.data - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - gnt_beat := ognt.addr_beat - returning_data := Bool(true) - } - - when (send_done) { returning_data := Bool(false) } - - def select_data(data: UInt, sel: UInt): UInt = - data >> (sel << log2Up(innerDataBits)) - - val gnt_switch = smallget_switch(ognt.client_xact_id) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = Cat(gnt_beat, send_idx), - data = select_data(gnt_data, send_idx)) - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = Cat(ognt.addr_beat, gnt_switch), - data = select_data(ognt.data, gnt_switch)) - - val default_grant = Grant( - is_builtin_type = Bool(true), - g_type = ognt.g_type, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat, - data = ognt.data) - - io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) - io.in.grant.bits := MuxCase(default_grant, Seq( - returning_data -> get_block_grant, - smallgnt -> get_grant)) - io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) -} - -class TileLinkIONarrower(innerTLId: String, outerTLId: String) - (implicit p: Parameters) extends TLModule()(p) { - - val innerParams = p(TLKey(innerTLId)) - val outerParams = p(TLKey(outerTLId)) - val innerDataBeats = innerParams.dataBeats - val innerDataBits = innerParams.dataBitsPerBeat - val innerWriteMaskBits = innerParams.writeMaskBits - val innerByteAddrBits = log2Up(innerWriteMaskBits) - val outerDataBeats = outerParams.dataBeats - val outerDataBits = outerParams.dataBitsPerBeat - val outerWriteMaskBits = outerParams.writeMaskBits - val outerByteAddrBits = log2Up(outerWriteMaskBits) - val outerBeatAddrBits = log2Up(outerDataBeats) - val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits - val outerMaxClients = outerParams.maxClientsPerPort - val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) - - require(outerDataBeats > innerDataBeats) - require(outerDataBeats % innerDataBeats == 0) - require(outerDataBits < innerDataBits) - require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) - - val factor = outerDataBeats / innerDataBeats - - val io = new Bundle { - val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip - val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) - } - - val iacq = io.in.acquire.bits - val ognt = io.out.grant.bits - - val stretch = iacq.a_type === Acquire.putBlockType - val shrink = iacq.a_type === Acquire.getBlockType - val smallput = iacq.a_type === Acquire.putType - val smallget = iacq.a_type === Acquire.getType - - val acq_data_buffer = Reg(UInt(width = innerDataBits)) - val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) - val acq_client_id = Reg(iacq.client_xact_id) - val acq_addr_block = Reg(iacq.addr_block) - val acq_addr_beat = Reg(iacq.addr_beat) - val oacq_ctr = Counter(factor) - - val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) - val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) - - val mask_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerWriteMaskBits - val msb = (i + 1) * outerWriteMaskBits - 1 - iacq.wmask()(msb, lsb) - } - - val data_chunks = Vec.tabulate(factor) { i => - val lsb = i * outerDataBits - val msb = (i + 1) * outerDataBits - 1 - iacq.data(msb, lsb) - } - - val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) - - val smallput_data = Mux1H(beat_sel, data_chunks) - val smallput_wmask = Mux1H(beat_sel, mask_chunks) - val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) - - assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), - "Can't perform Put wider than outer width") - - val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( - MT_B -> Bool(true), - MT_BU -> Bool(true), - MT_H -> Bool(outerDataBits >= 16), - MT_HU -> Bool(outerDataBits >= 16), - MT_W -> Bool(outerDataBits >= 32), - MT_WU -> Bool(outerDataBits >= 32), - MT_D -> Bool(outerDataBits >= 64), - MT_Q -> Bool(false))) - - assert(!io.in.acquire.valid || !smallget || read_size_ok, - "Can't perform Get wider than outer width") - - val outerConfig = p.alterPartial({ case TLId => outerTLId }) - val innerConfig = p.alterPartial({ case TLId => innerTLId }) - - val get_block_acquire = GetBlock( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - alloc = iacq.allocate())(outerConfig) - - val put_block_acquire = PutBlock( - client_xact_id = acq_client_id, - addr_block = acq_addr_block, - addr_beat = if (factor > 1) - Cat(acq_addr_beat, oacq_ctr.value) - else acq_addr_beat, - data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) - - val get_acquire = Get( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = outer_beat_addr, - addr_byte = outer_byte_addr, - operand_size = iacq.op_size(), - alloc = iacq.allocate())(outerConfig) - - val put_acquire = Put( - client_xact_id = iacq.client_xact_id, - addr_block = iacq.addr_block, - addr_beat = smallput_beat, - data = smallput_data, - wmask = Some(smallput_wmask))(outerConfig) - - val sending_put = Reg(init = Bool(false)) - - val pass_valid = io.in.acquire.valid && !stretch - - io.out.acquire.bits := MuxCase(Wire(io.out.acquire.bits, init=iacq), Seq( - (sending_put, put_block_acquire), - (shrink, get_block_acquire), - (smallput, put_acquire), - (smallget, get_acquire))) - io.out.acquire.valid := sending_put || pass_valid - io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) - - when (io.in.acquire.fire() && stretch) { - acq_data_buffer := iacq.data - acq_wmask_buffer := iacq.wmask() - acq_client_id := iacq.client_xact_id - acq_addr_block := iacq.addr_block - acq_addr_beat := iacq.addr_beat - sending_put := Bool(true) - } - - when (sending_put && io.out.acquire.ready) { - acq_data_buffer := acq_data_buffer >> outerDataBits - acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits - when (oacq_ctr.inc()) { sending_put := Bool(false) } - } - - val ognt_block = ognt.hasMultibeatData() - val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) - val gnt_client_id = Reg(ognt.client_xact_id) - val gnt_manager_id = Reg(ognt.manager_xact_id) - - val ignt_ctr = Counter(innerDataBeats) - val ognt_ctr = Counter(factor) - val sending_get = Reg(init = Bool(false)) - - val get_block_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBlockType, - client_xact_id = gnt_client_id, - manager_xact_id = gnt_manager_id, - addr_beat = ignt_ctr.value, - data = gnt_data_buffer.toBits)(innerConfig) - - val smallget_grant = ognt.g_type === Grant.getDataBeatType - - val get_grant = Grant( - is_builtin_type = Bool(true), - g_type = Grant.getDataBeatType, - client_xact_id = ognt.client_xact_id, - manager_xact_id = ognt.manager_xact_id, - addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), - data = Fill(factor, ognt.data))(innerConfig) - - io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) - io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) - - io.in.grant.bits := MuxCase(Wire(io.in.grant.bits, init=ognt), Seq( - sending_get -> get_block_grant, - smallget_grant -> get_grant)) - - when (io.out.grant.valid && ognt_block && !sending_get) { - gnt_data_buffer(ognt_ctr.value) := ognt.data - when (ognt_ctr.inc()) { - gnt_client_id := ognt.client_xact_id - gnt_manager_id := ognt.manager_xact_id - sending_get := Bool(true) - } - } - - when (io.in.grant.ready && sending_get) { - ignt_ctr.inc() - sending_get := Bool(false) - } -} - -class MMIOTileLinkManagerData(implicit p: Parameters) - extends TLBundle()(p) - with HasClientId - with HasClientTransactionId - -class MMIOTileLinkManager(implicit p: Parameters) - extends CoherenceAgentModule()(p) { - val io = new ManagerTLIO - - // MMIO requests should never need probe or release - io.inner.probe.valid := Bool(false) - io.inner.release.ready := Bool(false) - - val multibeat_fire = io.outer.acquire.fire() && io.oacq().hasMultibeatData() - val multibeat_start = multibeat_fire && io.oacq().addr_beat === UInt(0) - val multibeat_end = multibeat_fire && io.oacq().addr_beat === UInt(outerDataBeats - 1) - - // Acquire and Grant are basically passthru, - // except client_id and client_xact_id need to be converted. - // Associate the inner client_id and client_xact_id - // with the outer client_xact_id. - val xact_pending = Reg(init = UInt(0, maxManagerXacts)) - val xact_id_sel = PriorityEncoder(~xact_pending) - val xact_id_reg = RegEnable(xact_id_sel, multibeat_start) - val xact_multibeat = Reg(init = Bool(false)) - val outer_xact_id = Mux(xact_multibeat, xact_id_reg, xact_id_sel) - val xact_free = !xact_pending.andR - val xact_buffer = Reg(Vec(maxManagerXacts, new MMIOTileLinkManagerData)) - - io.inner.acquire.ready := io.outer.acquire.ready && xact_free - io.outer.acquire.valid := io.inner.acquire.valid && xact_free - io.outer.acquire.bits := io.inner.acquire.bits - io.outer.acquire.bits.client_xact_id := outer_xact_id - - def isLastBeat[T <: TileLinkChannel with HasTileLinkBeatId](in: T): Bool = - !in.hasMultibeatData() || in.addr_beat === UInt(outerDataBeats - 1) - - def addPendingBitOnAcq[T <: AcquireMetadata](in: DecoupledIO[T]): UInt = - Mux(in.fire() && isLastBeat(in.bits), UIntToOH(in.bits.client_xact_id), UInt(0)) - - def clearPendingBitOnGnt[T <: GrantMetadata](in: DecoupledIO[T]): UInt = - ~Mux(in.fire() && isLastBeat(in.bits) && !in.bits.requiresAck(), - UIntToOH(in.bits.manager_xact_id), UInt(0)) - - def clearPendingBitOnFin(in: DecoupledIO[Finish]): UInt = - ~Mux(in.fire(), UIntToOH(in.bits.manager_xact_id), UInt(0)) - - xact_pending := (xact_pending | addPendingBitOnAcq(io.outer.acquire)) & - clearPendingBitOnFin(io.inner.finish) & - clearPendingBitOnGnt(io.inner.grant) - - when (io.outer.acquire.fire() && isLastBeat(io.outer.acquire.bits)) { - xact_buffer(outer_xact_id) := io.iacq() - } - - when (multibeat_start) { xact_multibeat := Bool(true) } - when (multibeat_end) { xact_multibeat := Bool(false) } - - val gnt_xact = xact_buffer(io.ognt().client_xact_id) - io.outer.grant.ready := io.inner.grant.ready - io.inner.grant.valid := io.outer.grant.valid - io.inner.grant.bits := io.outer.grant.bits - io.inner.grant.bits.client_id := gnt_xact.client_id - io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id - io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id - io.inner.finish.ready := Bool(true) -} diff --git a/uncore/src/main/scala/ahb.scala b/uncore/src/main/scala/converters/Ahb.scala similarity index 99% rename from uncore/src/main/scala/ahb.scala rename to uncore/src/main/scala/converters/Ahb.scala index adee3f8a..0fca9517 100644 --- a/uncore/src/main/scala/ahb.scala +++ b/uncore/src/main/scala/converters/Ahb.scala @@ -1,7 +1,10 @@ -package uncore +package uncore.converters import Chisel._ import junctions._ +import uncore.tilelink._ +import uncore.util._ +import uncore.constants._ import cde.{Parameters, Field} import HastiConstants._ diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala new file mode 100644 index 00000000..006485ca --- /dev/null +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -0,0 +1,353 @@ +package uncore.converters + +import Chisel._ +import junctions._ +import uncore.tilelink._ +import uncore.constants._ +import cde.Parameters + +class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module + with HasTileLinkParameters with HasNastiParameters { + val io = new Bundle { + val req = new Bundle { + val valid = Bool(INPUT) + val ready = Bool(OUTPUT) + val tl_id = UInt(INPUT, tlClientXactIdBits) + val nasti_id = UInt(OUTPUT, nastiXIdBits) + } + val resp = new Bundle { + val valid = Bool(INPUT) + val matches = Bool(OUTPUT) + val nasti_id = UInt(INPUT, nastiXIdBits) + val tl_id = UInt(OUTPUT, tlClientXactIdBits) + } + } + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + + if (tlClientXactIdBits <= nastiXIdBits) { + io.req.ready := Bool(true) + io.req.nasti_id := io.req.tl_id + io.resp.matches := Bool(true) + io.resp.tl_id := io.resp.nasti_id + } else if (nastiXIdBits <= 2) { + val nQueues = 1 << nastiXIdBits + val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) + io.req.ready := Bool(false) + io.resp.matches := Bool(false) + io.resp.tl_id := UInt(0) + io.req.nasti_id := req_nasti_id + for (i <- 0 until nQueues) { + val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) + queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) + queue.io.enq.bits := io.req.tl_id + when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } + + queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) + when (io.resp.nasti_id === UInt(i)) { + io.resp.matches := queue.io.deq.valid + io.resp.tl_id := queue.io.deq.bits + } + } + } else { + val maxNastiId = 1 << nastiXIdBits + val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) + val roq = Module(new ReorderQueue( + UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) + roq.io.enq.valid := io.req.valid + roq.io.enq.bits.data := io.req.tl_id + roq.io.enq.bits.tag := req_nasti_id + io.req.ready := roq.io.enq.ready + io.req.nasti_id := req_nasti_id + roq.io.deq.valid := io.resp.valid + roq.io.deq.tag := io.resp.nasti_id + io.resp.tl_id := roq.io.deq.data + io.resp.matches := roq.io.deq.matches + } +} + +class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { + val addr_beat = UInt(width = tlBeatAddrBits) + val subblock = Bool() +} + +class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val tl = new ClientUncachedTileLinkIO().flip + val nasti = new NastiIO + } + + private def opSizeToXSize(ops: UInt) = MuxLookup(ops, UInt("b111"), Seq( + MT_B -> UInt(0), + MT_BU -> UInt(0), + MT_H -> UInt(1), + MT_HU -> UInt(1), + MT_W -> UInt(2), + MT_WU -> UInt(2), + MT_D -> UInt(3), + MT_Q -> UInt(log2Up(tlDataBytes)))) + + val dataBits = tlDataBits*tlDataBeats + require(tlDataBits == nastiXDataBits, "Data sizes between LLC and MC don't agree") // TODO: remove this restriction + require(tlDataBeats < (1 << nastiXLenBits), "Can't have that many beats") + + val has_data = io.tl.acquire.bits.hasData() + + val is_subblock = io.tl.acquire.bits.isSubBlockType() + val is_multibeat = io.tl.acquire.bits.hasMultibeatData() + val (tl_cnt_out, tl_wrap_out) = Counter( + io.tl.acquire.fire() && is_multibeat, tlDataBeats) + + val get_valid = io.tl.acquire.valid && !has_data + val put_valid = io.tl.acquire.valid && has_data + + val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + + // Reorder queue saves extra information needed to send correct + // grant back to TL client + val roq = Module(new ReorderQueue( + new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) + + val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + + val get_id_ready = get_id_mapper.io.req.ready + val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) + val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask + + // For Get/GetBlock, make sure Reorder queue can accept new entry + val get_helper = DecoupledHelper( + get_valid, + roq.io.enq.ready, + io.nasti.ar.ready, + get_id_ready) + + val w_inflight = Reg(init = Bool(false)) + + // For Put/PutBlock, make sure aw and w channel are both ready before + // we send the first beat + val aw_ready = w_inflight || io.nasti.aw.ready + val put_helper = DecoupledHelper( + put_valid, + aw_ready, + io.nasti.w.ready, + put_id_ready) + + val (nasti_cnt_out, nasti_wrap_out) = Counter( + io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) + + roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) + roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat + roq.io.enq.bits.data.subblock := is_subblock + roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) + roq.io.deq.tag := io.nasti.r.bits.id + + get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) + get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last + get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id + + put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) + put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + put_id_mapper.io.resp.valid := io.nasti.b.fire() + put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id + + // Decompose outgoing TL Acquires into Nasti address and data channels + io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) + io.nasti.ar.bits := NastiReadAddressChannel( + id = get_id_mapper.io.req.nasti_id, + addr = io.tl.acquire.bits.full_addr(), + size = Mux(is_subblock, + opSizeToXSize(io.tl.acquire.bits.op_size()), + UInt(log2Ceil(tlDataBytes))), + len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) + io.nasti.aw.bits := NastiWriteAddressChannel( + id = put_id_mapper.io.req.nasti_id, + addr = io.tl.acquire.bits.full_addr(), + size = UInt(log2Ceil(tlDataBytes)), + len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) + + io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) + io.nasti.w.bits := NastiWriteDataChannel( + id = put_id_mapper.io.req.nasti_id, + data = io.tl.acquire.bits.data, + strb = io.tl.acquire.bits.wmask(), + last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) + + io.tl.acquire.ready := Mux(has_data, + put_helper.fire(put_valid), + get_helper.fire(get_valid)) + + when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { + w_inflight := Bool(true) + } + + when (w_inflight) { + when (tl_wrap_out) { w_inflight := Bool(false) } + } + + // Aggregate incoming NASTI responses into TL Grants + val (tl_cnt_in, tl_wrap_in) = Counter( + io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + io.tl.grant <> gnt_arb.io.out + + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).bits := Grant( + is_builtin_type = Bool(true), + g_type = Mux(roq.io.deq.data.subblock, + Grant.getDataBeatType, Grant.getDataBlockType), + client_xact_id = get_id_mapper.io.resp.tl_id, + manager_xact_id = UInt(0), + addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), + data = io.nasti.r.bits.data) + + assert(!roq.io.deq.valid || roq.io.deq.matches, + "TL -> NASTI converter ReorderQueue: NASTI tag error") + assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, + "TL -> NASTI ID Mapper: NASTI tag error") + + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).bits := Grant( + is_builtin_type = Bool(true), + g_type = Grant.putAckType, + client_xact_id = put_id_mapper.io.resp.tl_id, + manager_xact_id = UInt(0), + addr_beat = UInt(0), + data = Bits(0)) + assert(!gnt_arb.io.in(1).valid || put_id_mapper.io.resp.matches, "NASTI tag error") + + assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") + assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") +} + +class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) + with HasNastiParameters { + val io = new Bundle { + val nasti = (new NastiIO).flip + val tl = new ClientUncachedTileLinkIO + } + + val (s_idle :: s_put :: Nil) = Enum(Bits(), 2) + val state = Reg(init = s_idle) + + private val blockOffset = tlByteAddrBits + tlBeatAddrBits + + val aw_req = Reg(new NastiWriteAddressChannel) + + def is_singlebeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(0) + + def is_multibeat(chan: NastiAddressChannel): Bool = + chan.len === UInt(tlDataBeats - 1) && chan.size === UInt(log2Up(tlDataBytes)) + + def nasti_addr_block(chan: NastiAddressChannel): UInt = + chan.addr(nastiXAddrBits - 1, blockOffset) + + def nasti_addr_beat(chan: NastiAddressChannel): UInt = + chan.addr(blockOffset - 1, tlByteAddrBits) + + def nasti_addr_byte(chan: NastiAddressChannel): UInt = + chan.addr(tlByteAddrBits - 1, 0) + + def nasti_operand_size(chan: NastiAddressChannel): UInt = + MuxLookup(chan.size, MT_Q, Seq( + UInt(0) -> MT_BU, + UInt(1) -> MT_HU, + UInt(2) -> MT_WU, + UInt(3) -> MT_D)) + + def size_mask(size: UInt): UInt = + (UInt(1) << (UInt(1) << size)) - UInt(1) + + def nasti_wmask(aw: NastiWriteAddressChannel, w: NastiWriteDataChannel): UInt = { + val base = w.strb & size_mask(aw.size) + val addr_byte = nasti_addr_byte(aw) + w.strb & (size_mask(aw.size) << addr_byte) + } + + def tl_last(gnt: GrantMetadata): Bool = + !gnt.hasMultibeatData() || gnt.addr_beat === UInt(tlDataBeats - 1) + + def tl_b_grant(gnt: GrantMetadata): Bool = + gnt.g_type === Grant.putAckType + + assert(!io.nasti.ar.valid || + is_singlebeat(io.nasti.ar.bits) || is_multibeat(io.nasti.ar.bits), + "NASTI read transaction cannot convert to TileLInk") + + assert(!io.nasti.aw.valid || + is_singlebeat(io.nasti.aw.bits) || is_multibeat(io.nasti.aw.bits), + "NASTI write transaction cannot convert to TileLInk") + + val put_count = Reg(init = UInt(0, tlBeatAddrBits)) + + when (io.nasti.aw.fire()) { + aw_req := io.nasti.aw.bits + state := s_put + } + + when (io.nasti.w.fire()) { + put_count := put_count + UInt(1) + when (io.nasti.w.bits.last) { + put_count := UInt(0) + state := s_idle + } + } + + val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), + GetBlock( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits)), + Get( + client_xact_id = io.nasti.ar.bits.id, + addr_block = nasti_addr_block(io.nasti.ar.bits), + addr_beat = nasti_addr_beat(io.nasti.ar.bits), + addr_byte = nasti_addr_byte(io.nasti.ar.bits), + operand_size = nasti_operand_size(io.nasti.ar.bits), + alloc = Bool(false))) + + val put_acquire = Mux(is_multibeat(aw_req), + PutBlock( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = put_count, + data = io.nasti.w.bits.data, + wmask = io.nasti.w.bits.strb), + Put( + client_xact_id = aw_req.id, + addr_block = nasti_addr_block(aw_req), + addr_beat = nasti_addr_beat(aw_req), + data = io.nasti.w.bits.data, + wmask = Some(nasti_wmask(aw_req, io.nasti.w.bits)))) + + io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) + io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || + (state === s_put && io.nasti.w.valid) + io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) + io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) + io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) + + val nXacts = tlMaxClientXacts * tlMaxClientsPerPort + + io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) + io.nasti.b.bits := NastiWriteResponseChannel( + id = io.tl.grant.bits.client_xact_id) + + io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) + io.nasti.r.bits := NastiReadDataChannel( + id = io.tl.grant.bits.client_xact_id, + data = io.tl.grant.bits.data, + last = tl_last(io.tl.grant.bits)) + + io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), + io.nasti.b.ready, io.nasti.r.ready) +} + + diff --git a/uncore/src/main/scala/smi.scala b/uncore/src/main/scala/converters/Smi.scala similarity index 94% rename from uncore/src/main/scala/smi.scala rename to uncore/src/main/scala/converters/Smi.scala index dc8b158e..6ec47950 100644 --- a/uncore/src/main/scala/smi.scala +++ b/uncore/src/main/scala/converters/Smi.scala @@ -1,9 +1,10 @@ // See LICENSE for details -package uncore +package uncore.converters import Chisel._ import junctions._ +import uncore.tilelink._ import cde.Parameters /** Convert TileLink protocol to Smi protocol */ diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala new file mode 100644 index 00000000..4899f05d --- /dev/null +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -0,0 +1,538 @@ +package uncore.converters + +import Chisel._ +import junctions._ +import uncore.tilelink._ +import uncore.util._ +import uncore.constants._ +import cde.Parameters + +/** Utilities for safely wrapping a *UncachedTileLink by pinning probe.ready and release.valid low */ +object TileLinkIOWrapper { + def apply(tl: ClientUncachedTileLinkIO)(implicit p: Parameters): ClientTileLinkIO = { + val conv = Module(new ClientTileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: UncachedTileLinkIO)(implicit p: Parameters): TileLinkIO = { + val conv = Module(new TileLinkIOWrapper) + conv.io.in <> tl + conv.io.out + } + def apply(tl: ClientTileLinkIO): ClientTileLinkIO = tl + def apply(tl: TileLinkIO): TileLinkIO = tl +} + +class TileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new UncachedTileLinkIO().flip + val out = new TileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.finish <> io.in.finish + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOWrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientTileLinkIO + } + io.out.acquire <> io.in.acquire + io.in.grant <> io.out.grant + io.out.probe.ready := Bool(true) + io.out.release.valid := Bool(false) +} + +class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientTileLinkIO().flip + val out = new ClientUncachedTileLinkIO + } + + val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, + Some((acq: Acquire) => acq.hasMultibeatData()))) + + val acqRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val relRoq = Module(new ReorderQueue( + Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + + val iacq = io.in.acquire.bits + val irel = io.in.release.bits + val ognt = io.out.grant.bits + + val acq_roq_enq = iacq.first() + val rel_roq_enq = irel.first() + + val acq_roq_ready = !acq_roq_enq || acqRoq.io.enq.ready + val rel_roq_ready = !rel_roq_enq || relRoq.io.enq.ready + + val acq_helper = DecoupledHelper( + io.in.acquire.valid, + acq_roq_ready, + acqArb.io.in(0).ready) + + val rel_helper = DecoupledHelper( + io.in.release.valid, + rel_roq_ready, + acqArb.io.in(1).ready) + + acqRoq.io.enq.valid := acq_helper.fire(acq_roq_ready, acq_roq_enq) + acqRoq.io.enq.bits.data := iacq.isBuiltInType() + acqRoq.io.enq.bits.tag := iacq.client_xact_id + + acqArb.io.in(0).valid := acq_helper.fire(acqArb.io.in(0).ready) + acqArb.io.in(0).bits := Acquire( + is_builtin_type = Bool(true), + a_type = Mux(iacq.isBuiltInType(), + iacq.a_type, Acquire.getBlockType), + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = iacq.addr_beat, + data = iacq.data, + union = Mux(iacq.isBuiltInType(), + iacq.union, Cat(MT_Q, M_XRD, Bool(true)))) + io.in.acquire.ready := acq_helper.fire(io.in.acquire.valid) + + relRoq.io.enq.valid := rel_helper.fire(rel_roq_ready, rel_roq_enq) + relRoq.io.enq.bits.data := irel.isVoluntary() + relRoq.io.enq.bits.tag := irel.client_xact_id + + acqArb.io.in(1).valid := rel_helper.fire(acqArb.io.in(1).ready) + acqArb.io.in(1).bits := PutBlock( + client_xact_id = irel.client_xact_id, + addr_block = irel.addr_block, + addr_beat = irel.addr_beat, + data = irel.data, + wmask = Acquire.fullWriteMask) + io.in.release.ready := rel_helper.fire(io.in.release.valid) + + io.out.acquire <> acqArb.io.out + + val grant_deq_roq = io.out.grant.fire() && ognt.last() + + acqRoq.io.deq.valid := acqRoq.io.deq.matches && grant_deq_roq + acqRoq.io.deq.tag := ognt.client_xact_id + + relRoq.io.deq.valid := !acqRoq.io.deq.matches && grant_deq_roq + relRoq.io.deq.tag := ognt.client_xact_id + + assert(!grant_deq_roq || acqRoq.io.deq.matches || relRoq.io.deq.matches, + "TileLink Unwrapper: client_xact_id mismatch") + + val gnt_builtin = acqRoq.io.deq.data + val gnt_voluntary = relRoq.io.deq.data + + val acq_grant = Grant( + is_builtin_type = gnt_builtin, + g_type = Mux(gnt_builtin, ognt.g_type, tlCoh.getExclusiveGrantType), + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + assert(!io.in.release.valid || io.in.release.bits.isVoluntary(), "Unwrapper can only process voluntary releases.") + val rel_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.voluntaryAckType, // We should only every be working with voluntary releases + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := io.out.grant.valid + io.in.grant.bits := Mux(acqRoq.io.deq.matches, acq_grant, rel_grant) + io.out.grant.ready := io.in.grant.ready + + io.in.probe.valid := Bool(false) +} + +object TileLinkWidthAdapter { + def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { + require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) + + if (out.tlDataBits > in.tlDataBits) { + val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + widener.io.in <> in + out <> widener.io.out + } else if (out.tlDataBits < in.tlDataBits) { + val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) + narrower.io.in <> in + out <> narrower.io.out + } else { + out <> in + } + } +} + +class TileLinkIOWidener(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val paddrBits = p(PAddrBits) + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val innerMaxXacts = innerParams.maxClientXacts * innerParams.maxClientsPerPort + val innerXactIdBits = log2Up(innerMaxXacts) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerClientIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + val outerManagerIdBits = log2Up(outerParams.maxManagerXacts) + val outerBlockAddrBits = paddrBits - outerBlockOffset + + require(outerDataBeats <= innerDataBeats) + require(outerDataBits >= innerDataBits) + require(outerDataBits % innerDataBits == 0) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = innerDataBeats / outerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + val ignt = io.in.grant.bits + + val shrink = iacq.a_type === Acquire.putBlockType + val stretch = ognt.g_type === Grant.getDataBlockType + val smallget = iacq.a_type === Acquire.getType + val smallput = iacq.a_type === Acquire.putType + val smallgnt = ognt.g_type === Grant.getDataBeatType + + val sending_put = Reg(init = Bool(false)) + val collecting = Reg(init = Bool(false)) + val put_block = Reg(UInt(width = outerBlockAddrBits)) + val put_id = Reg(UInt(width = outerClientIdBits)) + val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) + val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) + val put_allocate = Reg(Bool()) + val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) + val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) + + val in_addr = iacq.full_addr() + val out_addr_block = in_addr(paddrBits - 1, outerBlockOffset) + val out_addr_beat = in_addr(outerBlockOffset - 1, outerByteAddrBits) + val out_addr_byte = in_addr(outerByteAddrBits - 1, 0) + + val switch_addr = in_addr(outerByteAddrBits - 1, innerByteAddrBits) + val smallget_switch = Reg(Vec(innerMaxXacts, switch_addr)) + + def align_data(addr: UInt, data: UInt): UInt = + data << Cat(addr, UInt(0, log2Up(innerDataBits))) + + def align_wmask(addr: UInt, wmask: UInt): UInt = + wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + addr_byte = out_addr_byte, + operand_size = iacq.op_size(), + alloc = iacq.allocate()) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + alloc = iacq.allocate()) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = out_addr_block, + addr_beat = out_addr_beat, + data = align_data(switch_addr, iacq.data), + wmask = Some(align_wmask(switch_addr, iacq.wmask())), + alloc = iacq.allocate()) + + val put_block_acquire = PutBlock( + client_xact_id = put_id, + addr_block = put_block, + addr_beat = put_beat, + data = put_data.toBits, + wmask = put_wmask.toBits) + + io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) + io.out.acquire.bits := MuxCase(get_block_acquire, Seq( + sending_put -> put_block_acquire, + smallget -> get_acquire, + smallput -> put_acquire)) + io.in.acquire.ready := !sending_put && (shrink || io.out.acquire.ready) + + when (io.in.acquire.fire() && shrink) { + when (!collecting) { + put_block := out_addr_block + put_id := iacq.client_xact_id + put_allocate := iacq.allocate() + collecting := Bool(true) + } + put_data(recv_idx) := iacq.data + put_wmask(recv_idx) := iacq.wmask() + } + + when (io.in.acquire.fire() && smallget) { + smallget_switch(iacq.client_xact_id) := switch_addr + } + + when (recv_done) { sending_put := Bool(true) } + when (sending_put && io.out.acquire.ready) { sending_put := Bool(false) } + when (put_done) { collecting := Bool(false) } + + val returning_data = Reg(init = Bool(false)) + val (send_idx, send_done) = Counter( + io.in.grant.ready && returning_data, factor) + + val gnt_beat = Reg(UInt(width = outerBeatAddrBits)) + val gnt_client_id = Reg(UInt(width = outerClientIdBits)) + val gnt_manager_id = Reg(UInt(width = outerManagerIdBits)) + val gnt_data = Reg(UInt(width = outerDataBits)) + + when (io.out.grant.fire() && stretch) { + gnt_data := ognt.data + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + gnt_beat := ognt.addr_beat + returning_data := Bool(true) + } + + when (send_done) { returning_data := Bool(false) } + + def select_data(data: UInt, sel: UInt): UInt = + data >> (sel << log2Up(innerDataBits)) + + val gnt_switch = smallget_switch(ognt.client_xact_id) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = Cat(gnt_beat, send_idx), + data = select_data(gnt_data, send_idx)) + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = Cat(ognt.addr_beat, gnt_switch), + data = select_data(ognt.data, gnt_switch)) + + val default_grant = Grant( + is_builtin_type = Bool(true), + g_type = ognt.g_type, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat, + data = ognt.data) + + io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) + io.in.grant.bits := MuxCase(default_grant, Seq( + returning_data -> get_block_grant, + smallgnt -> get_grant)) + io.out.grant.ready := !returning_data && (stretch || io.in.grant.ready) +} + +class TileLinkIONarrower(innerTLId: String, outerTLId: String) + (implicit p: Parameters) extends TLModule()(p) { + + val innerParams = p(TLKey(innerTLId)) + val outerParams = p(TLKey(outerTLId)) + val innerDataBeats = innerParams.dataBeats + val innerDataBits = innerParams.dataBitsPerBeat + val innerWriteMaskBits = innerParams.writeMaskBits + val innerByteAddrBits = log2Up(innerWriteMaskBits) + val outerDataBeats = outerParams.dataBeats + val outerDataBits = outerParams.dataBitsPerBeat + val outerWriteMaskBits = outerParams.writeMaskBits + val outerByteAddrBits = log2Up(outerWriteMaskBits) + val outerBeatAddrBits = log2Up(outerDataBeats) + val outerBlockOffset = outerBeatAddrBits + outerByteAddrBits + val outerMaxClients = outerParams.maxClientsPerPort + val outerIdBits = log2Up(outerParams.maxClientXacts * outerMaxClients) + + require(outerDataBeats > innerDataBeats) + require(outerDataBeats % innerDataBeats == 0) + require(outerDataBits < innerDataBits) + require(outerDataBits * outerDataBeats == innerDataBits * innerDataBeats) + + val factor = outerDataBeats / innerDataBeats + + val io = new Bundle { + val in = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => innerTLId})).flip + val out = new ClientUncachedTileLinkIO()(p.alterPartial({case TLId => outerTLId})) + } + + val iacq = io.in.acquire.bits + val ognt = io.out.grant.bits + + val stretch = iacq.a_type === Acquire.putBlockType + val shrink = iacq.a_type === Acquire.getBlockType + val smallput = iacq.a_type === Acquire.putType + val smallget = iacq.a_type === Acquire.getType + + val acq_data_buffer = Reg(UInt(width = innerDataBits)) + val acq_wmask_buffer = Reg(UInt(width = innerWriteMaskBits)) + val acq_client_id = Reg(iacq.client_xact_id) + val acq_addr_block = Reg(iacq.addr_block) + val acq_addr_beat = Reg(iacq.addr_beat) + val oacq_ctr = Counter(factor) + + val outer_beat_addr = iacq.full_addr()(outerBlockOffset - 1, outerByteAddrBits) + val outer_byte_addr = iacq.full_addr()(outerByteAddrBits - 1, 0) + + val mask_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerWriteMaskBits + val msb = (i + 1) * outerWriteMaskBits - 1 + iacq.wmask()(msb, lsb) + } + + val data_chunks = Vec.tabulate(factor) { i => + val lsb = i * outerDataBits + val msb = (i + 1) * outerDataBits - 1 + iacq.data(msb, lsb) + } + + val beat_sel = Cat(mask_chunks.map(mask => mask.orR).reverse) + + val smallput_data = Mux1H(beat_sel, data_chunks) + val smallput_wmask = Mux1H(beat_sel, mask_chunks) + val smallput_beat = Cat(iacq.addr_beat, PriorityEncoder(beat_sel)) + + assert(!io.in.acquire.valid || !smallput || PopCount(beat_sel) <= UInt(1), + "Can't perform Put wider than outer width") + + val read_size_ok = MuxLookup(iacq.op_size(), Bool(false), Seq( + MT_B -> Bool(true), + MT_BU -> Bool(true), + MT_H -> Bool(outerDataBits >= 16), + MT_HU -> Bool(outerDataBits >= 16), + MT_W -> Bool(outerDataBits >= 32), + MT_WU -> Bool(outerDataBits >= 32), + MT_D -> Bool(outerDataBits >= 64), + MT_Q -> Bool(false))) + + assert(!io.in.acquire.valid || !smallget || read_size_ok, + "Can't perform Get wider than outer width") + + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + + val get_block_acquire = GetBlock( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + alloc = iacq.allocate())(outerConfig) + + val put_block_acquire = PutBlock( + client_xact_id = acq_client_id, + addr_block = acq_addr_block, + addr_beat = if (factor > 1) + Cat(acq_addr_beat, oacq_ctr.value) + else acq_addr_beat, + data = acq_data_buffer(outerDataBits - 1, 0), + wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + + val get_acquire = Get( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = outer_beat_addr, + addr_byte = outer_byte_addr, + operand_size = iacq.op_size(), + alloc = iacq.allocate())(outerConfig) + + val put_acquire = Put( + client_xact_id = iacq.client_xact_id, + addr_block = iacq.addr_block, + addr_beat = smallput_beat, + data = smallput_data, + wmask = Some(smallput_wmask))(outerConfig) + + val sending_put = Reg(init = Bool(false)) + + val pass_valid = io.in.acquire.valid && !stretch + + io.out.acquire.bits := MuxCase(Wire(io.out.acquire.bits, init=iacq), Seq( + (sending_put, put_block_acquire), + (shrink, get_block_acquire), + (smallput, put_acquire), + (smallget, get_acquire))) + io.out.acquire.valid := sending_put || pass_valid + io.in.acquire.ready := !sending_put && (stretch || io.out.acquire.ready) + + when (io.in.acquire.fire() && stretch) { + acq_data_buffer := iacq.data + acq_wmask_buffer := iacq.wmask() + acq_client_id := iacq.client_xact_id + acq_addr_block := iacq.addr_block + acq_addr_beat := iacq.addr_beat + sending_put := Bool(true) + } + + when (sending_put && io.out.acquire.ready) { + acq_data_buffer := acq_data_buffer >> outerDataBits + acq_wmask_buffer := acq_wmask_buffer >> outerWriteMaskBits + when (oacq_ctr.inc()) { sending_put := Bool(false) } + } + + val ognt_block = ognt.hasMultibeatData() + val gnt_data_buffer = Reg(Vec(factor, UInt(width = outerDataBits))) + val gnt_client_id = Reg(ognt.client_xact_id) + val gnt_manager_id = Reg(ognt.manager_xact_id) + + val ignt_ctr = Counter(innerDataBeats) + val ognt_ctr = Counter(factor) + val sending_get = Reg(init = Bool(false)) + + val get_block_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBlockType, + client_xact_id = gnt_client_id, + manager_xact_id = gnt_manager_id, + addr_beat = ignt_ctr.value, + data = gnt_data_buffer.toBits)(innerConfig) + + val smallget_grant = ognt.g_type === Grant.getDataBeatType + + val get_grant = Grant( + is_builtin_type = Bool(true), + g_type = Grant.getDataBeatType, + client_xact_id = ognt.client_xact_id, + manager_xact_id = ognt.manager_xact_id, + addr_beat = ognt.addr_beat >> UInt(log2Up(factor)), + data = Fill(factor, ognt.data))(innerConfig) + + io.in.grant.valid := sending_get || (io.out.grant.valid && !ognt_block) + io.out.grant.ready := !sending_get && (ognt_block || io.in.grant.ready) + + io.in.grant.bits := MuxCase(Wire(io.in.grant.bits, init=ognt), Seq( + sending_get -> get_block_grant, + smallget_grant -> get_grant)) + + when (io.out.grant.valid && ognt_block && !sending_get) { + gnt_data_buffer(ognt_ctr.value) := ognt.data + when (ognt_ctr.inc()) { + gnt_client_id := ognt.client_xact_id + gnt_manager_id := ognt.manager_xact_id + sending_get := Bool(true) + } + } + + when (io.in.grant.ready && sending_get) { + ignt_ctr.inc() + sending_get := Bool(false) + } +} diff --git a/uncore/src/main/scala/bram.scala b/uncore/src/main/scala/devices/Bram.scala similarity index 98% rename from uncore/src/main/scala/bram.scala rename to uncore/src/main/scala/devices/Bram.scala index 379dfa0d..1be27216 100644 --- a/uncore/src/main/scala/bram.scala +++ b/uncore/src/main/scala/devices/Bram.scala @@ -1,8 +1,9 @@ -package uncore +package uncore.devices import Chisel._ import cde.{Parameters, Field} import junctions._ +import uncore.tilelink._ import HastiConstants._ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module diff --git a/uncore/src/main/scala/debug.scala b/uncore/src/main/scala/devices/Debug.scala similarity index 99% rename from uncore/src/main/scala/debug.scala rename to uncore/src/main/scala/devices/Debug.scala index a065d442..9513e171 100644 --- a/uncore/src/main/scala/debug.scala +++ b/uncore/src/main/scala/devices/Debug.scala @@ -1,9 +1,9 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ - +import uncore.tilelink._ import junctions._ import cde.{Parameters, Config, Field} diff --git a/uncore/src/main/scala/dma.scala b/uncore/src/main/scala/devices/Dma.scala similarity index 99% rename from uncore/src/main/scala/dma.scala rename to uncore/src/main/scala/devices/Dma.scala index fb157e07..eadff9d7 100644 --- a/uncore/src/main/scala/dma.scala +++ b/uncore/src/main/scala/devices/Dma.scala @@ -1,9 +1,10 @@ -package uncore +package uncore.devices import Chisel._ import cde.{Parameters, Field} import junctions._ import junctions.NastiConstants._ +import uncore.tilelink._ case object NDmaTransactors extends Field[Int] case object NDmaXacts extends Field[Int] diff --git a/uncore/src/main/scala/plic.scala b/uncore/src/main/scala/devices/Plic.scala similarity index 99% rename from uncore/src/main/scala/plic.scala rename to uncore/src/main/scala/devices/Plic.scala index 7635e3aa..776c581b 100644 --- a/uncore/src/main/scala/plic.scala +++ b/uncore/src/main/scala/devices/Plic.scala @@ -1,11 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ import Chisel.ImplicitConversions._ import junctions._ +import uncore.tilelink._ import cde.Parameters class GatewayPLICIO extends Bundle { diff --git a/uncore/src/main/scala/prci.scala b/uncore/src/main/scala/devices/Prci.scala similarity index 98% rename from uncore/src/main/scala/prci.scala rename to uncore/src/main/scala/devices/Prci.scala index 7f7db90d..19497aff 100644 --- a/uncore/src/main/scala/prci.scala +++ b/uncore/src/main/scala/devices/Prci.scala @@ -1,11 +1,12 @@ // See LICENSE for license details. -package uncore +package uncore.devices import Chisel._ import Chisel.ImplicitConversions._ import junctions._ import junctions.NastiConstants._ +import uncore.tilelink._ import cde.{Parameters, Field} /** Number of tiles */ diff --git a/uncore/src/main/scala/rom.scala b/uncore/src/main/scala/devices/Rom.scala similarity index 97% rename from uncore/src/main/scala/rom.scala rename to uncore/src/main/scala/devices/Rom.scala index 1ac56e1d..0fd9dd3e 100644 --- a/uncore/src/main/scala/rom.scala +++ b/uncore/src/main/scala/devices/Rom.scala @@ -1,7 +1,9 @@ -package uncore +package uncore.devices import Chisel._ import junctions._ +import uncore.tilelink._ +import uncore.util._ import cde.{Parameters, Field} class ROMSlave(contents: Seq[Byte])(implicit val p: Parameters) extends Module diff --git a/uncore/src/main/scala/network.scala b/uncore/src/main/scala/network.scala deleted file mode 100644 index 88f1a1b6..00000000 --- a/uncore/src/main/scala/network.scala +++ /dev/null @@ -1,122 +0,0 @@ -// See LICENSE for license details. - -package uncore -import Chisel._ -import cde.{Parameters, Field} - -case object LNEndpoints extends Field[Int] -case object LNHeaderBits extends Field[Int] - -class PhysicalHeader(n: Int) extends Bundle { - val src = UInt(width = log2Up(n)) - val dst = UInt(width = log2Up(n)) -} - -class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { - val header = new PhysicalHeader(n) - val payload = dType.cloneType - override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] -} - -class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { - val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip - val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) -} - -abstract class PhysicalNetwork extends Module - -case class CrossbarConfig[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) - -abstract class AbstractCrossbar[T <: Data](conf: CrossbarConfig[T]) extends PhysicalNetwork { - val io = new BasicCrossbarIO(conf.n, conf.dType) -} - -class BasicBus[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { - val arb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) - arb.io.in <> io.in - - arb.io.out.ready := io.out(arb.io.out.bits.header.dst).ready - for ((out, i) <- io.out zipWithIndex) { - out.valid := arb.io.out.valid && arb.io.out.bits.header.dst === UInt(i) - out.bits := arb.io.out.bits - } -} - -class BasicCrossbar[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { - io.in.foreach { _.ready := Bool(false) } - - io.out.zipWithIndex.map{ case (out, i) => { - val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) - (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { - val destined = in.bits.header.dst === UInt(i) - arb.valid := in.valid && destined - arb.bits := in.bits - when (arb.ready && destined) { in.ready := Bool(true) } - }} - out <> rrarb.io.out - }} -} - -abstract class LogicalNetwork extends Module - -class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { - val src = UInt(width = p(LNHeaderBits)) - val dst = UInt(width = p(LNHeaderBits)) -} - -class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { - val header = new LogicalHeader - val payload = dType.cloneType - override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] -} - -object DecoupledLogicalNetworkIOWrapper { - def apply[T <: Data]( - in: DecoupledIO[T], - src: UInt = UInt(0), - dst: UInt = UInt(0)) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) - out.valid := in.valid - out.bits.payload := in.bits - out.bits.header.dst := dst - out.bits.header.src := src - in.ready := out.ready - out - } -} - -object DecoupledLogicalNetworkIOUnwrapper { - def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[T] = { - val out = Wire(Decoupled(in.bits.payload)) - out.valid := in.valid - out.bits := in.bits.payload - in.ready := out.ready - out - } -} - -object DefaultFromPhysicalShim { - def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { - val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) - out.bits.header := in.bits.header - out.bits.payload := in.bits.payload - out.valid := in.valid - in.ready := out.ready - out - } -} - -object DefaultToPhysicalShim { - def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) - (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { - val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) - out.bits.header := in.bits.header - out.bits.payload := in.bits.payload - out.valid := in.valid - in.ready := out.ready - out - } -} diff --git a/uncore/src/main/scala/package.scala b/uncore/src/main/scala/package.scala deleted file mode 100644 index 2c6c4a5f..00000000 --- a/uncore/src/main/scala/package.scala +++ /dev/null @@ -1,6 +0,0 @@ -// See LICENSE for license details. - -package object uncore extends uncore.constants.MemoryOpConstants -{ - implicit def toOption[A](a: A) = Option(a) -} diff --git a/uncore/src/main/scala/tilelink/Arbiters.scala b/uncore/src/main/scala/tilelink/Arbiters.scala new file mode 100644 index 00000000..ab1f05ae --- /dev/null +++ b/uncore/src/main/scala/tilelink/Arbiters.scala @@ -0,0 +1,196 @@ +package uncore.tilelink +import Chisel._ +import junctions._ +import cde.{Parameters, Field} + +/** Utility functions for constructing TileLinkIO arbiters */ +trait TileLinkArbiterLike extends HasTileLinkParameters { + // Some shorthand type variables + type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId + type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId + type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData + + val arbN: Int // The number of ports on the client side + + // These abstract funcs are filled in depending on whether the arbiter mucks with the + // outgoing client ids to track sourcing and then needs to revert them on the way back + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits + def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits + def arbIdx(in: ManagerSourcedWithId): UInt + + // The following functions are all wiring helpers for each of the different types of TileLink channels + + def hookupClientSource[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) + req.ready := arb.ready + }} + mngr <> arb.io.out + } + + def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + def hasData(m: M) = m.hasMultibeatData() + val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) + clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { + arb.valid := req.valid + arb.bits := req.bits + arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) + req.ready := arb.ready + }} + mngr <> arb.io.out + } + + def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (mngr.bits.header.dst === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + } + } + + def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], + mngr: DecoupledIO[LogicalNetworkIO[M]]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits.payload) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) + } + } + + def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( + clts: Seq[DecoupledIO[M]], + mngr: DecoupledIO[M]) { + mngr.ready := Bool(false) + for (i <- 0 until arbN) { + clts(i).valid := Bool(false) + when (arbIdx(mngr.bits) === UInt(i)) { + clts(i).valid := mngr.valid + mngr.ready := clts(i).ready + } + clts(i).bits := mngr.bits + clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) + } + } + + def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + clts.map{ _.valid := mngr.valid } + clts.map{ _.bits := mngr.bits } + mngr.ready := clts.map(_.ready).reduce(_&&_) + } + + def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { + val arb = Module(new RRArbiter(mngr.bits, arbN)) + arb.io.in <> clts + mngr <> arb.io.out + } +} + +/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ +abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec(arbN, new UncachedTileLinkIO).flip + val out = new UncachedTileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Abstract base case for any Arbiters that have cached TileLinkIOs */ +abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module + with TileLinkArbiterLike { + val io = new Bundle { + val in = Vec(arbN, new TileLinkIO).flip + val out = new TileLinkIO + } + hookupClientSource(io.in.map(_.acquire), io.out.acquire) + hookupClientSource(io.in.map(_.release), io.out.release) + hookupFinish(io.in.map(_.finish), io.out.finish) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) +} + +/** Appends the port index of the arbiter to the client_xact_id */ +trait AppendsArbiterId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = + Cat(in.client_xact_id, UInt(id, log2Up(arbN))) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = { + /* This shouldn't be necessary, but Chisel3 doesn't emit correct Verilog + * when right shifting by too many bits. See + * https://github.com/ucb-bar/firrtl/issues/69 */ + if (in.client_xact_id.getWidth > log2Up(arbN)) + in.client_xact_id >> log2Up(arbN) + else + UInt(0) + } + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt +} + +/** Uses the client_xact_id as is (assumes it has been set to port index) */ +trait PassesId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id + def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +/** Overwrites some default client_xact_id with the port idx */ +trait UsesNewId extends TileLinkArbiterLike { + def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) + def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) + def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id +} + +// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes +class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with AppendsArbiterId +class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with PassesId +class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with UsesNewId +class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with AppendsArbiterId +class TileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with PassesId +class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with UsesNewId + +/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec(arbN, new ClientUncachedTileLinkIO).flip + val out = new ClientUncachedTileLinkIO + } + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } +} + +/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ +class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { + val io = new Bundle { + val in = Vec(arbN, new ClientTileLinkIO).flip + val out = new ClientTileLinkIO + } + if (arbN > 1) { + hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) + hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) + hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) + hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) + } else { io.out <> io.in.head } +} diff --git a/uncore/src/main/scala/tilelink.scala b/uncore/src/main/scala/tilelink/Definitions.scala similarity index 77% rename from uncore/src/main/scala/tilelink.scala rename to uncore/src/main/scala/tilelink/Definitions.scala index 5599217b..1917e403 100644 --- a/uncore/src/main/scala/tilelink.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -1,11 +1,16 @@ // See LICENSE for license details. -package uncore +package uncore.tilelink import Chisel._ import junctions._ +import uncore.coherence.CoherencePolicy import scala.math.max +import uncore.constants._ import cde.{Parameters, Field} +case object CacheBlockOffsetBits extends Field[Int] +case object AmoAluOperandBits extends Field[Int] + case object TLId extends Field[String] case class TLKey(id: String) extends Field[TileLinkParameters] @@ -966,246 +971,3 @@ class ManagerTileLinkIO(implicit p: Parameters) extends TLBundle()(p) { val probe = new DecoupledIO(new ProbeToDst) val release = new DecoupledIO(new ReleaseFromSrc).flip } - -/** Struct for describing per-channel queue depths */ -case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) - -/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ -class TileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { - val io = new Bundle { - val client = new TileLinkIO().flip - val manager = new TileLinkIO - } - io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) - io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) - io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) - io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) - io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) -} - -object TileLinkEnqueuer { - def apply(in: TileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): TileLinkIO = { - val t = Module(new TileLinkEnqueuer(depths)) - t.io.client <> in - t.io.manager - } - def apply(in: TileLinkIO, depth: Int)(implicit p: Parameters): TileLinkIO = { - apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) - } -} - -class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { - val io = new Bundle { - val inner = new ClientTileLinkIO().flip - val outer = new ClientTileLinkIO - } - - io.outer.acquire <> (if(depths.acq > 0) Queue(io.inner.acquire, depths.acq) else io.inner.acquire) - io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) - io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) - io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) - io.outer.finish <> (if(depths.fin > 0) Queue(io.inner.finish, depths.fin) else io.inner.finish) -} - -object ClientTileLinkEnqueuer { - def apply(in: ClientTileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): ClientTileLinkIO = { - val t = Module(new ClientTileLinkEnqueuer(depths)) - t.io.inner <> in - t.io.outer - } - def apply(in: ClientTileLinkIO, depth: Int)(implicit p: Parameters): ClientTileLinkIO = { - apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) - } -} - -/** Utility functions for constructing TileLinkIO arbiters */ -trait TileLinkArbiterLike extends HasTileLinkParameters { - // Some shorthand type variables - type ManagerSourcedWithId = ManagerToClientChannel with HasClientTransactionId - type ClientSourcedWithId = ClientToManagerChannel with HasClientTransactionId - type ClientSourcedWithIdAndData = ClientToManagerChannel with HasClientTransactionId with HasTileLinkData - - val arbN: Int // The number of ports on the client side - - // These abstract funcs are filled in depending on whether the arbiter mucks with the - // outgoing client ids to track sourcing and then needs to revert them on the way back - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int): Bits - def managerSourcedClientXactId(in: ManagerSourcedWithId): Bits - def arbIdx(in: ManagerSourcedWithId): UInt - - // The following functions are all wiring helpers for each of the different types of TileLink channels - - def hookupClientSource[M <: ClientSourcedWithIdAndData]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - def hasData(m: LogicalNetworkIO[M]) = m.payload.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) - clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { - arb.valid := req.valid - arb.bits := req.bits - arb.bits.payload.client_xact_id := clientSourcedClientXactId(req.bits.payload, id) - req.ready := arb.ready - }} - mngr <> arb.io.out - } - - def hookupClientSourceHeaderless[M <: ClientSourcedWithIdAndData]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - def hasData(m: M) = m.hasMultibeatData() - val arb = Module(new LockingRRArbiter(mngr.bits, arbN, tlDataBeats, Some(hasData _))) - clts.zipWithIndex.zip(arb.io.in).map{ case ((req, id), arb) => { - arb.valid := req.valid - arb.bits := req.bits - arb.bits.client_xact_id := clientSourcedClientXactId(req.bits, id) - req.ready := arb.ready - }} - mngr <> arb.io.out - } - - def hookupManagerSourceWithHeader[M <: ManagerToClientChannel]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (mngr.bits.header.dst === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - } - } - - def hookupManagerSourceWithId[M <: ManagerSourcedWithId]( - clts: Seq[DecoupledIO[LogicalNetworkIO[M]]], - mngr: DecoupledIO[LogicalNetworkIO[M]]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (arbIdx(mngr.bits.payload) === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - clts(i).bits.payload.client_xact_id := managerSourcedClientXactId(mngr.bits.payload) - } - } - - def hookupManagerSourceHeaderlessWithId[M <: ManagerSourcedWithId]( - clts: Seq[DecoupledIO[M]], - mngr: DecoupledIO[M]) { - mngr.ready := Bool(false) - for (i <- 0 until arbN) { - clts(i).valid := Bool(false) - when (arbIdx(mngr.bits) === UInt(i)) { - clts(i).valid := mngr.valid - mngr.ready := clts(i).ready - } - clts(i).bits := mngr.bits - clts(i).bits.client_xact_id := managerSourcedClientXactId(mngr.bits) - } - } - - def hookupManagerSourceBroadcast[M <: Data](clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { - clts.map{ _.valid := mngr.valid } - clts.map{ _.bits := mngr.bits } - mngr.ready := clts.map(_.ready).reduce(_&&_) - } - - def hookupFinish[M <: LogicalNetworkIO[Finish]]( clts: Seq[DecoupledIO[M]], mngr: DecoupledIO[M]) { - val arb = Module(new RRArbiter(mngr.bits, arbN)) - arb.io.in <> clts - mngr <> arb.io.out - } -} - -/** Abstract base case for any Arbiters that have UncachedTileLinkIOs */ -abstract class UncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module - with TileLinkArbiterLike { - val io = new Bundle { - val in = Vec(arbN, new UncachedTileLinkIO).flip - val out = new UncachedTileLinkIO - } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupFinish(io.in.map(_.finish), io.out.finish) - hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) -} - -/** Abstract base case for any Arbiters that have cached TileLinkIOs */ -abstract class TileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module - with TileLinkArbiterLike { - val io = new Bundle { - val in = Vec(arbN, new TileLinkIO).flip - val out = new TileLinkIO - } - hookupClientSource(io.in.map(_.acquire), io.out.acquire) - hookupClientSource(io.in.map(_.release), io.out.release) - hookupFinish(io.in.map(_.finish), io.out.finish) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceWithId(io.in.map(_.grant), io.out.grant) -} - -/** Appends the port index of the arbiter to the client_xact_id */ -trait AppendsArbiterId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = - Cat(in.client_xact_id, UInt(id, log2Up(arbN))) - def managerSourcedClientXactId(in: ManagerSourcedWithId) = { - /* This shouldn't be necessary, but Chisel3 doesn't emit correct Verilog - * when right shifting by too many bits. See - * https://github.com/ucb-bar/firrtl/issues/69 */ - if (in.client_xact_id.getWidth > log2Up(arbN)) - in.client_xact_id >> log2Up(arbN) - else - UInt(0) - } - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id(log2Up(arbN)-1,0).toUInt -} - -/** Uses the client_xact_id as is (assumes it has been set to port index) */ -trait PassesId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = in.client_xact_id - def managerSourcedClientXactId(in: ManagerSourcedWithId) = in.client_xact_id - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id -} - -/** Overwrites some default client_xact_id with the port idx */ -trait UsesNewId extends TileLinkArbiterLike { - def clientSourcedClientXactId(in: ClientSourcedWithId, id: Int) = UInt(id, log2Up(arbN)) - def managerSourcedClientXactId(in: ManagerSourcedWithId) = UInt(0) - def arbIdx(in: ManagerSourcedWithId) = in.client_xact_id -} - -// Now we can mix-in thevarious id-generation traits to make concrete arbiter classes -class UncachedTileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with AppendsArbiterId -class UncachedTileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with PassesId -class UncachedTileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends UncachedTileLinkIOArbiter(n)(p) with UsesNewId -class TileLinkIOArbiterThatAppendsArbiterId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with AppendsArbiterId -class TileLinkIOArbiterThatPassesId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with PassesId -class TileLinkIOArbiterThatUsesNewId(val n: Int)(implicit p: Parameters) extends TileLinkIOArbiter(n)(p) with UsesNewId - -/** Concrete uncached client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientUncachedTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { - val io = new Bundle { - val in = Vec(arbN, new ClientUncachedTileLinkIO).flip - val out = new ClientUncachedTileLinkIO - } - if (arbN > 1) { - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) - } else { io.out <> io.in.head } -} - -/** Concrete client-side arbiter that appends the arbiter's port id to client_xact_id */ -class ClientTileLinkIOArbiter(val arbN: Int)(implicit val p: Parameters) extends Module with TileLinkArbiterLike with AppendsArbiterId { - val io = new Bundle { - val in = Vec(arbN, new ClientTileLinkIO).flip - val out = new ClientTileLinkIO - } - if (arbN > 1) { - hookupClientSourceHeaderless(io.in.map(_.acquire), io.out.acquire) - hookupClientSourceHeaderless(io.in.map(_.release), io.out.release) - hookupManagerSourceBroadcast(io.in.map(_.probe), io.out.probe) - hookupManagerSourceHeaderlessWithId(io.in.map(_.grant), io.out.grant) - } else { io.out <> io.in.head } -} diff --git a/uncore/src/main/scala/interconnect.scala b/uncore/src/main/scala/tilelink/Interconnect.scala similarity index 99% rename from uncore/src/main/scala/interconnect.scala rename to uncore/src/main/scala/tilelink/Interconnect.scala index eb9773a1..353dbb80 100644 --- a/uncore/src/main/scala/interconnect.scala +++ b/uncore/src/main/scala/tilelink/Interconnect.scala @@ -1,8 +1,9 @@ -package uncore +package uncore.tilelink import Chisel._ import junctions._ import scala.collection.mutable.ArraySeq +import uncore.util._ import cde.{Parameters, Field} diff --git a/uncore/src/main/scala/tilelink/Network.scala b/uncore/src/main/scala/tilelink/Network.scala new file mode 100644 index 00000000..1c094013 --- /dev/null +++ b/uncore/src/main/scala/tilelink/Network.scala @@ -0,0 +1,308 @@ +// See LICENSE for license details. + +package uncore.tilelink + +import Chisel._ +import uncore.util._ +import cde.{Parameters, Field} + +case object LNEndpoints extends Field[Int] +case object LNHeaderBits extends Field[Int] + +class PhysicalHeader(n: Int) extends Bundle { + val src = UInt(width = log2Up(n)) + val dst = UInt(width = log2Up(n)) +} + +class PhysicalNetworkIO[T <: Data](n: Int, dType: T) extends Bundle { + val header = new PhysicalHeader(n) + val payload = dType.cloneType + override def cloneType = new PhysicalNetworkIO(n,dType).asInstanceOf[this.type] +} + +class BasicCrossbarIO[T <: Data](n: Int, dType: T) extends Bundle { + val in = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))).flip + val out = Vec(n, Decoupled(new PhysicalNetworkIO(n,dType))) +} + +abstract class PhysicalNetwork extends Module + +case class CrossbarConfig[T <: Data](n: Int, dType: T, count: Int = 1, needsLock: Option[PhysicalNetworkIO[T] => Bool] = None) + +abstract class AbstractCrossbar[T <: Data](conf: CrossbarConfig[T]) extends PhysicalNetwork { + val io = new BasicCrossbarIO(conf.n, conf.dType) +} + +class BasicBus[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { + val arb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) + arb.io.in <> io.in + + arb.io.out.ready := io.out(arb.io.out.bits.header.dst).ready + for ((out, i) <- io.out zipWithIndex) { + out.valid := arb.io.out.valid && arb.io.out.bits.header.dst === UInt(i) + out.bits := arb.io.out.bits + } +} + +class BasicCrossbar[T <: Data](conf: CrossbarConfig[T]) extends AbstractCrossbar(conf) { + io.in.foreach { _.ready := Bool(false) } + + io.out.zipWithIndex.map{ case (out, i) => { + val rrarb = Module(new LockingRRArbiter(io.in(0).bits, conf.n, conf.count, conf.needsLock)) + (rrarb.io.in, io.in).zipped.map{ case (arb, in) => { + val destined = in.bits.header.dst === UInt(i) + arb.valid := in.valid && destined + arb.bits := in.bits + when (arb.ready && destined) { in.ready := Bool(true) } + }} + out <> rrarb.io.out + }} +} + +abstract class LogicalNetwork extends Module + +class LogicalHeader(implicit p: Parameters) extends junctions.ParameterizedBundle()(p) { + val src = UInt(width = p(LNHeaderBits)) + val dst = UInt(width = p(LNHeaderBits)) +} + +class LogicalNetworkIO[T <: Data](dType: T)(implicit p: Parameters) extends Bundle { + val header = new LogicalHeader + val payload = dType.cloneType + override def cloneType = new LogicalNetworkIO(dType)(p).asInstanceOf[this.type] +} + +object DecoupledLogicalNetworkIOWrapper { + def apply[T <: Data]( + in: DecoupledIO[T], + src: UInt = UInt(0), + dst: UInt = UInt(0)) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits))) + out.valid := in.valid + out.bits.payload := in.bits + out.bits.header.dst := dst + out.bits.header.src := src + in.ready := out.ready + out + } +} + +object DecoupledLogicalNetworkIOUnwrapper { + def apply[T <: Data](in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[T] = { + val out = Wire(Decoupled(in.bits.payload)) + out.valid := in.valid + out.bits := in.bits.payload + in.ready := out.ready + out + } +} + +object DefaultFromPhysicalShim { + def apply[T <: Data](in: DecoupledIO[PhysicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(Decoupled(new LogicalNetworkIO(in.bits.payload))) + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +object DefaultToPhysicalShim { + def apply[T <: Data](n: Int, in: DecoupledIO[LogicalNetworkIO[T]]) + (implicit p: Parameters): DecoupledIO[PhysicalNetworkIO[T]] = { + val out = Wire(Decoupled(new PhysicalNetworkIO(n, in.bits.payload))) + out.bits.header := in.bits.header + out.bits.payload := in.bits.payload + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A helper module that automatically issues [[uncore.Finish]] messages in repsonse + * to [[uncore.Grant]] that it receives from a manager and forwards to a client + */ +class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) extends TLModule()(p) + with HasDataBeatCounters { + val io = new Bundle { + val grant = Decoupled(new LogicalNetworkIO(new Grant)).flip + val refill = Decoupled(new Grant) + val finish = Decoupled(new LogicalNetworkIO(new Finish)) + val ready = Bool(OUTPUT) + } + + val g = io.grant.bits.payload + + if(tlNetworkPreservesPointToPointOrdering) { + io.finish.valid := Bool(false) + io.refill.valid := io.grant.valid + io.refill.bits := g + io.grant.ready := io.refill.ready + io.ready := Bool(true) + } else { + // We only want to send Finishes after we have collected all beats of + // a multibeat Grant. But Grants from multiple managers or transactions may + // get interleaved, so we could need a counter for each. + val done = if(tlNetworkDoesNotInterleaveBeats) { + connectIncomingDataBeatCounterWithHeader(io.grant) + } else { + val entries = 1 << tlClientXactIdBits + def getId(g: LogicalNetworkIO[Grant]) = g.payload.client_xact_id + assert(getId(io.grant.bits) <= UInt(entries), "Not enough grant beat counters, only " + entries + " entries.") + connectIncomingDataBeatCountersWithHeader(io.grant, entries, getId).reduce(_||_) + } + val q = Module(new FinishQueue(outstanding)) + q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done) + q.io.enq.bits := g.makeFinish() + q.io.enq.bits.manager_id := io.grant.bits.header.src + + io.finish.bits.header.src := UInt(srcId) + io.finish.bits.header.dst := q.io.deq.bits.manager_id + io.finish.bits.payload := q.io.deq.bits + io.finish.valid := q.io.deq.valid + q.io.deq.ready := io.finish.ready + + io.refill.valid := (q.io.enq.ready || !g.requiresAck()) && io.grant.valid + io.refill.bits := g + io.grant.ready := (q.io.enq.ready || !g.requiresAck()) && io.refill.ready + io.ready := q.io.enq.ready + } +} + +class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries) + +/** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Probe Probes]]. + * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response, + * setting up the headers for each. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientTileLinkIO().flip + val network = new TileLinkIO + } + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert) + val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId) + val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe) + val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant) + + io.network.acquire <> acq_with_header + io.network.release <> rel_with_header + io.network.finish <> fin_with_header + io.client.probe <> prb_without_header + io.client.grant.bits.manager_id := io.network.grant.bits.header.src + io.client.grant <> gnt_without_header +} + +/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]] + * + * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages, + * calculating header.dst and filling in header.src. + * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers. + * + * @param clientId network port id of this agent + * @param addrConvert how a physical address maps to a destination manager port id + */ +class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val client = new ClientUncachedTileLinkIO().flip + val network = new TileLinkIO + } + + val finisher = Module(new FinishUnit(clientId)) + finisher.io.grant <> io.network.grant + io.network.finish <> finisher.io.finish + + val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert) + val gnt_without_header = finisher.io.refill + + io.network.acquire.bits := acq_with_header.bits + io.network.acquire.valid := acq_with_header.valid && finisher.io.ready + acq_with_header.ready := io.network.acquire.ready && finisher.io.ready + io.client.grant <> gnt_without_header + io.network.probe.ready := Bool(false) + io.network.release.valid := Bool(false) +} + +object ClientTileLinkHeaderCreator { + def apply[T <: ClientToManagerChannel with HasManagerId]( + in: DecoupledIO[T], + clientId: Int) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := in.bits.manager_id + out.valid := in.valid + in.ready := out.ready + out + } + def apply[T <: ClientToManagerChannel with HasCacheBlockAddress]( + in: DecoupledIO[T], + clientId: Int, + addrConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(clientId) + out.bits.header.dst := addrConvert(in.bits.addr_block) + out.valid := in.valid + in.ready := out.ready + out + } +} + +/** A port to convert [[uncore.ManagerTileLinkIO]].flip into [[uncore.TileLinkIO]].flip + * + * Creates network headers for [[uncore.Probe]] and [[uncore.Grant]] messagess, + * calculating header.dst and filling in header.src. + * Strips headers from [[uncore.Acquire]], [[uncore.Release]] and [[uncore.Finish]], + * but supplies client_id instead. + * + * @param managerId the network port id of this agent + * @param idConvert how a sharer id maps to a destination client port id + */ +class ManagerTileLinkNetworkPort(managerId: Int, idConvert: UInt => UInt) + (implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val manager = new ManagerTileLinkIO().flip + val network = new TileLinkIO().flip + } + io.network.grant <> ManagerTileLinkHeaderCreator(io.manager.grant, managerId, (u: UInt) => u) + io.network.probe <> ManagerTileLinkHeaderCreator(io.manager.probe, managerId, idConvert) + io.manager.acquire <> DecoupledLogicalNetworkIOUnwrapper(io.network.acquire) + io.manager.acquire.bits.client_id := io.network.acquire.bits.header.src + io.manager.release <> DecoupledLogicalNetworkIOUnwrapper(io.network.release) + io.manager.release.bits.client_id := io.network.release.bits.header.src + io.manager.finish <> DecoupledLogicalNetworkIOUnwrapper(io.network.finish) +} + +object ManagerTileLinkHeaderCreator { + def apply[T <: ManagerToClientChannel with HasClientId]( + in: DecoupledIO[T], + managerId: Int, + idConvert: UInt => UInt) + (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = { + val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits))) + out.bits.payload := in.bits + out.bits.header.src := UInt(managerId) + out.bits.header.dst := idConvert(in.bits.client_id) + out.valid := in.valid + in.ready := out.ready + out + } +} diff --git a/uncore/src/main/scala/amoalu.scala b/uncore/src/main/scala/util/AmoAlu.scala similarity index 97% rename from uncore/src/main/scala/amoalu.scala rename to uncore/src/main/scala/util/AmoAlu.scala index 6459d558..d6ff9ce8 100644 --- a/uncore/src/main/scala/amoalu.scala +++ b/uncore/src/main/scala/util/AmoAlu.scala @@ -1,8 +1,11 @@ // See LICENSE for license details. -package uncore +package uncore.util + import Chisel._ -import cde.{Parameters, Field} +import uncore.tilelink._ +import cde.Parameters +import uncore.constants._ class StoreGen(typ: UInt, addr: UInt, dat: UInt, maxSize: Int) { val size = typ(log2Up(log2Up(maxSize)+1)-1,0) diff --git a/uncore/src/main/scala/util/Counters.scala b/uncore/src/main/scala/util/Counters.scala new file mode 100644 index 00000000..3bc2d85b --- /dev/null +++ b/uncore/src/main/scala/util/Counters.scala @@ -0,0 +1,134 @@ +package uncore.util + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +// Produces 0-width value when counting to 1 +class ZCounter(val n: Int) { + val value = Reg(init=UInt(0, log2Ceil(n))) + def inc(): Bool = { + if (n == 1) Bool(true) + else { + val wrap = value === UInt(n-1) + value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) + wrap + } + } +} + +object ZCounter { + def apply(n: Int) = new ZCounter(n) + def apply(cond: Bool, n: Int): (UInt, Bool) = { + val c = new ZCounter(n) + var wrap: Bool = null + when (cond) { wrap = c.inc() } + (c.value, cond && wrap) + } +} + +object TwoWayCounter { + def apply(up: Bool, down: Bool, max: Int): UInt = { + val cnt = Reg(init = UInt(0, log2Up(max+1))) + when (up && !down) { cnt := cnt + UInt(1) } + when (down && !up) { cnt := cnt - UInt(1) } + cnt + } +} + +class BeatCounterStatus extends Bundle { + val idx = UInt() + val done = Bool() +} + +class TwoWayBeatCounterStatus extends Bundle { + val pending = Bool() + val up = new BeatCounterStatus() + val down = new BeatCounterStatus() +} + +/** Utility trait containing wiring functions to keep track of how many data beats have + * been sent or recieved over a particular [[uncore.TileLinkChannel]] or pair of channels. + * + * Won't count message types that don't have data. + * Used in [[uncore.XactTracker]] and [[uncore.FinishUnit]]. + */ +trait HasDataBeatCounters { + type HasBeat = TileLinkChannel with HasTileLinkBeatId + type HasId = TileLinkChannel with HasClientId + + /** Returns the current count on this channel and when a message is done + * @param inc increment the counter (usually .valid or .fire()) + * @param data the actual channel data + * @param beat count to return for single-beat messages + */ + def connectDataBeatCounter[S <: TileLinkChannel](inc: Bool, data: S, beat: UInt) = { + val multi = data.hasMultibeatData() + val (multi_cnt, multi_done) = Counter(inc && multi, data.tlDataBeats) + val cnt = Mux(multi, multi_cnt, beat) + val done = Mux(multi, multi_done, inc) + (cnt, done) + } + + /** Counter for beats on outgoing [[chisel.DecoupledIO]] */ + def connectOutgoingDataBeatCounter[T <: TileLinkChannel]( + out: DecoupledIO[T], + beat: UInt = UInt(0)): (UInt, Bool) = + connectDataBeatCounter(out.fire(), out.bits, beat) + + /** Returns done but not cnt. Use the addr_beat subbundle instead of cnt for beats on + * incoming channels in case of network reordering. + */ + def connectIncomingDataBeatCounter[T <: TileLinkChannel](in: DecoupledIO[T]): Bool = + connectDataBeatCounter(in.fire(), in.bits, UInt(0))._2 + + /** Counter for beats on incoming DecoupledIO[LogicalNetworkIO[]]s returns done */ + def connectIncomingDataBeatCounterWithHeader[T <: TileLinkChannel](in: DecoupledIO[LogicalNetworkIO[T]]): Bool = + connectDataBeatCounter(in.fire(), in.bits.payload, UInt(0))._2 + + /** If the network might interleave beats from different messages, we need a Vec of counters, + * one for every outstanding message id that might be interleaved. + * + * @param getId mapping from Message to counter id + */ + def connectIncomingDataBeatCountersWithHeader[T <: TileLinkChannel with HasClientTransactionId]( + in: DecoupledIO[LogicalNetworkIO[T]], + entries: Int, + getId: LogicalNetworkIO[T] => UInt): Vec[Bool] = { + Vec((0 until entries).map { i => + connectDataBeatCounter(in.fire() && getId(in.bits) === UInt(i), in.bits.payload, UInt(0))._2 + }) + } + + /** Provides counters on two channels, as well a meta-counter that tracks how many + * messages have been sent over the up channel but not yet responded to over the down channel + * + * @param status bundle of status of the counters + * @param up outgoing channel + * @param down incoming channel + * @param max max number of outstanding ups with no down + * @param beat overrides cnts on single-beat messages + * @param track whether up's message should be tracked + * @return a tuple containing whether their are outstanding messages, up's count, + * up's done, down's count, down's done + */ + def connectTwoWayBeatCounters[T <: TileLinkChannel, S <: TileLinkChannel]( + status: TwoWayBeatCounterStatus, + up: DecoupledIO[T], + down: DecoupledIO[S], + max: Int = 1, + beat: UInt = UInt(0), + trackUp: T => Bool = (t: T) => Bool(true), + trackDown: S => Bool = (s: S) => Bool(true)) { + val (up_idx, up_done) = connectDataBeatCounter(up.fire() && trackUp(up.bits), up.bits, beat) + val (dn_idx, dn_done) = connectDataBeatCounter(down.fire() && trackDown(down.bits), down.bits, beat) + val cnt = TwoWayCounter(up_done, dn_done, max) + status.pending := cnt > UInt(0) + status.up.idx := up_idx + status.up.done := up_done + status.down.idx := dn_idx + status.down.done := dn_done + } +} + + diff --git a/uncore/src/main/scala/util/Enqueuer.scala b/uncore/src/main/scala/util/Enqueuer.scala new file mode 100644 index 00000000..f6f3587a --- /dev/null +++ b/uncore/src/main/scala/util/Enqueuer.scala @@ -0,0 +1,56 @@ +package uncore.util + +import Chisel._ +import uncore.tilelink._ +import cde.Parameters + +/** Struct for describing per-channel queue depths */ +case class TileLinkDepths(acq: Int, prb: Int, rel: Int, gnt: Int, fin: Int) + +/** Optionally enqueues each [[uncore.TileLinkChannel]] individually */ +class TileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { + val io = new Bundle { + val client = new TileLinkIO().flip + val manager = new TileLinkIO + } + io.manager.acquire <> (if(depths.acq > 0) Queue(io.client.acquire, depths.acq) else io.client.acquire) + io.client.probe <> (if(depths.prb > 0) Queue(io.manager.probe, depths.prb) else io.manager.probe) + io.manager.release <> (if(depths.rel > 0) Queue(io.client.release, depths.rel) else io.client.release) + io.client.grant <> (if(depths.gnt > 0) Queue(io.manager.grant, depths.gnt) else io.manager.grant) + io.manager.finish <> (if(depths.fin > 0) Queue(io.client.finish, depths.fin) else io.client.finish) +} + +object TileLinkEnqueuer { + def apply(in: TileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): TileLinkIO = { + val t = Module(new TileLinkEnqueuer(depths)) + t.io.client <> in + t.io.manager + } + def apply(in: TileLinkIO, depth: Int)(implicit p: Parameters): TileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) + } +} + +class ClientTileLinkEnqueuer(depths: TileLinkDepths)(implicit p: Parameters) extends Module { + val io = new Bundle { + val inner = new ClientTileLinkIO().flip + val outer = new ClientTileLinkIO + } + + io.outer.acquire <> (if(depths.acq > 0) Queue(io.inner.acquire, depths.acq) else io.inner.acquire) + io.inner.probe <> (if(depths.prb > 0) Queue(io.outer.probe, depths.prb) else io.outer.probe) + io.outer.release <> (if(depths.rel > 0) Queue(io.inner.release, depths.rel) else io.inner.release) + io.inner.grant <> (if(depths.gnt > 0) Queue(io.outer.grant, depths.gnt) else io.outer.grant) + io.outer.finish <> (if(depths.fin > 0) Queue(io.inner.finish, depths.fin) else io.inner.finish) +} + +object ClientTileLinkEnqueuer { + def apply(in: ClientTileLinkIO, depths: TileLinkDepths)(implicit p: Parameters): ClientTileLinkIO = { + val t = Module(new ClientTileLinkEnqueuer(depths)) + t.io.inner <> in + t.io.outer + } + def apply(in: ClientTileLinkIO, depth: Int)(implicit p: Parameters): ClientTileLinkIO = { + apply(in, TileLinkDepths(depth, depth, depth, depth, depth)) + } +} diff --git a/uncore/src/main/scala/util.scala b/uncore/src/main/scala/util/Serializer.scala similarity index 69% rename from uncore/src/main/scala/util.scala rename to uncore/src/main/scala/util/Serializer.scala index b6d11fd5..8cc0caa2 100644 --- a/uncore/src/main/scala/util.scala +++ b/uncore/src/main/scala/util/Serializer.scala @@ -1,40 +1,9 @@ // See LICENSE for license details. -package uncore +package uncore.util import Chisel._ - -// Produces 0-width value when counting to 1 -class ZCounter(val n: Int) { - val value = Reg(init=UInt(0, log2Ceil(n))) - def inc(): Bool = { - if (n == 1) Bool(true) - else { - val wrap = value === UInt(n-1) - value := Mux(Bool(!isPow2(n)) && wrap, UInt(0), value + UInt(1)) - wrap - } - } -} - -object ZCounter { - def apply(n: Int) = new ZCounter(n) - def apply(cond: Bool, n: Int): (UInt, Bool) = { - val c = new ZCounter(n) - var wrap: Bool = null - when (cond) { wrap = c.inc() } - (c.value, cond && wrap) - } -} - -object TwoWayCounter { - def apply(up: Bool, down: Bool, max: Int): UInt = { - val cnt = Reg(init = UInt(0, log2Up(max+1))) - when (up && !down) { cnt := cnt + UInt(1) } - when (down && !up) { cnt := cnt - UInt(1) } - cnt - } -} +import uncore.tilelink._ class FlowThroughSerializer[T <: Bundle with HasTileLinkData](gen: T, n: Int) extends Module { val io = new Bundle { From 9feca99d5df3f0684e200ca3fff0d9d395ee0b5f Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 28 Jun 2016 13:09:20 -0700 Subject: [PATCH 642/688] make PutBlock wmask argument match Put --- uncore/src/main/scala/converters/Nasti.scala | 2 +- .../src/main/scala/converters/Tilelink.scala | 7 +++---- uncore/src/main/scala/devices/Dma.scala | 2 +- .../src/main/scala/tilelink/Definitions.scala | 19 ++----------------- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 006485ca..2f31d9d6 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -319,7 +319,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) addr_block = nasti_addr_block(aw_req), addr_beat = put_count, data = io.nasti.w.bits.data, - wmask = io.nasti.w.bits.strb), + wmask = Some(io.nasti.w.bits.strb)), Put( client_xact_id = aw_req.id, addr_block = nasti_addr_block(aw_req), diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index 4899f05d..e3cd4474 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -107,8 +107,7 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { client_xact_id = irel.client_xact_id, addr_block = irel.addr_block, addr_beat = irel.addr_beat, - data = irel.data, - wmask = Acquire.fullWriteMask) + data = irel.data) io.in.release.ready := rel_helper.fire(io.in.release.valid) io.out.acquire <> acqArb.io.out @@ -264,7 +263,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) addr_block = put_block, addr_beat = put_beat, data = put_data.toBits, - wmask = put_wmask.toBits) + wmask = Some(put_wmask.toBits)) io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) io.out.acquire.bits := MuxCase(get_block_acquire, Seq( @@ -444,7 +443,7 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) Cat(acq_addr_beat, oacq_ctr.value) else acq_addr_beat, data = acq_data_buffer(outerDataBits - 1, 0), - wmask = acq_wmask_buffer(outerWriteMaskBits - 1, 0))(outerConfig) + wmask = Some(acq_wmask_buffer(outerWriteMaskBits - 1, 0)))(outerConfig) val get_acquire = Get( client_xact_id = iacq.client_xact_id, diff --git a/uncore/src/main/scala/devices/Dma.scala b/uncore/src/main/scala/devices/Dma.scala index eadff9d7..036bf95b 100644 --- a/uncore/src/main/scala/devices/Dma.scala +++ b/uncore/src/main/scala/devices/Dma.scala @@ -354,7 +354,7 @@ class DmaTracker(implicit p: Parameters) extends DmaModule()(p) addr_block = dst_block, addr_beat = put_beat, data = put_data, - wmask = put_mask) + wmask = Some(put_mask)) val get_acquire = GetBlock( client_xact_id = get_half, diff --git a/uncore/src/main/scala/tilelink/Definitions.scala b/uncore/src/main/scala/tilelink/Definitions.scala index 1917e403..dd16e1e8 100644 --- a/uncore/src/main/scala/tilelink/Definitions.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -585,22 +585,7 @@ object PutBlock { addr_block: UInt, addr_beat: UInt, data: UInt, - wmask: UInt) - (implicit p: Parameters): Acquire = { - BuiltInAcquireBuilder( - a_type = Acquire.putBlockType, - client_xact_id = client_xact_id, - addr_block = addr_block, - addr_beat = addr_beat, - data = data, - wmask = wmask, - alloc = Bool(true)) - } - def apply( - client_xact_id: UInt, - addr_block: UInt, - addr_beat: UInt, - data: UInt, + wmask: Option[UInt] = None, alloc: Bool = Bool(true)) (implicit p: Parameters): Acquire = { BuiltInAcquireBuilder( @@ -609,7 +594,7 @@ object PutBlock { addr_block = addr_block, addr_beat = addr_beat, data = data, - wmask = Acquire.fullWriteMask, + wmask = wmask.getOrElse(Acquire.fullWriteMask), alloc = alloc) } } From a0b1772404fb8f184b15910bb9f18d88f7c1f2b8 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 30 Jun 2016 15:50:23 -0700 Subject: [PATCH 643/688] change TileLinkWidthAdapter interface --- .../src/main/scala/converters/Tilelink.scala | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index e3cd4474..d196776b 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -151,20 +151,21 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { } object TileLinkWidthAdapter { - def apply(in: ClientUncachedTileLinkIO, out: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { - require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) - - if (out.tlDataBits > in.tlDataBits) { - val widener = Module(new TileLinkIOWidener(in.p(TLId), out.p(TLId))) + def apply(in: ClientUncachedTileLinkIO, outerId: String)(implicit p: Parameters) = { + val outerDataBits = p(TLKey(outerId)).dataBitsPerBeat + if (outerDataBits > in.tlDataBits) { + val widener = Module(new TileLinkIOWidener(in.p(TLId), outerId)) widener.io.in <> in - out <> widener.io.out - } else if (out.tlDataBits < in.tlDataBits) { - val narrower = Module(new TileLinkIONarrower(in.p(TLId), out.p(TLId))) + widener.io.out + } else if (outerDataBits < in.tlDataBits) { + val narrower = Module(new TileLinkIONarrower(in.p(TLId), outerId)) narrower.io.in <> in - out <> narrower.io.out - } else { - out <> in - } + narrower.io.out + } else { in } + } + def apply(out: ClientUncachedTileLinkIO, in: ClientUncachedTileLinkIO)(implicit p: Parameters): Unit = { + require(out.tlDataBits * out.tlDataBeats == in.tlDataBits * in.tlDataBeats) + out <> apply(in, out.p(TLId)) } } From ce46f523c91f842d86dca86fc6370d486bf5c69d Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 30 Jun 2016 18:17:16 -0700 Subject: [PATCH 644/688] make sure Widener uses proper parameters to generate acquire/grant --- .../src/main/scala/converters/Tilelink.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index d196776b..3da4b07f 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -238,18 +238,20 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) def align_wmask(addr: UInt, wmask: UInt): UInt = wmask << Cat(addr, UInt(0, log2Up(innerWriteMaskBits))) + val outerConfig = p.alterPartial({ case TLId => outerTLId }) + val get_acquire = Get( client_xact_id = iacq.client_xact_id, addr_block = out_addr_block, addr_beat = out_addr_beat, addr_byte = out_addr_byte, operand_size = iacq.op_size(), - alloc = iacq.allocate()) + alloc = iacq.allocate())(outerConfig) val get_block_acquire = GetBlock( client_xact_id = iacq.client_xact_id, addr_block = out_addr_block, - alloc = iacq.allocate()) + alloc = iacq.allocate())(outerConfig) val put_acquire = Put( client_xact_id = iacq.client_xact_id, @@ -257,14 +259,14 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) addr_beat = out_addr_beat, data = align_data(switch_addr, iacq.data), wmask = Some(align_wmask(switch_addr, iacq.wmask())), - alloc = iacq.allocate()) + alloc = iacq.allocate())(outerConfig) val put_block_acquire = PutBlock( client_xact_id = put_id, addr_block = put_block, addr_beat = put_beat, data = put_data.toBits, - wmask = Some(put_wmask.toBits)) + wmask = Some(put_wmask.toBits))(outerConfig) io.out.acquire.valid := sending_put || (!shrink && io.in.acquire.valid) io.out.acquire.bits := MuxCase(get_block_acquire, Seq( @@ -316,13 +318,15 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) val gnt_switch = smallget_switch(ognt.client_xact_id) + val innerConfig = p.alterPartial({ case TLId => innerTLId }) + val get_block_grant = Grant( is_builtin_type = Bool(true), g_type = Grant.getDataBlockType, client_xact_id = gnt_client_id, manager_xact_id = gnt_manager_id, addr_beat = Cat(gnt_beat, send_idx), - data = select_data(gnt_data, send_idx)) + data = select_data(gnt_data, send_idx))(innerConfig) val get_grant = Grant( is_builtin_type = Bool(true), @@ -330,7 +334,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, addr_beat = Cat(ognt.addr_beat, gnt_switch), - data = select_data(ognt.data, gnt_switch)) + data = select_data(ognt.data, gnt_switch))(innerConfig) val default_grant = Grant( is_builtin_type = Bool(true), @@ -338,7 +342,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) client_xact_id = ognt.client_xact_id, manager_xact_id = ognt.manager_xact_id, addr_beat = ognt.addr_beat, - data = ognt.data) + data = ognt.data)(innerConfig) io.in.grant.valid := returning_data || (!stretch && io.out.grant.valid) io.in.grant.bits := MuxCase(default_grant, Seq( From 0eedffa82fa66efcf17f625da0e8327923a36af9 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 29 Jun 2016 16:02:31 -0700 Subject: [PATCH 645/688] WIP: Fix BufferlessBroadcastHub --- uncore/src/main/scala/agents/Bufferless.scala | 12 +++-- uncore/src/main/scala/agents/Cache.scala | 2 +- uncore/src/main/scala/agents/Trackers.scala | 47 ++++++++++--------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index 959367f8..fbece364 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -79,8 +79,7 @@ class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Par (!io.irel().hasData() || io.outer.release.ready) // Dispatch outer release - outerRelease(coh = outer_coh.onHit(M_XWR)) - io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data + outerRelease(coh = outer_coh.onHit(M_XWR), buffering = Bool(false)) quiesce() {} } @@ -128,15 +127,20 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // Send outer request for miss outerAcquire( caching = !xact_iacq.isBuiltInType(), + block_outer_acquire = vol_ognt_counter.pending, buffering = Bool(false), coh = outer_coh, next = s_busy) // Handle the response from outer memory - io.outer.grant.ready := state === s_busy && io.inner.grant.ready // bypass data + when (ognt_counter.pending && io.ognt().hasData()) { + io.outer.grant.ready := io.inner.grant.ready // bypass data + } // Acknowledge or respond with data - innerGrant(external_pending = pending_orel || ognt_counter.pending || vol_ognt_counter.pending) + innerGrant( + external_pending = pending_orel || vol_ognt_counter.pending, + buffering = Bool(false)) when(iacq_is_allocating) { initializeProbes() } diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 6f7e1cae..b88ece50 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -955,7 +955,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) innerGrant( data = Mux(xact_iacq.isAtomic(), amo_result, data_buffer(ignt_data_idx)), external_pending = pending_writes.orR || ognt_counter.pending, - add = addPendingBitInternal(io.data.resp)) + add_pending_bits = addPendingBitInternal(io.data.resp)) updatePendingCohWhen(io.inner.grant.fire() && io.ignt().last(), pending_coh_on_ignt) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 13bcb70d..add92bba 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -287,9 +287,10 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { add_pending_send_bit: Bool = Bool(false)) { when (state =/= s_idle || io.alloc.irel.should) { - pending_orel_data := (pending_orel_data & dropPendingBitWhenBeatHasData(io.outer.release)) | - addPendingBitWhenBeatHasData(io.inner.release) | - add_pending_data_bits + pending_orel_data := (pending_orel_data | + addPendingBitWhenBeatHasData(io.inner.release) | + add_pending_data_bits) & + dropPendingBitWhenBeatHasData(io.outer.release) } when (add_pending_send_bit) { pending_orel_send := Bool(true) } when (io.outer.release.fire()) { pending_orel_send := Bool(false) } @@ -301,13 +302,13 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(), trackDown = (g: Grant) => g.isVoluntary()) - io.outer.release.valid := state === s_busy && - Mux(io.orel().hasData(), - Mux(buffering, - pending_orel_data(vol_ognt_counter.up.idx), - io.inner.release.valid), - pending_orel) - + io.outer.release.valid := Mux(buffering, + (state === s_busy) && Mux(io.orel().hasData(), + pending_orel_data(vol_ognt_counter.up.idx), + pending_orel_send), + // only writebacks need to be forwarded to the outer interface + (io.alloc.irel.should || io.alloc.irel.matches) && + io.irel().hasData() && io.inner.release.valid) io.outer.release.bits := coh.makeVoluntaryWriteback( client_xact_id = UInt(0), // TODO was tracker id, but not needed? @@ -315,7 +316,7 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { addr_beat = vol_ognt_counter.up.idx, data = data) - io.outer.grant.ready := state === s_busy + when (vol_ognt_counter.pending) { io.outer.grant.ready := Bool(true) } scoreboard += (pending_orel, vol_ognt_counter.pending) } @@ -457,7 +458,8 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer def innerGrant( data: UInt = io.ognt().data, external_pending: Bool = Bool(false), - add: UInt = UInt(0)) { + buffering: Bool = Bool(true), + add_pending_bits: UInt = UInt(0)) { // Track the number of outstanding inner.finishes connectTwoWayBeatCounters( status = ifin_counter, @@ -471,14 +473,13 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer pending_ignt_data := (pending_ignt_data & dropPendingBitWhenBeatHasData(io.inner.grant)) | addPendingBitWhenBeatHasData(io.inner.release) | addPendingBitWhenBeatHasData(io.outer.grant) | - add + add_pending_bits } - // We can issue a grant for a pending write once all data is - // received and committed to the data array or outer memory - val ignt_ack_ready = !(state === s_idle || - state === s_meta_read || - pending_put_data.orR) + // Have we finished receiving the complete inner acquire transaction? + val iacq_finished = !(state === s_idle || + state === s_meta_read || + pending_put_data.orR) val ignt_from_iacq = inner_coh.makeGrant( sec = ignt_q.io.deq.bits, @@ -495,8 +496,10 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer io.inner.grant.bits := ignt_from_iacq io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter when (state === s_busy && pending_ignt) { - io.inner.grant.valid := !external_pending && - Mux(io.ignt().hasData(), pending_ignt_data(ignt_data_idx), ignt_ack_ready) + io.inner.grant.valid := !external_pending && Mux(buffering, + Mux(io.ignt().hasData(), + pending_ignt_data(ignt_data_idx), iacq_finished), + io.outer.grant.valid) } } @@ -519,6 +522,7 @@ trait EmitsOuterAcquires extends AcceptsInnerAcquires { def outerAcquire( caching: Bool, coh: ClientMetadata, + block_outer_acquire: Bool = Bool(false), buffering: Bool = Bool(true), data: UInt = io.iacq().data, wmask: UInt = io.iacq().wmask(), @@ -533,6 +537,7 @@ trait EmitsOuterAcquires extends AcceptsInnerAcquires { trackDown = (g: Grant) => !g.isVoluntary()) io.outer.acquire.valid := state === s_outer_acquire && + !block_outer_acquire && (xact_allocate || Mux(buffering, !pending_put_data(ognt_counter.up.idx), @@ -559,7 +564,7 @@ trait EmitsOuterAcquires extends AcceptsInnerAcquires { when(state === s_outer_acquire && ognt_counter.up.done) { state := next } - io.outer.grant.ready := state === s_busy + when (ognt_counter.pending) { io.outer.grant.ready := Bool(true) } scoreboard += ognt_counter.pending } From 61e3e5b45a29f2c7178ccb5b399bf2d0e024269e Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 30 Jun 2016 17:55:33 -0700 Subject: [PATCH 646/688] more WIP on fixing Bufferless --- uncore/src/main/scala/agents/Broadcast.scala | 2 +- uncore/src/main/scala/agents/Bufferless.scala | 10 +++-- uncore/src/main/scala/agents/Cache.scala | 2 +- uncore/src/main/scala/agents/Trackers.scala | 37 +++++++++++-------- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index cedda9b9..6c466a92 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -142,7 +142,7 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) can_alloc = Bool(false), next = s_inner_probe) - io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact // Track which clients yet need to be probed and make Probe message // If a writeback occurs, we can forward its data via the buffer, diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index fbece364..eaa5011f 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -97,9 +97,13 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) can_alloc = Bool(false), next = s_inner_probe) - val iacq_could_accept = state === s_outer_acquire || iacq_can_merge || iacq_same_xact - io.inner.acquire.ready := iacq_could_accept && - (!io.iacq().hasData() || io.outer.acquire.fire()) + // We are never going to merge anything in the bufferless hub + // Therefore, we only need to concern ourselves with the allocated + // transaction and (in case of PutBlock) subsequent tail beats + val iacq_can_forward = iacq_same_xact && !vol_ognt_counter.pending + io.inner.acquire.ready := Mux(io.iacq().hasData(), + state === s_outer_acquire && iacq_can_forward, + state === s_idle && io.alloc.iacq.should) // Track which clients yet need to be probed and make Probe message innerProbe( diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index b88ece50..889045cd 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -850,7 +850,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) can_alloc = Bool(true), next = s_meta_read) - io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact // Begin a transaction by getting the current block metadata // Defined here because of Chisel default wire demands, used in s_meta_resp diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index add92bba..0efeac2a 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -407,12 +407,10 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer val pending_put_data = Reg(init=Bits(0, width = innerDataBeats)) val pending_ignt_data = Reg(init=Bits(0, width = innerDataBeats)) - def iacq_same_xact: Bool = { + def iacq_same_xact: Bool = (xact_iacq.client_xact_id === io.iacq().client_xact_id) && - xact_iacq.hasMultibeatData() && - pending_ignt && - pending_put_data(io.iacq().addr_beat) - } + (xact_iacq.client_id === io.iacq().client_id) && + pending_ignt def iacq_can_merge: Bool def iacq_is_allocating: Bool = state === s_idle && io.alloc.iacq.should && io.inner.acquire.valid def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid @@ -420,7 +418,9 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer def innerAcquire(can_alloc: Bool, next: UInt) { // Enqueue some metadata information that we'll use to make coherence updates with later - ignt_q.io.enq.valid := iacq_is_allocating || (iacq_is_merging && io.iacq().first()) + ignt_q.io.enq.valid := iacq_is_allocating || + (!iacq_same_xact && pending_ignt && + io.inner.acquire.fire() && io.iacq().first()) ignt_q.io.enq.bits := io.iacq() // Use the outputs of the queue to make further messages @@ -496,10 +496,12 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer io.inner.grant.bits := ignt_from_iacq io.inner.grant.bits.addr_beat := ignt_data_idx // override based on outgoing counter when (state === s_busy && pending_ignt) { - io.inner.grant.valid := !external_pending && Mux(buffering, + io.inner.grant.valid := !external_pending && Mux(io.ignt().hasData(), - pending_ignt_data(ignt_data_idx), iacq_finished), - io.outer.grant.valid) + Mux(buffering, + pending_ignt_data(ignt_data_idx), + io.outer.grant.valid), + iacq_finished) } } @@ -536,12 +538,17 @@ trait EmitsOuterAcquires extends AcceptsInnerAcquires { beat = xact_addr_beat, trackDown = (g: Grant) => !g.isVoluntary()) - io.outer.acquire.valid := state === s_outer_acquire && - !block_outer_acquire && - (xact_allocate || - Mux(buffering, - !pending_put_data(ognt_counter.up.idx), - io.inner.acquire.valid)) + io.outer.acquire.valid := + state === s_outer_acquire && !block_outer_acquire && + (xact_allocate || + Mux(buffering, + !pending_put_data(ognt_counter.up.idx), + // If not buffering, we should only send an outer acquire if + // the ignt_q is not empty (pending_ignt) and the enqueued + // transaction does not have data or we are receiving the + // inner acquire and it is the same transaction as the one enqueued. + pending_ignt && (!xact_iacq.hasData() || + (io.inner.acquire.valid && iacq_same_xact)))) io.outer.acquire.bits := Mux(caching, From 10a46a36ae6cd00b3e295bb0b9cdf98c49336fb1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 1 Jul 2016 15:17:41 -0700 Subject: [PATCH 647/688] fix full_addr() function in TileLink --- uncore/src/main/scala/tilelink/Definitions.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink/Definitions.scala b/uncore/src/main/scala/tilelink/Definitions.scala index dd16e1e8..608f6042 100644 --- a/uncore/src/main/scala/tilelink/Definitions.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -293,7 +293,10 @@ class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel with HasAcquireType with HasAcquireUnion { /** Complete physical address for block, beat or operand */ - def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, this.addr_byte()) + def full_addr(dummy: Int = 0) = + Cat(this.addr_block, this.addr_beat, + Mux(isBuiltInType() && Acquire.typesWithAddrByte.contains(this.a_type), + this.addr_byte(), UInt(0, tlByteAddrBits))) } /** [[uncore.AcquireMetadata]] with an extra field containing the data beat */ @@ -349,6 +352,7 @@ object Acquire { def typesWithData = Vec(putType, putBlockType, putAtomicType) def typesWithMultibeatData = Vec(putBlockType) def typesOnSubBlocks = Vec(putType, getType, putAtomicType) + def typesWithAddrByte = Vec(getType, putAtomicType) /** Mapping between each built-in Acquire type and a built-in Grant type. */ def getBuiltInGrantType(a_type: UInt): UInt = { From e163a235838d2f520f465b5768ddaa154ad99ab9 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 1 Jul 2016 16:24:48 -0700 Subject: [PATCH 648/688] fix another bug in Widener --- uncore/src/main/scala/converters/Tilelink.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index 3da4b07f..6605ab63 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -205,6 +205,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) } val iacq = io.in.acquire.bits + val oacq = io.out.acquire.bits val ognt = io.out.grant.bits val ignt = io.in.grant.bits @@ -221,7 +222,7 @@ class TileLinkIOWidener(innerTLId: String, outerTLId: String) val put_data = Reg(Vec(factor, UInt(width = innerDataBits))) val put_wmask = Reg(Vec(factor, UInt(width = innerWriteMaskBits))) val put_allocate = Reg(Bool()) - val (put_beat, put_done) = Counter(io.out.acquire.fire() && iacq.hasMultibeatData(), outerDataBeats) + val (put_beat, put_done) = Counter(io.out.acquire.fire() && oacq.hasMultibeatData(), outerDataBeats) val (recv_idx, recv_done) = Counter(io.in.acquire.fire() && iacq.hasMultibeatData(), factor) val in_addr = iacq.full_addr() From 39bee5198d53a667950f2b349c3104907907fc5b Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 29 Jun 2016 18:39:46 -0700 Subject: [PATCH 649/688] Nasti Puts: decode wmask to determine addr_byte() and op_size() This change is TL0 specific; TL2 knows the op_size, and can use this to do a much simpler one-hot decode of the address. --- uncore/src/main/scala/converters/Nasti.scala | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 2f31d9d6..09953906 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -164,11 +164,31 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) UInt(log2Ceil(tlDataBytes))), len = Mux(is_subblock, UInt(0), UInt(tlDataBeats - 1))) + def mask_helper(all_inside_0: Seq[Bool], defsize: Int): (Seq[Bool], UInt, UInt) = { + val len = all_inside_0.size + if (len == 1) { + (Seq(Bool(true)), UInt(0), UInt(defsize)) + } else { + val sub_inside_0 = Seq.tabulate (len/2) { i => all_inside_0(2*i) && all_inside_0(2*i+1) } + val (sub_outside_0, sub_offset, sub_size) = mask_helper(sub_inside_0, defsize+1) + val all_outside_0 = Seq.tabulate (len) { i => sub_outside_0(i/2) && all_inside_0(i^1) } + val odd_outside_0 = Seq.tabulate (len/2) { i => all_outside_0(2*i+1) } + val odd_outside = odd_outside_0.reduce (_ || _) + val all_outside = all_outside_0.reduce (_ || _) + val offset = Cat(sub_offset, odd_outside.toBits) + val size = Mux(all_outside, UInt(defsize), sub_size) + (all_outside_0, offset, size) + } + } + + val all_inside_0 = (~io.tl.acquire.bits.wmask()).toBools + val (_, put_offset, put_size) = mask_helper(all_inside_0, 0) + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) io.nasti.aw.bits := NastiWriteAddressChannel( id = put_id_mapper.io.req.nasti_id, - addr = io.tl.acquire.bits.full_addr(), - size = UInt(log2Ceil(tlDataBytes)), + addr = io.tl.acquire.bits.full_addr()| put_offset, + size = put_size, len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) From caa9ca24b9cd442825d22e12f6d23fc27ccae793 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 1 Jul 2016 18:11:29 -0700 Subject: [PATCH 650/688] NASTI -> TL converter also uses ID mapper --- uncore/src/main/scala/converters/Nasti.scala | 139 +++++++++++-------- 1 file changed, 83 insertions(+), 56 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 09953906..528009b1 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -6,62 +6,62 @@ import uncore.tilelink._ import uncore.constants._ import cde.Parameters -class NastiIOTileLinkIOIdMapper(implicit val p: Parameters) extends Module - with HasTileLinkParameters with HasNastiParameters { +class IdMapper(val inIdBits: Int, val outIdBits: Int) + (implicit val p: Parameters) extends Module { val io = new Bundle { val req = new Bundle { - val valid = Bool(INPUT) - val ready = Bool(OUTPUT) - val tl_id = UInt(INPUT, tlClientXactIdBits) - val nasti_id = UInt(OUTPUT, nastiXIdBits) + val valid = Bool(INPUT) + val ready = Bool(OUTPUT) + val in_id = UInt(INPUT, inIdBits) + val out_id = UInt(OUTPUT, outIdBits) } val resp = new Bundle { - val valid = Bool(INPUT) + val valid = Bool(INPUT) val matches = Bool(OUTPUT) - val nasti_id = UInt(INPUT, nastiXIdBits) - val tl_id = UInt(OUTPUT, tlClientXactIdBits) + val out_id = UInt(INPUT, inIdBits) + val in_id = UInt(OUTPUT, outIdBits) } } - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort + val maxInXacts = 1 << inIdBits - if (tlClientXactIdBits <= nastiXIdBits) { + if (inIdBits <= outIdBits) { io.req.ready := Bool(true) - io.req.nasti_id := io.req.tl_id + io.req.out_id := io.req.in_id io.resp.matches := Bool(true) - io.resp.tl_id := io.resp.nasti_id - } else if (nastiXIdBits <= 2) { - val nQueues = 1 << nastiXIdBits - val entriesPerQueue = (tlMaxXacts - 1) / nQueues + 1 - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, nQueues) + io.resp.in_id := io.resp.out_id + } else if (outIdBits <= 2) { + val nQueues = 1 << outIdBits + val entriesPerQueue = (maxInXacts - 1) / nQueues + 1 + val (req_out_id, req_out_flip) = Counter(io.req.valid && io.req.ready, nQueues) io.req.ready := Bool(false) io.resp.matches := Bool(false) - io.resp.tl_id := UInt(0) - io.req.nasti_id := req_nasti_id + io.resp.in_id := UInt(0) + io.req.out_id := req_out_id for (i <- 0 until nQueues) { - val queue = Module(new Queue(UInt(width = tlClientXactIdBits), entriesPerQueue)) - queue.io.enq.valid := io.req.valid && req_nasti_id === UInt(i) - queue.io.enq.bits := io.req.tl_id - when (req_nasti_id === UInt(i)) { io.req.ready := queue.io.enq.ready } + val queue = Module(new Queue(UInt(width = inIdBits), entriesPerQueue)) + queue.io.enq.valid := io.req.valid && req_out_id === UInt(i) + queue.io.enq.bits := io.req.in_id + when (req_out_id === UInt(i)) { io.req.ready := queue.io.enq.ready } - queue.io.deq.ready := io.resp.valid && io.resp.nasti_id === UInt(i) - when (io.resp.nasti_id === UInt(i)) { + queue.io.deq.ready := io.resp.valid && io.resp.out_id === UInt(i) + when (io.resp.out_id === UInt(i)) { io.resp.matches := queue.io.deq.valid - io.resp.tl_id := queue.io.deq.bits + io.resp.in_id := queue.io.deq.bits } } } else { - val maxNastiId = 1 << nastiXIdBits - val (req_nasti_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxNastiId) + val maxOutId = 1 << outIdBits + val (req_out_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxOutId) val roq = Module(new ReorderQueue( - UInt(width = tlClientXactIdBits), nastiXIdBits, tlMaxXacts)) + UInt(width = inIdBits), outIdBits, maxInXacts)) roq.io.enq.valid := io.req.valid - roq.io.enq.bits.data := io.req.tl_id - roq.io.enq.bits.tag := req_nasti_id + roq.io.enq.bits.data := io.req.in_id + roq.io.enq.bits.tag := req_out_id io.req.ready := roq.io.enq.ready - io.req.nasti_id := req_nasti_id + io.req.out_id := req_out_id roq.io.deq.valid := io.resp.valid - roq.io.deq.tag := io.resp.nasti_id - io.resp.tl_id := roq.io.deq.data + roq.io.deq.tag := io.resp.out_id + io.resp.in_id := roq.io.deq.data io.resp.matches := roq.io.deq.matches } } @@ -109,8 +109,8 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val roq = Module(new ReorderQueue( new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) - val get_id_mapper = Module(new NastiIOTileLinkIOIdMapper) - val put_id_mapper = Module(new NastiIOTileLinkIOIdMapper) + val get_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) + val put_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) val get_id_ready = get_id_mapper.io.req.ready val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) @@ -145,19 +145,19 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) roq.io.deq.tag := io.nasti.r.bits.id get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) - get_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + get_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last - get_id_mapper.io.resp.nasti_id := io.nasti.r.bits.id + get_id_mapper.io.resp.out_id := io.nasti.r.bits.id put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) - put_id_mapper.io.req.tl_id := io.tl.acquire.bits.client_xact_id + put_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id put_id_mapper.io.resp.valid := io.nasti.b.fire() - put_id_mapper.io.resp.nasti_id := io.nasti.b.bits.id + put_id_mapper.io.resp.out_id := io.nasti.b.bits.id // Decompose outgoing TL Acquires into Nasti address and data channels io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) io.nasti.ar.bits := NastiReadAddressChannel( - id = get_id_mapper.io.req.nasti_id, + id = get_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr(), size = Mux(is_subblock, opSizeToXSize(io.tl.acquire.bits.op_size()), @@ -186,14 +186,14 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) io.nasti.aw.bits := NastiWriteAddressChannel( - id = put_id_mapper.io.req.nasti_id, + id = put_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr()| put_offset, size = put_size, len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) io.nasti.w.bits := NastiWriteDataChannel( - id = put_id_mapper.io.req.nasti_id, + id = put_id_mapper.io.req.out_id, data = io.tl.acquire.bits.data, strb = io.tl.acquire.bits.wmask(), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) @@ -222,7 +222,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) is_builtin_type = Bool(true), g_type = Mux(roq.io.deq.data.subblock, Grant.getDataBeatType, Grant.getDataBlockType), - client_xact_id = get_id_mapper.io.resp.tl_id, + client_xact_id = get_id_mapper.io.resp.in_id, manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), data = io.nasti.r.bits.data) @@ -237,7 +237,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) gnt_arb.io.in(1).bits := Grant( is_builtin_type = Bool(true), g_type = Grant.putAckType, - client_xact_id = put_id_mapper.io.resp.tl_id, + client_xact_id = put_id_mapper.io.resp.in_id, manager_xact_id = UInt(0), addr_beat = UInt(0), data = Bits(0)) @@ -260,6 +260,7 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) private val blockOffset = tlByteAddrBits + tlBeatAddrBits val aw_req = Reg(new NastiWriteAddressChannel) + val w_tl_id = Reg(io.tl.acquire.bits.client_xact_id) def is_singlebeat(chan: NastiAddressChannel): Bool = chan.len === UInt(0) @@ -307,9 +308,12 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) "NASTI write transaction cannot convert to TileLInk") val put_count = Reg(init = UInt(0, tlBeatAddrBits)) + val get_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits)) + val put_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits)) when (io.nasti.aw.fire()) { aw_req := io.nasti.aw.bits + w_tl_id := put_id_mapper.io.req.out_id state := s_put } @@ -323,10 +327,10 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) val get_acquire = Mux(is_multibeat(io.nasti.ar.bits), GetBlock( - client_xact_id = io.nasti.ar.bits.id, + client_xact_id = get_id_mapper.io.req.out_id, addr_block = nasti_addr_block(io.nasti.ar.bits)), Get( - client_xact_id = io.nasti.ar.bits.id, + client_xact_id = get_id_mapper.io.req.out_id, addr_block = nasti_addr_block(io.nasti.ar.bits), addr_beat = nasti_addr_beat(io.nasti.ar.bits), addr_byte = nasti_addr_byte(io.nasti.ar.bits), @@ -335,39 +339,62 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) val put_acquire = Mux(is_multibeat(aw_req), PutBlock( - client_xact_id = aw_req.id, + client_xact_id = w_tl_id, addr_block = nasti_addr_block(aw_req), addr_beat = put_count, data = io.nasti.w.bits.data, wmask = Some(io.nasti.w.bits.strb)), Put( - client_xact_id = aw_req.id, + client_xact_id = w_tl_id, addr_block = nasti_addr_block(aw_req), addr_beat = nasti_addr_beat(aw_req), data = io.nasti.w.bits.data, wmask = Some(nasti_wmask(aw_req, io.nasti.w.bits)))) + val get_helper = DecoupledHelper( + io.nasti.ar.valid, + get_id_mapper.io.req.ready, + io.tl.acquire.ready) + + get_id_mapper.io.req.valid := get_helper.fire( + get_id_mapper.io.req.ready, state === s_idle) + get_id_mapper.io.req.in_id := io.nasti.ar.bits.id + get_id_mapper.io.resp.out_id := io.tl.grant.bits.client_xact_id + get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last + + val aw_ok = (state === s_idle && !io.nasti.ar.valid) + + put_id_mapper.io.req.valid := aw_ok && io.nasti.aw.valid + put_id_mapper.io.req.in_id := io.nasti.aw.bits.id + put_id_mapper.io.resp.out_id := io.tl.grant.bits.client_xact_id + put_id_mapper.io.resp.valid := io.nasti.b.fire() + io.tl.acquire.bits := Mux(state === s_put, put_acquire, get_acquire) - io.tl.acquire.valid := (state === s_idle && io.nasti.ar.valid) || + io.tl.acquire.valid := get_helper.fire(io.tl.acquire.ready, state === s_idle) || (state === s_put && io.nasti.w.valid) - io.nasti.ar.ready := (state === s_idle && io.tl.acquire.ready) - io.nasti.aw.ready := (state === s_idle && !io.nasti.ar.valid) + + io.nasti.ar.ready := get_helper.fire(io.nasti.ar.valid, state === s_idle) + io.nasti.aw.ready := aw_ok && put_id_mapper.io.req.ready io.nasti.w.ready := (state === s_put && io.tl.acquire.ready) val nXacts = tlMaxClientXacts * tlMaxClientsPerPort io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) io.nasti.b.bits := NastiWriteResponseChannel( - id = io.tl.grant.bits.client_xact_id) + id = put_id_mapper.io.req.out_id) + + assert(!io.nasti.b.valid || put_id_mapper.io.resp.matches, + "Put ID does not match") io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) io.nasti.r.bits := NastiReadDataChannel( - id = io.tl.grant.bits.client_xact_id, + id = get_id_mapper.io.req.out_id, data = io.tl.grant.bits.data, last = tl_last(io.tl.grant.bits)) + assert(!io.nasti.r.valid || get_id_mapper.io.resp.matches, + "Get ID does not match") + io.tl.grant.ready := Mux(tl_b_grant(io.tl.grant.bits), io.nasti.b.ready, io.nasti.r.ready) } - - From 85808f8cbb76145c978b40cd9888e856426a293f Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Sat, 2 Jul 2016 15:09:12 -0700 Subject: [PATCH 651/688] Clean up PseudoLRU code --- uncore/src/main/scala/agents/Cache.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 889045cd..9adbf49c 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -90,9 +90,7 @@ class PseudoLRU(n: Int) var idx = UInt(1,1) for (i <- log2Up(n)-1 to 0 by -1) { val bit = way(i) - val mask = (UInt(1,n) << idx)(n-1,0) - next_state = next_state & ~mask | Mux(bit, UInt(0), mask) - //next_state.bitSet(idx, !bit) + next_state = next_state.bitSet(idx, !bit) idx = Cat(idx, bit) } next_state From afc51c4a353aaf3ec57c3c3057e02c0901105a8a Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 4 Jul 2016 16:33:28 -0700 Subject: [PATCH 652/688] make sure TL -> NASTI converter handles multibeat transactions properly --- uncore/src/main/scala/converters/Nasti.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 528009b1..734e0175 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -187,8 +187,9 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) io.nasti.aw.bits := NastiWriteAddressChannel( id = put_id_mapper.io.req.out_id, - addr = io.tl.acquire.bits.full_addr()| put_offset, - size = put_size, + addr = io.tl.acquire.bits.full_addr() | + Mux(is_multibeat, UInt(0), put_offset), + size = Mux(is_multibeat, UInt(log2Ceil(tlDataBytes)), put_size), len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) From 51f7bf1511bd0a5cad0f1e0372143fd8fa7aba9e Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 4 Jul 2016 16:58:49 -0700 Subject: [PATCH 653/688] fix Bufferless voluntary release issue --- uncore/src/main/scala/agents/Bufferless.scala | 6 +++--- uncore/src/main/scala/agents/Trackers.scala | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index eaa5011f..ecc5de00 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -74,9 +74,9 @@ class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Par // A release beat can be accepted if we are idle, if its a mergeable transaction, or if its a tail beat // and if the outer relase path is clear - val irel_could_accept = state === s_idle || irel_can_merge || irel_same_xact - io.inner.release.ready := irel_could_accept && - (!io.irel().hasData() || io.outer.release.ready) + io.inner.release.ready := Mux(io.irel().hasData(), + (state =/= s_idle) && (irel_can_merge || irel_same_xact) && io.outer.release.ready, + (state === s_idle) || irel_can_merge || irel_same_xact) // Dispatch outer release outerRelease(coh = outer_coh.onHit(M_XWR), buffering = Bool(false)) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 0efeac2a..9881d35d 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -244,11 +244,16 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { when(irel_is_allocating) { xact_addr_block := io.irel().addr_block + // Set all of them to pending in the beginning as a precaution + // If it turns out we don't need some or all of the beats, they will + // be overridden below + pending_irel_data := ~UInt(0, innerDataBeats) state := next } - when(io.inner.release.fire()) { - when(io.alloc.irel.should || (irel_can_merge && io.irel().first())) { + val irel_fire = (irel_is_allocating || irel_is_merging) && io.inner.release.ready + when (irel_fire) { + when (io.irel().first()) { xact_vol_ir_r_type := io.irel().r_type xact_vol_ir_src := io.irel().client_id xact_vol_ir_client_xact_id := io.irel().client_xact_id @@ -307,7 +312,7 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { pending_orel_data(vol_ognt_counter.up.idx), pending_orel_send), // only writebacks need to be forwarded to the outer interface - (io.alloc.irel.should || io.alloc.irel.matches) && + state =/= s_idle && io.alloc.irel.matches && io.irel().hasData() && io.inner.release.valid) io.outer.release.bits := coh.makeVoluntaryWriteback( From a35388bc2701f596acac27caee0aeaafd5d29aa1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 4 Jul 2016 17:00:03 -0700 Subject: [PATCH 654/688] fix merging of same xact ID puts/gets --- uncore/src/main/scala/agents/Broadcast.scala | 2 +- uncore/src/main/scala/agents/Cache.scala | 3 ++- uncore/src/main/scala/agents/Trackers.scala | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index 6c466a92..0922fa89 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -142,7 +142,7 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) can_alloc = Bool(false), next = s_inner_probe) - io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact_multibeat // Track which clients yet need to be probed and make Probe message // If a writeback occurs, we can forward its data via the buffer, diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 9adbf49c..109658cf 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -848,7 +848,8 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) can_alloc = Bool(true), next = s_meta_read) - io.inner.acquire.ready := state === s_idle || iacq_can_merge || iacq_same_xact + io.inner.acquire.ready := state === s_idle || iacq_can_merge || + iacq_same_xact_multibeat // Begin a transaction by getting the current block metadata // Defined here because of Chisel default wire demands, used in s_meta_resp diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 9881d35d..71c96f40 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -416,15 +416,17 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer (xact_iacq.client_xact_id === io.iacq().client_xact_id) && (xact_iacq.client_id === io.iacq().client_id) && pending_ignt + def iacq_same_xact_multibeat = iacq_same_xact && io.iacq().hasMultibeatData() def iacq_can_merge: Bool def iacq_is_allocating: Bool = state === s_idle && io.alloc.iacq.should && io.inner.acquire.valid def iacq_is_merging: Bool = (iacq_can_merge || iacq_same_xact) && io.inner.acquire.valid def innerAcquire(can_alloc: Bool, next: UInt) { + val iacq_matches_head = iacq_same_xact && xact_iacq.addr_beat === io.iacq().addr_beat // Enqueue some metadata information that we'll use to make coherence updates with later ignt_q.io.enq.valid := iacq_is_allocating || - (!iacq_same_xact && pending_ignt && + (!iacq_matches_head && pending_ignt && io.inner.acquire.fire() && io.iacq().first()) ignt_q.io.enq.bits := io.iacq() From 67bac383e37a7d35a406c23bc0366e80d01ef2c8 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 4 Jul 2016 17:00:27 -0700 Subject: [PATCH 655/688] hopefully fixed last bugs in Bufferless --- uncore/src/main/scala/agents/Bufferless.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index ecc5de00..93b75785 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -42,7 +42,9 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren allocs = trackerList.map(_.io.alloc.iacq), allocOverride = Some(!irel_vs_iacq_conflict)) io.outer.acquire.bits.data := io.inner.acquire.bits.data - io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat + when (io.oacq().hasData()) { + io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat + } // Handle releases, which might be voluntary and might have data doInputRoutingWithAllocation( @@ -50,7 +52,9 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren outs = trackerList.map(_.io.inner.release), allocs = trackerList.map(_.io.alloc.irel)) io.outer.release.bits.data := io.inner.release.bits.data - io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat + when (io.orel().hasData()) { + io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat + } // Wire probe requests and grant reply to clients, finish acks from clients doOutputArbitration(io.inner.probe, trackerList.map(_.io.inner.probe)) @@ -102,7 +106,7 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // transaction and (in case of PutBlock) subsequent tail beats val iacq_can_forward = iacq_same_xact && !vol_ognt_counter.pending io.inner.acquire.ready := Mux(io.iacq().hasData(), - state === s_outer_acquire && iacq_can_forward, + state === s_outer_acquire && iacq_can_forward && io.outer.acquire.ready, state === s_idle && io.alloc.iacq.should) // Track which clients yet need to be probed and make Probe message From 06ed9c57942a50def79a20b01b49ad58263e1c44 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Jul 2016 10:18:25 -0700 Subject: [PATCH 656/688] add a single-entry queue in front of acquire and release for bufferless broadcast hub --- uncore/src/main/scala/agents/Bufferless.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index 93b75785..f0554ceb 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -36,24 +36,26 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren io.inner.release.valid && io.irel().conflicts(io.iacq()) + val iacq = Queue(io.inner.acquire, 1, pipe=true) doInputRoutingWithAllocation( - in = io.inner.acquire, + in = iacq, outs = trackerList.map(_.io.inner.acquire), allocs = trackerList.map(_.io.alloc.iacq), allocOverride = Some(!irel_vs_iacq_conflict)) - io.outer.acquire.bits.data := io.inner.acquire.bits.data + io.outer.acquire.bits.data := iacq.bits.data when (io.oacq().hasData()) { - io.outer.acquire.bits.addr_beat := io.inner.acquire.bits.addr_beat + io.outer.acquire.bits.addr_beat := iacq.bits.addr_beat } // Handle releases, which might be voluntary and might have data + val irel = Queue(io.inner.release, 1, pipe=true) doInputRoutingWithAllocation( - in = io.inner.release, + in = irel, outs = trackerList.map(_.io.inner.release), allocs = trackerList.map(_.io.alloc.irel)) - io.outer.release.bits.data := io.inner.release.bits.data + io.outer.release.bits.data := irel.bits.data when (io.orel().hasData()) { - io.outer.release.bits.addr_beat := io.inner.release.bits.addr_beat + io.outer.release.bits.addr_beat := irel.bits.addr_beat } // Wire probe requests and grant reply to clients, finish acks from clients From 702444709a70c922a15bce529650520350d74de2 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Jul 2016 11:00:09 -0700 Subject: [PATCH 657/688] make sure pending bits updated for all releases --- uncore/src/main/scala/agents/Trackers.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 71c96f40..440dd495 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -262,7 +262,7 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { pending_irel_data := Mux(io.irel().hasMultibeatData(), dropPendingBitWhenBeatHasData(io.inner.release), UInt(0)) - } .elsewhen (irel_same_xact) { + } .otherwise { pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) } } From 4c07aedfad25a8964f2001c6aaf2eb2733fed737 Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Tue, 5 Jul 2016 14:21:21 -0700 Subject: [PATCH 658/688] Rewrite BRAMSlave to infer a single BRAM instance --- uncore/src/main/scala/devices/Bram.scala | 106 ++++++++++++----------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/uncore/src/main/scala/devices/Bram.scala b/uncore/src/main/scala/devices/Bram.scala index 1be27216..7e277192 100644 --- a/uncore/src/main/scala/devices/Bram.scala +++ b/uncore/src/main/scala/devices/Bram.scala @@ -10,74 +10,78 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters { val io = new ClientUncachedTileLinkIO().flip - val bram = SeqMem(depth, Vec(tlDataBytes, UInt(width = 8))) + val bram = SeqMem(depth, Bits(width = tlDataBits)) - val s_idle :: s_getblk :: s_putblk :: s_resp :: Nil = Enum(Bits(), 4) - val state = Reg(init = s_idle) + val fire_acq = io.acquire.fire() + val fire_gnt = io.grant.fire() - val acq = io.acquire.bits - val r_acq = Reg(new AcquireMetadata) + val state_getblk = Reg(init = Bool(false)) + val state_putblk = Reg(init = Bool(false)) + val state_init = !(state_getblk || state_putblk) - val (is_get :: is_getblk :: is_put :: is_putblk :: Nil) = Seq( - Acquire.getType, Acquire.getBlockType, Acquire.putType, Acquire.putBlockType - ).map(acq.isBuiltInType _) + private def last(acq: AcquireMetadata) = + (acq.addr_beat === UInt(tlDataBeats-1)) - val beats = Reg(UInt(width = tlBeatAddrBits)) + val s0_acq = io.acquire.bits + val s0_last = last(s0_acq) - when (io.acquire.fire()) { - r_acq := acq - when (is_get || is_put || acq.isPrefetch()) { - state := s_resp - } - when (is_getblk) { - beats := UInt(tlDataBeats - 1) - state := s_getblk - } - /** Need to collect the rest of the beats. - * Beat 0 has already been accepted. */ - when (is_putblk) { - beats := UInt(tlDataBeats - 2) - state := s_putblk - } - } + val s1_acq = RegEnable(s0_acq, fire_acq) + val s1_last = last(s1_acq) - when (state === s_getblk && io.grant.ready) { - r_acq.addr_beat := r_acq.addr_beat + UInt(1) - beats := beats - UInt(1) - when (beats === UInt(0)) { state := s_idle } - } + val (is_get :: is_getblk :: is_put :: is_putblk :: Nil) = + Seq(Acquire.getType, Acquire.getBlockType, + Acquire.putType, Acquire.putBlockType).map(s0_acq.isBuiltInType _) - when (state === s_putblk && io.acquire.valid) { - beats := beats - UInt(1) - when (beats === UInt(0)) { state := s_resp } - } + val is_read = is_get || is_getblk + val is_write = is_put || is_putblk + val ren_getblk = state_getblk && !s1_last - when (state === s_resp && io.grant.ready) { state := s_idle } + val s0_valid = (fire_acq && (!is_putblk || s0_last)) || ren_getblk + val s1_valid = RegNext(s0_valid, Bool(false)) - val acq_addr = Cat(acq.addr_block, acq.addr_beat) - val r_acq_addr = Cat(r_acq.addr_block, r_acq.addr_beat) + val ren = (fire_acq && is_read) || ren_getblk + val wen = (fire_acq && is_write) + + val s0_addr = Cat(s0_acq.addr_block, s0_acq.addr_beat) + val s1_addr_beat = s1_acq.addr_beat + Mux(io.grant.ready, UInt(1), UInt(0)) + val s1_addr = Cat(s1_acq.addr_block, s1_addr_beat) + + val raddr = Mux(state_getblk, s1_addr, s0_addr) + val waddr = s0_addr - val ren = (io.acquire.fire() && (is_get || is_getblk)) || - (state === s_getblk && io.grant.ready) - val raddr = Mux(state === s_idle, acq_addr, - Mux(io.grant.ready, r_acq_addr + UInt(1), r_acq_addr)) val rdata = bram.read(raddr, ren) + val wdata = s0_acq.data + val wmask = s0_acq.wmask() + when (wen) { + bram.write(waddr, wdata) + assert(wmask.andR, "BRAMSlave: partial write masks not supported") + } - val wen = (io.acquire.fire() && (is_put || is_putblk)) - val wdata = Vec.tabulate(tlDataBytes) { i => acq.data((i+1)*8-1, i*8) } - val wmask = Vec.tabulate(tlWriteMaskBits) { i => acq.wmask()(i) } + val stall = io.grant.valid && !io.grant.ready + io.acquire.ready := state_init && !stall - when (wen) { bram.write(acq_addr, wdata, wmask) } + when (fire_acq) { + state_getblk := is_getblk + state_putblk := is_putblk && s0_last + } - io.grant.valid := (state === s_resp) || (state === s_getblk) + when (state_getblk && fire_gnt) { + s1_acq.addr_beat := s1_addr_beat + state_getblk := !s1_last + } + + when (state_putblk && fire_gnt) { + state_putblk := Bool(false) + } + + io.grant.valid := s1_valid io.grant.bits := Grant( is_builtin_type = Bool(true), - g_type = r_acq.getBuiltInGrantType(), - client_xact_id = r_acq.client_xact_id, + g_type = s1_acq.getBuiltInGrantType(), + client_xact_id = s1_acq.client_xact_id, manager_xact_id = UInt(0), - addr_beat = r_acq.addr_beat, - data = rdata.toBits) - io.acquire.ready := (state === s_idle) || (state === s_putblk) + addr_beat = s1_acq.addr_beat, + data = rdata) } class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { From af7683797018c7dbdda660a58f52b70dbafaf430 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Jul 2016 16:03:52 -0700 Subject: [PATCH 659/688] conform to new NastiWriteDataChannel interface --- uncore/src/main/scala/converters/Nasti.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 734e0175..e646eef5 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -196,7 +196,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.w.bits := NastiWriteDataChannel( id = put_id_mapper.io.req.out_id, data = io.tl.acquire.bits.data, - strb = io.tl.acquire.bits.wmask(), + strb = Some(io.tl.acquire.bits.wmask()), last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) io.tl.acquire.ready := Mux(has_data, From b1050769960a4e693004b0afe0da68651cee2c18 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 5 Jul 2016 17:19:35 -0700 Subject: [PATCH 660/688] fix ID mapper to disallow two in-flight requests with the same inner ID --- uncore/src/main/scala/converters/Nasti.scala | 75 +++++++++----------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index e646eef5..91cdfbc5 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -6,8 +6,10 @@ import uncore.tilelink._ import uncore.constants._ import cde.Parameters -class IdMapper(val inIdBits: Int, val outIdBits: Int) - (implicit val p: Parameters) extends Module { +class IdMapper(val inIdBits: Int, val outIdBits: Int, + val forceMapping: Boolean = false) + (implicit val p: Parameters) extends Module { + val io = new Bundle { val req = new Bundle { val valid = Bool(INPUT) @@ -18,51 +20,44 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int) val resp = new Bundle { val valid = Bool(INPUT) val matches = Bool(OUTPUT) - val out_id = UInt(INPUT, inIdBits) - val in_id = UInt(OUTPUT, outIdBits) + val out_id = UInt(INPUT, outIdBits) + val in_id = UInt(OUTPUT, inIdBits) } } val maxInXacts = 1 << inIdBits - if (inIdBits <= outIdBits) { + if (inIdBits <= outIdBits && !forceMapping) { io.req.ready := Bool(true) io.req.out_id := io.req.in_id io.resp.matches := Bool(true) io.resp.in_id := io.resp.out_id - } else if (outIdBits <= 2) { - val nQueues = 1 << outIdBits - val entriesPerQueue = (maxInXacts - 1) / nQueues + 1 - val (req_out_id, req_out_flip) = Counter(io.req.valid && io.req.ready, nQueues) - io.req.ready := Bool(false) - io.resp.matches := Bool(false) - io.resp.in_id := UInt(0) - io.req.out_id := req_out_id - for (i <- 0 until nQueues) { - val queue = Module(new Queue(UInt(width = inIdBits), entriesPerQueue)) - queue.io.enq.valid := io.req.valid && req_out_id === UInt(i) - queue.io.enq.bits := io.req.in_id - when (req_out_id === UInt(i)) { io.req.ready := queue.io.enq.ready } - - queue.io.deq.ready := io.resp.valid && io.resp.out_id === UInt(i) - when (io.resp.out_id === UInt(i)) { - io.resp.matches := queue.io.deq.valid - io.resp.in_id := queue.io.deq.bits - } - } } else { - val maxOutId = 1 << outIdBits - val (req_out_id, req_nasti_flip) = Counter(io.req.valid && io.req.ready, maxOutId) - val roq = Module(new ReorderQueue( - UInt(width = inIdBits), outIdBits, maxInXacts)) - roq.io.enq.valid := io.req.valid - roq.io.enq.bits.data := io.req.in_id - roq.io.enq.bits.tag := req_out_id - io.req.ready := roq.io.enq.ready - io.req.out_id := req_out_id - roq.io.deq.valid := io.resp.valid - roq.io.deq.tag := io.resp.out_id - io.resp.in_id := roq.io.deq.data - io.resp.matches := roq.io.deq.matches + val nInXacts = 1 << inIdBits + val nOutXacts = 1 << outIdBits + + val out_id_free = Reg(init = Vec.fill(nOutXacts){Bool(true)}) + val next_out_id = PriorityEncoder(out_id_free) + val id_mapping = Reg(Vec(nInXacts, UInt(0, outIdBits))) + val id_valid = Reg(init = Vec.fill(nOutXacts){Bool(false)}) + + val req_fire = io.req.valid && io.req.ready + when (req_fire) { + out_id_free(io.req.out_id) := Bool(false) + id_valid(io.req.in_id) := Bool(true) + id_mapping(io.req.in_id) := io.req.out_id + } + when (io.resp.valid) { + out_id_free(io.resp.out_id) := Bool(true) + id_valid(io.resp.in_id) := Bool(false) + } + + io.req.ready := out_id_free.reduce(_ || _) && !id_valid(io.req.in_id) + io.req.out_id := next_out_id + + val id_matches = id_mapping.map(_ === io.resp.out_id) + val id_matches_valid = id_matches.zip(id_valid).map { case (m, v) => m && v } + io.resp.matches := id_matches_valid.reduce(_ || _) + io.resp.in_id := PriorityEncoder(id_matches_valid) } } @@ -309,8 +304,8 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) "NASTI write transaction cannot convert to TileLInk") val put_count = Reg(init = UInt(0, tlBeatAddrBits)) - val get_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits)) - val put_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits)) + val get_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits, true)) + val put_id_mapper = Module(new IdMapper(nastiXIdBits, tlClientXactIdBits, true)) when (io.nasti.aw.fire()) { aw_req := io.nasti.aw.bits From 64afc795fd7d9068497ae51bca6e9cd1ce844a18 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 6 Jul 2016 14:10:45 -0700 Subject: [PATCH 661/688] make sure voluntary releases don't get allocated to L2WritebackUnit --- uncore/src/main/scala/agents/Broadcast.scala | 6 ++---- uncore/src/main/scala/agents/Bufferless.scala | 6 ++---- uncore/src/main/scala/agents/Cache.scala | 9 +++++---- uncore/src/main/scala/agents/Trackers.scala | 11 +++++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index 0922fa89..d5c36f59 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -108,8 +108,7 @@ class BufferedBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Param with HasDataBuffer { // Tell the parent if any incoming messages conflict with the ongoing transaction - routeInParent() - io.alloc.iacq.can := Bool(false) + routeInParent(irelCanAlloc = Bool(true)) // Start transaction by accepting inner release innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) @@ -133,8 +132,7 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) with HasByteWriteMaskBuffer { // Setup IOs used for routing in the parent - routeInParent() - io.alloc.irel.can := Bool(false) + routeInParent(iacqCanAlloc = Bool(true)) // First, take care of accpeting new acquires or secondary misses // Handling of primary and secondary misses' data and write mask merging diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index f0554ceb..e108ca72 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -72,8 +72,7 @@ class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Par extends BroadcastVoluntaryReleaseTracker(trackerId)(p) { // Tell the parent if any incoming messages conflict with the ongoing transaction - routeInParent() - io.alloc.iacq.can := Bool(false) + routeInParent(irelCanAlloc = Bool(true)) // Start transaction by accepting inner release innerRelease(block_vol_ignt = pending_orel || vol_ognt_counter.pending) @@ -94,8 +93,7 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) extends BroadcastAcquireTracker(trackerId)(p) { // Setup IOs used for routing in the parent - routeInParent() - io.alloc.irel.can := Bool(false) + routeInParent(iacqCanAlloc = Bool(true)) // First, take care of accpeting new acquires or secondary misses // Handling of primary and secondary misses' data and write mask merging diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 109658cf..be41e312 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -729,8 +729,9 @@ class CacheVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) pinAllReadyValidLow(io) // Avoid metatdata races with writebacks - routeInParent(iacqMatches = inSameSet(_, xact_addr_block)) - io.alloc.iacq.can := Bool(false) + routeInParent( + iacqMatches = inSameSet(_, xact_addr_block), + irelCanAlloc = Bool(true)) // Initialize and accept pending Release beats innerRelease( @@ -811,8 +812,8 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) routeInParent( iacqMatches = inSameSet(_, xact_addr_block), irelMatches = (irel: HasCacheBlockAddress) => - Mux(before_wb_alloc, inSameSet(irel, xact_addr_block), exactAddrMatch(irel))) - io.alloc.irel.can := Bool(false) + Mux(before_wb_alloc, inSameSet(irel, xact_addr_block), exactAddrMatch(irel)), + iacqCanAlloc = Bool(true)) // TileLink allows for Gets-under-Get // and Puts-under-Put, and either may also merge with a preceding prefetch diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 440dd495..bf7c7b7a 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -380,13 +380,16 @@ trait RoutesInParent extends HasBlockAddressBuffer def exactAddrMatch(a: HasCacheBlockAddress): Bool = a.conflicts(xact_addr_block) def routeInParent(iacqMatches: AddrComparison = exactAddrMatch, irelMatches: AddrComparison = exactAddrMatch, - oprbMatches: AddrComparison = exactAddrMatch) { + oprbMatches: AddrComparison = exactAddrMatch, + iacqCanAlloc: Bool = Bool(false), + irelCanAlloc: Bool = Bool(false), + oprbCanAlloc: Bool = Bool(false)) { io.alloc.iacq.matches := (state =/= s_idle) && iacqMatches(io.iacq()) io.alloc.irel.matches := (state =/= s_idle) && irelMatches(io.irel()) io.alloc.oprb.matches := (state =/= s_idle) && oprbMatches(io.oprb()) - io.alloc.iacq.can := state === s_idle - io.alloc.irel.can := state === s_idle - io.alloc.oprb.can := Bool(false) + io.alloc.iacq.can := state === s_idle && iacqCanAlloc + io.alloc.irel.can := state === s_idle && irelCanAlloc + io.alloc.oprb.can := state === s_idle && oprbCanAlloc } } From b10d306b4ad3e3d99428c42923cf2ba196fcfa50 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 6 Jul 2016 14:59:09 -0700 Subject: [PATCH 662/688] add option to log L2 cache transactions for easier debugging --- uncore/src/main/scala/agents/Trackers.scala | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index bf7c7b7a..3db62194 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -6,9 +6,11 @@ import Chisel._ import uncore.coherence._ import uncore.tilelink._ import uncore.util._ -import cde.Parameters +import cde.{Field, Parameters} import scala.math.max +case object EnableL2Logging extends Field[Boolean] + class TrackerAllocation extends Bundle { val matches = Bool(OUTPUT) val can = Bool(OUTPUT) @@ -265,6 +267,12 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { } .otherwise { pending_irel_data := (pending_irel_data & dropPendingBitWhenBeatHasData(io.inner.release)) } + if (p(EnableL2Logging)) { + when (io.irel().hasData()) { + printf("[release] addr_block=%x addr_beat=%d data=%x\n", + io.irel().addr_block, io.irel().addr_beat, io.irel().data) + } + } } io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) && @@ -486,6 +494,13 @@ trait AcceptsInnerAcquires extends HasAcquireMetadataBuffer add_pending_bits } + if (p(EnableL2Logging)) { + when (io.inner.grant.fire() && io.ignt().hasData()) { + printf("[get] addr_block=%x addr_beat=%d data=%x\n", + xact_addr_block, io.ignt().addr_beat, io.ignt().data) + } + } + // Have we finished receiving the complete inner acquire transaction? val iacq_finished = !(state === s_idle || state === s_meta_read || From 5d8d5e598b580420d4c93e70cedad75654e93c73 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 6 Jul 2016 16:51:45 -0700 Subject: [PATCH 663/688] add buffering and locking to TL -> Nasti converter --- uncore/src/main/scala/converters/Nasti.scala | 100 ++++++++++++++----- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 91cdfbc5..0a7632d9 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -4,7 +4,9 @@ import Chisel._ import junctions._ import uncore.tilelink._ import uncore.constants._ -import cde.Parameters +import cde.{Field, Parameters} + +case object NastiResponseBufferDepth extends Field[Int] class IdMapper(val inIdBits: Int, val outIdBits: Int, val forceMapping: Boolean = false) @@ -61,6 +63,48 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int, } } +class NastiReadBuffer(implicit p: Parameters) extends NastiModule()(p) { + val bufferDepth = p(NastiResponseBufferDepth) + val io = new Bundle { + val in = new NastiReadIO().flip + val out = new NastiReadIO + } + + io.in.r <> Queue(io.out.r, bufferDepth) + + val inflight = Reg(init = UInt(0, width = log2Up(bufferDepth + 1))) + + val ar_delta = Mux(io.in.ar.fire(), io.in.ar.bits.len + UInt(1), UInt(0)) + val r_delta = Mux(io.in.r.fire(), UInt(1), UInt(0)) + inflight := inflight + ar_delta - r_delta + + val can_request = (inflight + io.in.ar.bits.len) < UInt(bufferDepth) + io.in.ar.ready := io.out.ar.ready && can_request + io.out.ar.valid := io.in.ar.valid && can_request + io.out.ar.bits := io.in.ar.bits +} + +class NastiWriteBuffer(implicit p: Parameters) extends NastiModule()(p) { + val bufferDepth = p(NastiResponseBufferDepth) + val io = new Bundle { + val in = new NastiWriteIO().flip + val out = new NastiWriteIO + } + + io.in.b <> Queue(io.out.b, bufferDepth) + io.out.w <> io.in.w + + val inflight = Reg(init = UInt(0, width = log2Up(bufferDepth + 1))) + val aw_delta = Mux(io.in.aw.fire(), UInt(1), UInt(0)) + val b_delta = Mux(io.in.b.fire(), UInt(1), UInt(0)) + inflight := inflight + aw_delta - b_delta + + val can_request = inflight < UInt(bufferDepth) + io.in.aw.ready := io.out.aw.ready && can_request + io.out.aw.valid := io.in.aw.valid && can_request + io.out.aw.bits := io.in.aw.bits +} + class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { val addr_beat = UInt(width = tlBeatAddrBits) val subblock = Bool() @@ -111,47 +155,47 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask + val get_buffer = Module(new NastiReadBuffer) + val put_buffer = Module(new NastiWriteBuffer) + // For Get/GetBlock, make sure Reorder queue can accept new entry val get_helper = DecoupledHelper( get_valid, roq.io.enq.ready, - io.nasti.ar.ready, + get_buffer.io.in.ar.ready, get_id_ready) val w_inflight = Reg(init = Bool(false)) // For Put/PutBlock, make sure aw and w channel are both ready before // we send the first beat - val aw_ready = w_inflight || io.nasti.aw.ready + val aw_ready = w_inflight || put_buffer.io.in.aw.ready val put_helper = DecoupledHelper( put_valid, aw_ready, - io.nasti.w.ready, + put_buffer.io.in.w.ready, put_id_ready) - val (nasti_cnt_out, nasti_wrap_out) = Counter( - io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) - roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) - roq.io.enq.bits.tag := io.nasti.ar.bits.id + roq.io.enq.bits.tag := get_id_mapper.io.req.out_id roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat roq.io.enq.bits.data.subblock := is_subblock - roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) - roq.io.deq.tag := io.nasti.r.bits.id + roq.io.deq.valid := get_buffer.io.in.r.fire() && get_buffer.io.in.r.bits.last + roq.io.deq.tag := get_buffer.io.in.r.bits.id get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) get_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id - get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last - get_id_mapper.io.resp.out_id := io.nasti.r.bits.id + get_id_mapper.io.resp.valid := get_buffer.io.in.r.fire() && get_buffer.io.in.r.bits.last + get_id_mapper.io.resp.out_id := get_buffer.io.in.r.bits.id put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) put_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id - put_id_mapper.io.resp.valid := io.nasti.b.fire() - put_id_mapper.io.resp.out_id := io.nasti.b.bits.id + put_id_mapper.io.resp.valid := put_buffer.io.in.b.fire() + put_id_mapper.io.resp.out_id := put_buffer.io.in.b.bits.id // Decompose outgoing TL Acquires into Nasti address and data channels - io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) - io.nasti.ar.bits := NastiReadAddressChannel( + get_buffer.io.in.ar.valid := get_helper.fire(get_buffer.io.in.ar.ready) + get_buffer.io.in.ar.bits := NastiReadAddressChannel( id = get_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr(), size = Mux(is_subblock, @@ -179,16 +223,16 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val all_inside_0 = (~io.tl.acquire.bits.wmask()).toBools val (_, put_offset, put_size) = mask_helper(all_inside_0, 0) - io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) - io.nasti.aw.bits := NastiWriteAddressChannel( + put_buffer.io.in.aw.valid := put_helper.fire(aw_ready, !w_inflight) + put_buffer.io.in.aw.bits := NastiWriteAddressChannel( id = put_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr() | Mux(is_multibeat, UInt(0), put_offset), size = Mux(is_multibeat, UInt(log2Ceil(tlDataBytes)), put_size), len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) - io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) - io.nasti.w.bits := NastiWriteDataChannel( + put_buffer.io.in.w.valid := put_helper.fire(put_buffer.io.in.w.ready) + put_buffer.io.in.w.bits := NastiWriteDataChannel( id = put_id_mapper.io.req.out_id, data = io.tl.acquire.bits.data, strb = Some(io.tl.acquire.bits.wmask()), @@ -209,11 +253,12 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) // Aggregate incoming NASTI responses into TL Grants val (tl_cnt_in, tl_wrap_in) = Counter( io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + val gnt_arb = Module(new LockingArbiter(new GrantToDst, 2, + tlDataBeats, Some((gnt: GrantToDst) => gnt.hasMultibeatData()))) io.tl.grant <> gnt_arb.io.out - gnt_arb.io.in(0).valid := io.nasti.r.valid - io.nasti.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).valid := get_buffer.io.in.r.valid + get_buffer.io.in.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( is_builtin_type = Bool(true), g_type = Mux(roq.io.deq.data.subblock, @@ -221,15 +266,15 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) client_xact_id = get_id_mapper.io.resp.in_id, manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = io.nasti.r.bits.data) + data = get_buffer.io.in.r.bits.data) assert(!roq.io.deq.valid || roq.io.deq.matches, "TL -> NASTI converter ReorderQueue: NASTI tag error") assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, "TL -> NASTI ID Mapper: NASTI tag error") - gnt_arb.io.in(1).valid := io.nasti.b.valid - io.nasti.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).valid := put_buffer.io.in.b.valid + put_buffer.io.in.b.ready := gnt_arb.io.in(1).ready gnt_arb.io.in(1).bits := Grant( is_builtin_type = Bool(true), g_type = Grant.putAckType, @@ -241,6 +286,9 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") + + io.nasti <> get_buffer.io.out + io.nasti <> put_buffer.io.out } class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) From 8ccc50a8f045ada44162374959522fd743df7a37 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 10:16:44 -0700 Subject: [PATCH 664/688] fix IdMapper and TL -> NASTI converter --- uncore/src/main/scala/converters/Nasti.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 0a7632d9..9f6a10a0 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -40,7 +40,7 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int, val out_id_free = Reg(init = Vec.fill(nOutXacts){Bool(true)}) val next_out_id = PriorityEncoder(out_id_free) val id_mapping = Reg(Vec(nInXacts, UInt(0, outIdBits))) - val id_valid = Reg(init = Vec.fill(nOutXacts){Bool(false)}) + val id_valid = Reg(init = Vec.fill(nInXacts){Bool(false)}) val req_fire = io.req.valid && io.req.ready when (req_fire) { @@ -236,7 +236,8 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) id = put_id_mapper.io.req.out_id, data = io.tl.acquire.bits.data, strb = Some(io.tl.acquire.bits.wmask()), - last = tl_wrap_out || (io.tl.acquire.fire() && is_subblock)) + last = Mux(w_inflight, + tl_cnt_out === UInt(tlDataBeats - 1), !is_multibeat)) io.tl.acquire.ready := Mux(has_data, put_helper.fire(put_valid), From 1c5e7be75be5443eb1380f69d3fce4395b727eb3 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 12:14:02 -0700 Subject: [PATCH 665/688] make sure Nasti write channel id is set correctly --- uncore/src/main/scala/converters/Nasti.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 9f6a10a0..587c77a9 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -166,6 +166,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) get_id_ready) val w_inflight = Reg(init = Bool(false)) + val w_id = Reg(init = UInt(0, nastiXIdBits)) // For Put/PutBlock, make sure aw and w channel are both ready before // we send the first beat @@ -233,7 +234,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) put_buffer.io.in.w.valid := put_helper.fire(put_buffer.io.in.w.ready) put_buffer.io.in.w.bits := NastiWriteDataChannel( - id = put_id_mapper.io.req.out_id, + id = w_id, data = io.tl.acquire.bits.data, strb = Some(io.tl.acquire.bits.wmask()), last = Mux(w_inflight, @@ -245,6 +246,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) when (!w_inflight && io.tl.acquire.fire() && is_multibeat) { w_inflight := Bool(true) + w_id := put_id_mapper.io.req.out_id } when (w_inflight) { From 7cc64011fb193c99bbca8ee55fe2730a302f7114 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 12:14:45 -0700 Subject: [PATCH 666/688] simplify amo_mask generation --- uncore/src/main/scala/tilelink/Definitions.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/tilelink/Definitions.scala b/uncore/src/main/scala/tilelink/Definitions.scala index 608f6042..a5fa4b55 100644 --- a/uncore/src/main/scala/tilelink/Definitions.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -183,8 +183,9 @@ trait HasAcquireUnion extends HasTileLinkParameters { /** Write mask for [[uncore.Put]], [[uncore.PutBlock]], [[uncore.PutAtomic]] */ def wmask(dummy: Int = 0): UInt = { val is_amo = isBuiltInType(Acquire.putAtomicType) - val amo_sel = if (tlByteAddrBits > log2Up(amoAluOperandBytes)) UIntToOH(amo_offset()) else UInt(1) - val amo_mask = FillInterleaved(amoAluOperandBytes, amo_sel) + val amo_mask = if (tlByteAddrBits > log2Up(amoAluOperandBytes)) + FillInterleaved(amoAluOperandBytes, UIntToOH(amo_offset())) + else Acquire.fullWriteMask val is_put = isBuiltInType(Acquire.putBlockType) || isBuiltInType(Acquire.putType) val put_mask = union(tlWriteMaskBits, 1) Mux(is_amo, amo_mask, Mux(is_put, put_mask, UInt(0))) From 16a6b1108123d18c7ae51e8f3ab0dbafdd132bff Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 14:34:24 -0700 Subject: [PATCH 667/688] fix bug in AXI -> TL converter --- uncore/src/main/scala/converters/Nasti.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 587c77a9..84ec2d47 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -428,14 +428,14 @@ class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) io.nasti.b.valid := io.tl.grant.valid && tl_b_grant(io.tl.grant.bits) io.nasti.b.bits := NastiWriteResponseChannel( - id = put_id_mapper.io.req.out_id) + id = put_id_mapper.io.resp.in_id) assert(!io.nasti.b.valid || put_id_mapper.io.resp.matches, "Put ID does not match") io.nasti.r.valid := io.tl.grant.valid && !tl_b_grant(io.tl.grant.bits) io.nasti.r.bits := NastiReadDataChannel( - id = get_id_mapper.io.req.out_id, + id = get_id_mapper.io.resp.in_id, data = io.tl.grant.bits.data, last = tl_last(io.tl.grant.bits)) From 70b677ecdad4d542e0333ab4152ad8c07330fb14 Mon Sep 17 00:00:00 2001 From: Andrew Waterman Date: Thu, 7 Jul 2016 19:25:57 -0700 Subject: [PATCH 668/688] Vec considered harmful; use isOneOf instead (#64) Vec is heavyweight and really should only be used for I/O and dynamic indexing. A recurring pattern in uncore is Vec(const1, const2, const3) contains x which is nice but has a deleterious effect on simulation copilation and execution time. This patch proposes an alternative: x isOneOf (const1, const2, const3) x isOneOf seqOfThings I think it's also more idiomatic. This is just a prototype; I'm not wed to the name or implementation. --- uncore/src/main/scala/Util.scala | 11 +++ uncore/src/main/scala/agents/Broadcast.scala | 3 +- uncore/src/main/scala/agents/Cache.scala | 5 +- uncore/src/main/scala/agents/Trackers.scala | 3 +- .../src/main/scala/coherence/Policies.scala | 87 ++++++++++--------- .../src/main/scala/tilelink/Definitions.scala | 21 ++--- 6 files changed, 73 insertions(+), 57 deletions(-) create mode 100644 uncore/src/main/scala/Util.scala diff --git a/uncore/src/main/scala/Util.scala b/uncore/src/main/scala/Util.scala new file mode 100644 index 00000000..aceee5c2 --- /dev/null +++ b/uncore/src/main/scala/Util.scala @@ -0,0 +1,11 @@ +package uncore + +import Chisel._ + +package object Util { + implicit class UIntIsOneOf(val x: UInt) extends AnyVal { + def isOneOf(s: Seq[UInt]): Bool = s.map(x === _).reduce(_||_) + + def isOneOf(u1: UInt, u2: UInt*): Bool = isOneOf(u1 +: u2.toSeq) + } +} diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index d5c36f59..86e02c5f 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -6,6 +6,7 @@ import Chisel._ import uncore.coherence._ import uncore.tilelink._ import uncore.constants._ +import uncore.Util._ import cde.Parameters class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { @@ -155,7 +156,7 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // and/or write back dirty data, and may be unexpected voluntary releases def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && - !Vec(s_idle, s_meta_write).contains(state) && + !state.isOneOf(s_idle, s_meta_write) && !all_pending_done && !io.outer.grant.fire() && !io.inner.grant.fire() && diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index be41e312..171555ff 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -9,6 +9,7 @@ import uncore.util.AMOALU import uncore.coherence._ import uncore.tilelink._ import uncore.constants._ +import uncore.Util._ import cde.{Parameters, Field} case object CacheName extends Field[String] @@ -807,7 +808,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) // Setup IOs used for routing in the parent - val before_wb_alloc = Vec(s_meta_read, s_meta_resp, s_wb_req).contains(state) + val before_wb_alloc = state isOneOf (s_meta_read, s_meta_resp, s_wb_req) routeInParent( iacqMatches = inSameSet(_, xact_addr_block), @@ -899,7 +900,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && - !Vec(s_idle, s_meta_read, s_meta_resp, s_meta_write).contains(state) && + !state.isOneOf(s_idle, s_meta_read, s_meta_resp, s_meta_write) && !all_pending_done && !io.outer.grant.fire() && !io.inner.grant.fire() && diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 3db62194..4d23a4cb 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -6,6 +6,7 @@ import Chisel._ import uncore.coherence._ import uncore.tilelink._ import uncore.util._ +import uncore.Util._ import cde.{Field, Parameters} import scala.math.max @@ -275,7 +276,7 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { } } - io.inner.grant.valid := Vec(s_wb_req, s_wb_resp, s_inner_probe, s_busy).contains(state) && + io.inner.grant.valid := state.isOneOf(s_wb_req, s_wb_resp, s_inner_probe, s_busy) && vol_ignt_counter.pending && !(pending_irel_data.orR || block_vol_ignt) diff --git a/uncore/src/main/scala/coherence/Policies.scala b/uncore/src/main/scala/coherence/Policies.scala index a28e4177..d4d04b71 100644 --- a/uncore/src/main/scala/coherence/Policies.scala +++ b/uncore/src/main/scala/coherence/Policies.scala @@ -5,6 +5,7 @@ package uncore.coherence import Chisel._ import uncore.tilelink._ import uncore.constants._ +import uncore.Util._ /** The entire CoherencePolicy API consists of the following three traits: * HasCustomTileLinkMessageTypes, used to define custom messages @@ -32,8 +33,8 @@ trait HasCustomTileLinkMessageTypes { def grantTypeWidth = log2Up(nGrantTypes) val acquireTypesWithData = Nil // Only built-in Acquire types have data for now - def releaseTypesWithData: Vec[UInt] - def grantTypesWithData: Vec[UInt] + def releaseTypesWithData: Seq[UInt] + def grantTypesWithData: Seq[UInt] } /** This API contains all functions required for client coherence agents. @@ -47,16 +48,16 @@ trait HasClientSideCoherencePolicy { // Client coherence states and their permissions val nClientStates: Int def clientStateWidth = log2Ceil(nClientStates) - def clientStatesWithReadPermission: Vec[UInt] - def clientStatesWithWritePermission: Vec[UInt] - def clientStatesWithDirtyData: Vec[UInt] + def clientStatesWithReadPermission: Seq[UInt] + def clientStatesWithWritePermission: Seq[UInt] + def clientStatesWithDirtyData: Seq[UInt] // Transaction initiation logic def isValid(meta: ClientMetadata): Bool def isHit(cmd: UInt, meta: ClientMetadata): Bool = { Mux(isWriteIntent(cmd), - clientStatesWithWritePermission.contains(meta.state), - clientStatesWithReadPermission.contains(meta.state)) + meta.state isOneOf clientStatesWithWritePermission, + meta.state isOneOf clientStatesWithReadPermission) } //TODO: Assumes all states with write permissions also have read permissions def requiresAcquireOnSecondaryMiss( @@ -68,7 +69,7 @@ trait HasClientSideCoherencePolicy { //TODO: Assumes all cache ctrl ops writeback dirty data, and // doesn't issue transaction when e.g. downgrading Exclusive to Shared: def requiresReleaseOnCacheControl(cmd: UInt, meta: ClientMetadata): Bool = - clientStatesWithDirtyData.contains(meta.state) + meta.state isOneOf clientStatesWithDirtyData // Determine which custom message type to use def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt @@ -131,23 +132,23 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseCopyData :: releaseInvalidateAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - def releaseTypesWithData = Vec(releaseInvalidateData, releaseCopyData) - def grantTypesWithData = Vec(grantExclusive) + def releaseTypesWithData = Seq(releaseInvalidateData, releaseCopyData) + def grantTypesWithData = Seq(grantExclusive) // Client states and functions val nClientStates = 2 val clientInvalid :: clientValid :: Nil = Enum(UInt(), nClientStates) - def clientStatesWithReadPermission = Vec(clientValid) - def clientStatesWithWritePermission = Vec(clientValid) - def clientStatesWithDirtyData = Vec(clientValid) + def clientStatesWithReadPermission = Seq(clientValid) + def clientStatesWithWritePermission = Seq(clientValid) + def clientStatesWithDirtyData = Seq(clientValid) def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid def getAcquireType(cmd: UInt, meta: ClientMetadata): UInt = acquireExclusive def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData MuxLookup(cmd, releaseCopyAck, Array( M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), M_PRODUCE -> Mux(dirty, releaseCopyData, releaseCopyAck), @@ -222,16 +223,16 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantExclusive :: Nil = Enum(UInt(), nGrantTypes) - def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - def grantTypesWithData = Vec(grantExclusive) + def releaseTypesWithData = Seq(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Seq(grantExclusive) // Client states and functions val nClientStates = 3 val clientInvalid :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def clientStatesWithReadPermission = Vec(clientExclusiveClean, clientExclusiveDirty) - def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - def clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Seq(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithWritePermission = Seq(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithDirtyData = Seq(clientExclusiveDirty) def isValid (meta: ClientMetadata) = meta.state =/= clientInvalid @@ -239,7 +240,7 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(isWriteIntent(cmd), acquireExclusiveDirty, acquireExclusiveClean) def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData MuxLookup(cmd, releaseCopyAck, Array( M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), @@ -324,16 +325,16 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - def grantTypesWithData = Vec(grantShared, grantExclusive) + def releaseTypesWithData = Seq(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Seq(grantShared, grantExclusive) // Client states and functions val nClientStates = 3 val clientInvalid :: clientShared :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveDirty) - def clientStatesWithWritePermission = Vec(clientExclusiveDirty) - def clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Seq(clientShared, clientExclusiveDirty) + def clientStatesWithWritePermission = Seq(clientExclusiveDirty) + def clientStatesWithDirtyData = Seq(clientExclusiveDirty) def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid @@ -341,7 +342,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData MuxLookup(cmd, releaseCopyAck, Array( M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), @@ -361,7 +362,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata( MuxLookup(cmd, meta.state, Array( M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + M_PRODUCE -> Mux(meta.state isOneOf clientStatesWithWritePermission, clientShared, meta.state))))(meta.p) def clientMetadataOnGrant(incoming: HasGrantType, cmd: UInt, meta: ClientMetadata) = @@ -442,16 +443,16 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: Nil = Enum(UInt(), nGrantTypes) - def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData) - def grantTypesWithData = Vec(grantShared, grantExclusive) + def releaseTypesWithData = Seq(releaseInvalidateData, releaseDowngradeData, releaseCopyData) + def grantTypesWithData = Seq(grantShared, grantExclusive) // Client states and functions val nClientStates = 4 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: Nil = Enum(UInt(), nClientStates) - def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty) - def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty) - def clientStatesWithDirtyData = Vec(clientExclusiveDirty) + def clientStatesWithReadPermission = Seq(clientShared, clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithWritePermission = Seq(clientExclusiveClean, clientExclusiveDirty) + def clientStatesWithDirtyData = Seq(clientExclusiveDirty) def isValid(meta: ClientMetadata): Bool = meta.state =/= clientInvalid @@ -459,7 +460,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { Mux(isWriteIntent(cmd), acquireExclusive, acquireShared) def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData MuxLookup(cmd, releaseCopyAck, Array( M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), @@ -479,7 +480,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata( MuxLookup(cmd, meta.state, Array( M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + M_PRODUCE -> Mux(meta.state isOneOf clientStatesWithWritePermission, clientShared, meta.state), M_CLEAN -> Mux(meta.state === clientExclusiveDirty, clientExclusiveClean, meta.state))))(meta.p) @@ -558,16 +559,16 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d val releaseInvalidateData :: releaseDowngradeData :: releaseCopyData :: releaseInvalidateAck :: releaseDowngradeAck :: releaseCopyAck :: releaseDowngradeDataMigratory :: releaseDowngradeAckHasCopy :: releaseInvalidateDataMigratory :: releaseInvalidateAckMigratory :: Nil = Enum(UInt(), nReleaseTypes) val grantShared :: grantExclusive :: grantExclusiveAck :: grantReadMigratory :: Nil = Enum(UInt(), nGrantTypes) - def releaseTypesWithData = Vec(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) - def grantTypesWithData = Vec(grantShared, grantExclusive, grantReadMigratory) + def releaseTypesWithData = Seq(releaseInvalidateData, releaseDowngradeData, releaseCopyData, releaseInvalidateDataMigratory, releaseDowngradeDataMigratory) + def grantTypesWithData = Seq(grantShared, grantExclusive, grantReadMigratory) // Client states and functions val nClientStates = 7 val clientInvalid :: clientShared :: clientExclusiveClean :: clientExclusiveDirty :: clientSharedByTwo :: clientMigratoryClean :: clientMigratoryDirty :: Nil = Enum(UInt(), nClientStates) - def clientStatesWithReadPermission = Vec(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) - def clientStatesWithWritePermission = Vec(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) - def clientStatesWithDirtyData = Vec(clientExclusiveDirty, clientMigratoryDirty) + def clientStatesWithReadPermission = Seq(clientShared, clientExclusiveClean, clientExclusiveDirty, clientSharedByTwo, clientMigratoryClean, clientMigratoryDirty) + def clientStatesWithWritePermission = Seq(clientExclusiveClean, clientExclusiveDirty, clientMigratoryClean, clientMigratoryDirty) + def clientStatesWithDirtyData = Seq(clientExclusiveDirty, clientMigratoryDirty) def isValid (meta: ClientMetadata): Bool = meta.state =/= clientInvalid @@ -577,7 +578,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d acquireShared) def getReleaseType(cmd: UInt, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData MuxLookup(cmd, releaseCopyAck, Array( M_FLUSH -> Mux(dirty, releaseInvalidateData, releaseInvalidateAck), M_PRODUCE -> Mux(dirty, releaseDowngradeData, releaseDowngradeAck), @@ -585,9 +586,9 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d } def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = { - val dirty = clientStatesWithDirtyData.contains(meta.state) + val dirty = meta.state isOneOf clientStatesWithDirtyData val with_data = MuxLookup(incoming.p_type, releaseInvalidateData, Array( - probeInvalidate -> Mux(Vec(clientExclusiveDirty, clientMigratoryDirty).contains(meta.state), + probeInvalidate -> Mux(meta.state isOneOf (clientExclusiveDirty, clientMigratoryDirty), releaseInvalidateDataMigratory, releaseInvalidateData), probeDowngrade -> Mux(meta.state === clientMigratoryDirty, releaseDowngradeDataMigratory, releaseDowngradeData), @@ -614,7 +615,7 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d ClientMetadata( MuxLookup(cmd, meta.state, Array( M_FLUSH -> clientInvalid, - M_PRODUCE -> Mux(clientStatesWithWritePermission.contains(meta.state), + M_PRODUCE -> Mux(meta.state isOneOf clientStatesWithWritePermission, clientShared, meta.state), M_CLEAN -> MuxLookup(meta.state, meta.state, Array( clientExclusiveDirty -> clientExclusiveClean, diff --git a/uncore/src/main/scala/tilelink/Definitions.scala b/uncore/src/main/scala/tilelink/Definitions.scala index a5fa4b55..892e0445 100644 --- a/uncore/src/main/scala/tilelink/Definitions.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -4,6 +4,7 @@ package uncore.tilelink import Chisel._ import junctions._ import uncore.coherence.CoherencePolicy +import uncore.Util._ import scala.math.max import uncore.constants._ import cde.{Parameters, Field} @@ -211,7 +212,7 @@ trait HasAcquireType extends HasTileLinkParameters { def isBuiltInType(t: UInt): Bool = is_builtin_type && a_type === t /** Does this message refer to subblock operands using info in the Acquire.union subbundle */ - def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesOnSubBlocks.contains(a_type) + def isSubBlockType(dummy: Int = 0): Bool = isBuiltInType() && a_type.isOneOf(Acquire.typesOnSubBlocks) /** Is this message a built-in prefetch message */ def isPrefetch(dummy: Int = 0): Bool = isBuiltInType() && @@ -224,11 +225,11 @@ trait HasAcquireType extends HasTileLinkParameters { def isGet(dummy: Int = 0): Bool = isBuiltInType() && (is(Acquire.getType) || is(Acquire.getBlockType)) /** Does this message contain data? Assumes that no custom message types have data. */ - def hasData(dummy: Int = 0): Bool = isBuiltInType() && Acquire.typesWithData.contains(a_type) + def hasData(dummy: Int = 0): Bool = isBuiltInType() && a_type.isOneOf(Acquire.typesWithData) /** Does this message contain multiple beats of data? Assumes that no custom message types have data. */ def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && isBuiltInType() && - Acquire.typesWithMultibeatData.contains(a_type) + a_type.isOneOf(Acquire.typesWithMultibeatData) /** Mapping between each built-in Acquire type and a built-in Grant type. */ def getBuiltInGrantType(dummy: Int = 0): UInt = Acquire.getBuiltInGrantType(this.a_type) @@ -251,9 +252,9 @@ trait HasReleaseType extends HasTileLinkParameters with MightBeVoluntary { val r_type = UInt(width = tlCoh.releaseTypeWidth) def is(t: UInt) = r_type === t - def hasData(dummy: Int = 0) = tlCoh.releaseTypesWithData.contains(r_type) + def hasData(dummy: Int = 0) = r_type.isOneOf(tlCoh.releaseTypesWithData) def hasMultibeatData(dummy: Int = 0) = Bool(tlDataBeats > 1) && - tlCoh.releaseTypesWithData.contains(r_type) + r_type.isOneOf(tlCoh.releaseTypesWithData) def isVoluntary(dummy: Int = 0) = voluntary def requiresAck(dummy: Int = 0) = !Bool(tlNetworkPreservesPointToPointOrdering) } @@ -267,12 +268,12 @@ trait HasGrantType extends HasTileLinkParameters with MightBeVoluntary { def isBuiltInType(t: UInt): Bool = is_builtin_type && g_type === t def is(t: UInt):Bool = g_type === t def hasData(dummy: Int = 0): Bool = Mux(isBuiltInType(), - Grant.typesWithData.contains(g_type), - tlCoh.grantTypesWithData.contains(g_type)) + g_type.isOneOf(Grant.typesWithData), + g_type.isOneOf(tlCoh.grantTypesWithData)) def hasMultibeatData(dummy: Int = 0): Bool = Bool(tlDataBeats > 1) && Mux(isBuiltInType(), - Grant.typesWithMultibeatData.contains(g_type), - tlCoh.grantTypesWithData.contains(g_type)) + g_type.isOneOf(Grant.typesWithMultibeatData), + g_type.isOneOf(tlCoh.grantTypesWithData)) def isVoluntary(dummy: Int = 0): Bool = isBuiltInType() && (g_type === Grant.voluntaryAckType) def requiresAck(dummy: Int = 0): Bool = !Bool(tlNetworkPreservesPointToPointOrdering) && !isVoluntary() } @@ -296,7 +297,7 @@ class AcquireMetadata(implicit p: Parameters) extends ClientToManagerChannel /** Complete physical address for block, beat or operand */ def full_addr(dummy: Int = 0) = Cat(this.addr_block, this.addr_beat, - Mux(isBuiltInType() && Acquire.typesWithAddrByte.contains(this.a_type), + Mux(isBuiltInType() && this.a_type.isOneOf(Acquire.typesWithAddrByte), this.addr_byte(), UInt(0, tlByteAddrBits))) } From a50ba39ea776285a0146eea31e1b8dd5e5a8d15e Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 18:54:02 -0700 Subject: [PATCH 669/688] Revert "add buffering and locking to TL -> Nasti converter" This reverts commit 2109a48e18719383942d535ff4c1d0a859dcc424. Conflicts: src/main/scala/converters/Nasti.scala --- uncore/src/main/scala/converters/Nasti.scala | 100 +++++-------------- 1 file changed, 26 insertions(+), 74 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 84ec2d47..ffe1c8e2 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -4,9 +4,7 @@ import Chisel._ import junctions._ import uncore.tilelink._ import uncore.constants._ -import cde.{Field, Parameters} - -case object NastiResponseBufferDepth extends Field[Int] +import cde.Parameters class IdMapper(val inIdBits: Int, val outIdBits: Int, val forceMapping: Boolean = false) @@ -63,48 +61,6 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int, } } -class NastiReadBuffer(implicit p: Parameters) extends NastiModule()(p) { - val bufferDepth = p(NastiResponseBufferDepth) - val io = new Bundle { - val in = new NastiReadIO().flip - val out = new NastiReadIO - } - - io.in.r <> Queue(io.out.r, bufferDepth) - - val inflight = Reg(init = UInt(0, width = log2Up(bufferDepth + 1))) - - val ar_delta = Mux(io.in.ar.fire(), io.in.ar.bits.len + UInt(1), UInt(0)) - val r_delta = Mux(io.in.r.fire(), UInt(1), UInt(0)) - inflight := inflight + ar_delta - r_delta - - val can_request = (inflight + io.in.ar.bits.len) < UInt(bufferDepth) - io.in.ar.ready := io.out.ar.ready && can_request - io.out.ar.valid := io.in.ar.valid && can_request - io.out.ar.bits := io.in.ar.bits -} - -class NastiWriteBuffer(implicit p: Parameters) extends NastiModule()(p) { - val bufferDepth = p(NastiResponseBufferDepth) - val io = new Bundle { - val in = new NastiWriteIO().flip - val out = new NastiWriteIO - } - - io.in.b <> Queue(io.out.b, bufferDepth) - io.out.w <> io.in.w - - val inflight = Reg(init = UInt(0, width = log2Up(bufferDepth + 1))) - val aw_delta = Mux(io.in.aw.fire(), UInt(1), UInt(0)) - val b_delta = Mux(io.in.b.fire(), UInt(1), UInt(0)) - inflight := inflight + aw_delta - b_delta - - val can_request = inflight < UInt(bufferDepth) - io.in.aw.ready := io.out.aw.ready && can_request - io.out.aw.valid := io.in.aw.valid && can_request - io.out.aw.bits := io.in.aw.bits -} - class NastiIOTileLinkIOConverterInfo(implicit p: Parameters) extends TLBundle()(p) { val addr_beat = UInt(width = tlBeatAddrBits) val subblock = Bool() @@ -155,14 +111,11 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val put_id_mask = is_subblock || io.tl.acquire.bits.addr_beat === UInt(0) val put_id_ready = put_id_mapper.io.req.ready || !put_id_mask - val get_buffer = Module(new NastiReadBuffer) - val put_buffer = Module(new NastiWriteBuffer) - // For Get/GetBlock, make sure Reorder queue can accept new entry val get_helper = DecoupledHelper( get_valid, roq.io.enq.ready, - get_buffer.io.in.ar.ready, + io.nasti.ar.ready, get_id_ready) val w_inflight = Reg(init = Bool(false)) @@ -170,33 +123,36 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) // For Put/PutBlock, make sure aw and w channel are both ready before // we send the first beat - val aw_ready = w_inflight || put_buffer.io.in.aw.ready + val aw_ready = w_inflight || io.nasti.aw.ready val put_helper = DecoupledHelper( put_valid, aw_ready, - put_buffer.io.in.w.ready, + io.nasti.w.ready, put_id_ready) + val (nasti_cnt_out, nasti_wrap_out) = Counter( + io.nasti.r.fire() && !roq.io.deq.data.subblock, tlDataBeats) + roq.io.enq.valid := get_helper.fire(roq.io.enq.ready) - roq.io.enq.bits.tag := get_id_mapper.io.req.out_id + roq.io.enq.bits.tag := io.nasti.ar.bits.id roq.io.enq.bits.data.addr_beat := io.tl.acquire.bits.addr_beat roq.io.enq.bits.data.subblock := is_subblock - roq.io.deq.valid := get_buffer.io.in.r.fire() && get_buffer.io.in.r.bits.last - roq.io.deq.tag := get_buffer.io.in.r.bits.id + roq.io.deq.valid := io.nasti.r.fire() && (nasti_wrap_out || roq.io.deq.data.subblock) + roq.io.deq.tag := io.nasti.r.bits.id get_id_mapper.io.req.valid := get_helper.fire(get_id_ready) get_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id - get_id_mapper.io.resp.valid := get_buffer.io.in.r.fire() && get_buffer.io.in.r.bits.last - get_id_mapper.io.resp.out_id := get_buffer.io.in.r.bits.id + get_id_mapper.io.resp.valid := io.nasti.r.fire() && io.nasti.r.bits.last + get_id_mapper.io.resp.out_id := io.nasti.r.bits.id put_id_mapper.io.req.valid := put_helper.fire(put_id_ready, put_id_mask) put_id_mapper.io.req.in_id := io.tl.acquire.bits.client_xact_id - put_id_mapper.io.resp.valid := put_buffer.io.in.b.fire() - put_id_mapper.io.resp.out_id := put_buffer.io.in.b.bits.id + put_id_mapper.io.resp.valid := io.nasti.b.fire() + put_id_mapper.io.resp.out_id := io.nasti.b.bits.id // Decompose outgoing TL Acquires into Nasti address and data channels - get_buffer.io.in.ar.valid := get_helper.fire(get_buffer.io.in.ar.ready) - get_buffer.io.in.ar.bits := NastiReadAddressChannel( + io.nasti.ar.valid := get_helper.fire(io.nasti.ar.ready) + io.nasti.ar.bits := NastiReadAddressChannel( id = get_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr(), size = Mux(is_subblock, @@ -224,16 +180,16 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val all_inside_0 = (~io.tl.acquire.bits.wmask()).toBools val (_, put_offset, put_size) = mask_helper(all_inside_0, 0) - put_buffer.io.in.aw.valid := put_helper.fire(aw_ready, !w_inflight) - put_buffer.io.in.aw.bits := NastiWriteAddressChannel( + io.nasti.aw.valid := put_helper.fire(aw_ready, !w_inflight) + io.nasti.aw.bits := NastiWriteAddressChannel( id = put_id_mapper.io.req.out_id, addr = io.tl.acquire.bits.full_addr() | Mux(is_multibeat, UInt(0), put_offset), size = Mux(is_multibeat, UInt(log2Ceil(tlDataBytes)), put_size), len = Mux(is_multibeat, UInt(tlDataBeats - 1), UInt(0))) - put_buffer.io.in.w.valid := put_helper.fire(put_buffer.io.in.w.ready) - put_buffer.io.in.w.bits := NastiWriteDataChannel( + io.nasti.w.valid := put_helper.fire(io.nasti.w.ready) + io.nasti.w.bits := NastiWriteDataChannel( id = w_id, data = io.tl.acquire.bits.data, strb = Some(io.tl.acquire.bits.wmask()), @@ -256,12 +212,11 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) // Aggregate incoming NASTI responses into TL Grants val (tl_cnt_in, tl_wrap_in) = Counter( io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new LockingArbiter(new GrantToDst, 2, - tlDataBeats, Some((gnt: GrantToDst) => gnt.hasMultibeatData()))) + val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) io.tl.grant <> gnt_arb.io.out - gnt_arb.io.in(0).valid := get_buffer.io.in.r.valid - get_buffer.io.in.r.ready := gnt_arb.io.in(0).ready + gnt_arb.io.in(0).valid := io.nasti.r.valid + io.nasti.r.ready := gnt_arb.io.in(0).ready gnt_arb.io.in(0).bits := Grant( is_builtin_type = Bool(true), g_type = Mux(roq.io.deq.data.subblock, @@ -269,15 +224,15 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) client_xact_id = get_id_mapper.io.resp.in_id, manager_xact_id = UInt(0), addr_beat = Mux(roq.io.deq.data.subblock, roq.io.deq.data.addr_beat, tl_cnt_in), - data = get_buffer.io.in.r.bits.data) + data = io.nasti.r.bits.data) assert(!roq.io.deq.valid || roq.io.deq.matches, "TL -> NASTI converter ReorderQueue: NASTI tag error") assert(!gnt_arb.io.in(0).valid || get_id_mapper.io.resp.matches, "TL -> NASTI ID Mapper: NASTI tag error") - gnt_arb.io.in(1).valid := put_buffer.io.in.b.valid - put_buffer.io.in.b.ready := gnt_arb.io.in(1).ready + gnt_arb.io.in(1).valid := io.nasti.b.valid + io.nasti.b.ready := gnt_arb.io.in(1).ready gnt_arb.io.in(1).bits := Grant( is_builtin_type = Bool(true), g_type = Grant.putAckType, @@ -289,9 +244,6 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) assert(!io.nasti.r.valid || io.nasti.r.bits.resp === UInt(0), "NASTI read error") assert(!io.nasti.b.valid || io.nasti.b.bits.resp === UInt(0), "NASTI write error") - - io.nasti <> get_buffer.io.out - io.nasti <> put_buffer.io.out } class TileLinkIONastiIOConverter(implicit p: Parameters) extends TLModule()(p) From 8aa73915a19464276e4d862bad1bc9a3ed747d15 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 7 Jul 2016 18:57:38 -0700 Subject: [PATCH 670/688] put locking arbiter back into converter --- uncore/src/main/scala/converters/Nasti.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index ffe1c8e2..f41ac764 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -212,7 +212,8 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) // Aggregate incoming NASTI responses into TL Grants val (tl_cnt_in, tl_wrap_in) = Counter( io.tl.grant.fire() && io.tl.grant.bits.hasMultibeatData(), tlDataBeats) - val gnt_arb = Module(new Arbiter(new GrantToDst, 2)) + val gnt_arb = Module(new LockingArbiter(new GrantToDst, 2, + tlDataBeats, Some((gnt: GrantToDst) => gnt.hasMultibeatData()))) io.tl.grant <> gnt_arb.io.out gnt_arb.io.in(0).valid := io.nasti.r.valid From de1e25f3d106cd5f5084d526ed300b09303cce13 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 13 Jul 2016 11:08:36 -0700 Subject: [PATCH 671/688] reduce usage of CAMs in converters --- uncore/src/main/scala/converters/Nasti.scala | 20 +++++++++---------- .../src/main/scala/converters/Tilelink.scala | 7 ++----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index f41ac764..0249e8f1 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -36,28 +36,26 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int, val nOutXacts = 1 << outIdBits val out_id_free = Reg(init = Vec.fill(nOutXacts){Bool(true)}) + val in_id_free = Reg(init = Vec.fill(nInXacts){Bool(true)}) val next_out_id = PriorityEncoder(out_id_free) - val id_mapping = Reg(Vec(nInXacts, UInt(0, outIdBits))) - val id_valid = Reg(init = Vec.fill(nInXacts){Bool(false)}) + val id_mapping = Reg(Vec(nOutXacts, UInt(0, inIdBits))) val req_fire = io.req.valid && io.req.ready when (req_fire) { out_id_free(io.req.out_id) := Bool(false) - id_valid(io.req.in_id) := Bool(true) - id_mapping(io.req.in_id) := io.req.out_id + in_id_free(io.req.in_id) := Bool(false) + id_mapping(io.req.out_id) := io.req.in_id } when (io.resp.valid) { out_id_free(io.resp.out_id) := Bool(true) - id_valid(io.resp.in_id) := Bool(false) + in_id_free(io.resp.in_id) := Bool(true) } - io.req.ready := out_id_free.reduce(_ || _) && !id_valid(io.req.in_id) + io.req.ready := out_id_free.reduce(_ || _) && in_id_free(io.req.in_id) io.req.out_id := next_out_id - val id_matches = id_mapping.map(_ === io.resp.out_id) - val id_matches_valid = id_matches.zip(id_valid).map { case (m, v) => m && v } - io.resp.matches := id_matches_valid.reduce(_ || _) - io.resp.in_id := PriorityEncoder(id_matches_valid) + io.resp.in_id := id_mapping(io.resp.out_id) + io.resp.matches := !out_id_free(io.resp.out_id) } } @@ -102,7 +100,7 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) // Reorder queue saves extra information needed to send correct // grant back to TL client val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, nastiRIdBits, tlMaxXacts)) + new NastiIOTileLinkIOConverterInfo, nastiRIdBits, Some(tlMaxXacts))) val get_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) val put_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index 6605ab63..a3c0349d 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -55,11 +55,8 @@ class ClientTileLinkIOUnwrapper(implicit p: Parameters) extends TLModule()(p) { val acqArb = Module(new LockingRRArbiter(new Acquire, 2, tlDataBeats, Some((acq: Acquire) => acq.hasMultibeatData()))) - val acqRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) - - val relRoq = Module(new ReorderQueue( - Bool(), tlClientXactIdBits, tlMaxClientsPerPort)) + val acqRoq = Module(new ReorderQueue(Bool(), tlClientXactIdBits)) + val relRoq = Module(new ReorderQueue(Bool(), tlClientXactIdBits)) val iacq = io.in.acquire.bits val irel = io.in.release.bits From b122a54c320ee76d1a419becf138fbbb32aa8470 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 13 Jul 2016 12:42:28 -0700 Subject: [PATCH 672/688] don't allow more outer IDs than inner IDs --- uncore/src/main/scala/converters/Nasti.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uncore/src/main/scala/converters/Nasti.scala b/uncore/src/main/scala/converters/Nasti.scala index 0249e8f1..fa090083 100644 --- a/uncore/src/main/scala/converters/Nasti.scala +++ b/uncore/src/main/scala/converters/Nasti.scala @@ -5,6 +5,7 @@ import junctions._ import uncore.tilelink._ import uncore.constants._ import cde.Parameters +import scala.math.min class IdMapper(val inIdBits: Int, val outIdBits: Int, val forceMapping: Boolean = false) @@ -33,7 +34,8 @@ class IdMapper(val inIdBits: Int, val outIdBits: Int, io.resp.in_id := io.resp.out_id } else { val nInXacts = 1 << inIdBits - val nOutXacts = 1 << outIdBits + // No point in allowing more out xacts than in xacts + val nOutXacts = min(1 << outIdBits, nInXacts) val out_id_free = Reg(init = Vec.fill(nOutXacts){Bool(true)}) val in_id_free = Reg(init = Vec.fill(nInXacts){Bool(true)}) @@ -95,12 +97,11 @@ class NastiIOTileLinkIOConverter(implicit p: Parameters) extends TLModule()(p) val get_valid = io.tl.acquire.valid && !has_data val put_valid = io.tl.acquire.valid && has_data - val tlMaxXacts = tlMaxClientXacts * tlMaxClientsPerPort - // Reorder queue saves extra information needed to send correct // grant back to TL client + val roqIdBits = min(tlClientXactIdBits, nastiXIdBits) val roq = Module(new ReorderQueue( - new NastiIOTileLinkIOConverterInfo, nastiRIdBits, Some(tlMaxXacts))) + new NastiIOTileLinkIOConverterInfo, roqIdBits)) val get_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) val put_id_mapper = Module(new IdMapper(tlClientXactIdBits, nastiXIdBits)) From 84098db81f34c632258ac910aaa7be0d3e68cd39 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Fri, 15 Jul 2016 11:03:26 -0700 Subject: [PATCH 673/688] add a TileLinkTestRAM --- uncore/src/main/scala/devices/Bram.scala | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/uncore/src/main/scala/devices/Bram.scala b/uncore/src/main/scala/devices/Bram.scala index 7e277192..e472b697 100644 --- a/uncore/src/main/scala/devices/Bram.scala +++ b/uncore/src/main/scala/devices/Bram.scala @@ -4,6 +4,7 @@ import Chisel._ import cde.{Parameters, Field} import junctions._ import uncore.tilelink._ +import uncore.util._ import HastiConstants._ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module @@ -123,3 +124,62 @@ class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { io.hready := Bool(true) io.hresp := HRESP_OKAY } + +/** + * This RAM is not meant to be particularly performant. + * It just supports the entire range of uncached TileLink operations in the + * simplest way possible. + */ +class TileLinkTestRAM(depth: Int)(implicit val p: Parameters) extends Module + with HasTileLinkParameters { + val io = new ClientUncachedTileLinkIO().flip + + val ram = Mem(depth, UInt(width = tlDataBits)) + + val responding = Reg(init = Bool(false)) + val acq = io.acquire.bits + val r_acq = Reg(io.acquire.bits) + val acq_addr = Cat(acq.addr_block, acq.addr_beat) + val r_acq_addr = Cat(r_acq.addr_block, r_acq.addr_beat) + + when (io.acquire.fire() && io.acquire.bits.last()) { + r_acq := io.acquire.bits + responding := Bool(true) + } + + when (io.grant.fire()) { + val is_getblk = r_acq.isBuiltInType(Acquire.getBlockType) + val last_beat = r_acq.addr_beat === UInt(tlDataBeats - 1) + when (is_getblk && !last_beat) { + r_acq.addr_beat := r_acq.addr_beat + UInt(1) + } .otherwise { responding := Bool(false) } + } + + io.acquire.ready := !responding + io.grant.valid := responding + io.grant.bits := Grant( + is_builtin_type = Bool(true), + g_type = r_acq.getBuiltInGrantType(), + client_xact_id = r_acq.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = r_acq.addr_beat, + data = ram(r_acq_addr)) + + val old_data = ram(acq_addr) + val new_data = acq.data + + val amo_shift_bits = acq.amo_shift_bytes() << UInt(3) + val amoalu = Module(new AMOALU) + amoalu.io.addr := Cat(acq.addr_block, acq.addr_beat, acq.addr_byte()) + amoalu.io.cmd := acq.op_code() + amoalu.io.typ := acq.op_size() + amoalu.io.lhs := old_data >> amo_shift_bits + amoalu.io.rhs := new_data >> amo_shift_bits + + val result = Mux(acq.isAtomic(), amoalu.io.out << amo_shift_bits, new_data) + val wmask = FillInterleaved(8, acq.wmask()) + + when (io.acquire.fire() && acq.hasData()) { + ram(acq_addr) := (old_data & ~wmask) | (result & wmask) + } +} From e5cccc05260062b8c30dec5484f9eae8c9d90ec1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 18 Jul 2016 17:05:23 -0700 Subject: [PATCH 674/688] don't update xact_vol_irel if not a voluntary irel --- uncore/src/main/scala/agents/Trackers.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 4d23a4cb..5b8d3080 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -257,9 +257,11 @@ trait AcceptsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { val irel_fire = (irel_is_allocating || irel_is_merging) && io.inner.release.ready when (irel_fire) { when (io.irel().first()) { - xact_vol_ir_r_type := io.irel().r_type - xact_vol_ir_src := io.irel().client_id - xact_vol_ir_client_xact_id := io.irel().client_xact_id + when (io.irel().isVoluntary()) { + xact_vol_ir_r_type := io.irel().r_type + xact_vol_ir_src := io.irel().client_id + xact_vol_ir_client_xact_id := io.irel().client_xact_id + } // If this release has data, set all the pending bits except the first. // Otherwise, clear all the pending bits pending_irel_data := Mux(io.irel().hasMultibeatData(), From 9eeb1112d42cb96ce30ce8f5f38587c4a734cd60 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 18 Jul 2016 17:38:20 -0700 Subject: [PATCH 675/688] fix Bufferless irel_vs_iacq_conflict signal --- uncore/src/main/scala/agents/Bufferless.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index e108ca72..7c67edc4 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -30,13 +30,15 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren outer_arb.io.in <> outerList io.outer <> outer_arb.io.out + val iacq = Queue(io.inner.acquire, 1, pipe=true) + val irel = Queue(io.inner.release, 1, pipe=true) + // Handle acquire transaction initiation val irel_vs_iacq_conflict = - io.inner.acquire.valid && - io.inner.release.valid && - io.irel().conflicts(io.iacq()) + iacq.valid && + irel.valid && + irel.bits.conflicts(iacq.bits) - val iacq = Queue(io.inner.acquire, 1, pipe=true) doInputRoutingWithAllocation( in = iacq, outs = trackerList.map(_.io.inner.acquire), @@ -48,7 +50,6 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren } // Handle releases, which might be voluntary and might have data - val irel = Queue(io.inner.release, 1, pipe=true) doInputRoutingWithAllocation( in = irel, outs = trackerList.map(_.io.inner.release), From e406d1bd73d72fe2a0e1bdd753f352b82e7ece4c Mon Sep 17 00:00:00 2001 From: Henry Cook Date: Mon, 18 Jul 2016 18:22:49 -0700 Subject: [PATCH 676/688] Make probeCopy have same behavior as probeDowngrade --- .../src/main/scala/coherence/Policies.scala | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/uncore/src/main/scala/coherence/Policies.scala b/uncore/src/main/scala/coherence/Policies.scala index d4d04b71..744b8e7d 100644 --- a/uncore/src/main/scala/coherence/Policies.scala +++ b/uncore/src/main/scala/coherence/Policies.scala @@ -158,7 +158,7 @@ class MICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) + probeCopy -> getReleaseType(M_FLUSH, meta))) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = meta @@ -250,8 +250,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { def getReleaseType(incoming: HasProbeType, meta: ClientMetadata): UInt = MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), - probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) + probeDowngrade -> getReleaseType(M_FLUSH, meta), + probeCopy -> getReleaseType(M_FLUSH, meta))) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) @@ -271,8 +271,8 @@ class MEICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, - probeDowngrade -> clientExclusiveClean, - probeCopy -> meta.state)))(meta.p) + probeDowngrade -> clientInvalid, + probeCopy -> clientInvalid)))(meta.p) // Manager states and functions: val nManagerStates = 0 // We don't actually need any states for this protocol @@ -353,7 +353,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) + probeCopy -> getReleaseType(M_PRODUCE, meta))) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) @@ -378,7 +378,7 @@ class MSICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, - probeCopy -> meta.state)))(meta.p) + probeCopy -> clientShared)))(meta.p) // Manager states and functions: val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing @@ -471,7 +471,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.p_type, releaseInvalidateAck, Array( probeInvalidate -> getReleaseType(M_FLUSH, meta), probeDowngrade -> getReleaseType(M_PRODUCE, meta), - probeCopy -> getReleaseType(M_CLEAN, meta))) + probeCopy -> getReleaseType(M_PRODUCE, meta))) def clientMetadataOnHit(cmd: UInt, meta: ClientMetadata) = ClientMetadata(Mux(isWrite(cmd), clientExclusiveDirty, meta.state))(meta.p) @@ -498,7 +498,7 @@ class MESICoherence(dir: DirectoryRepresentation) extends CoherencePolicy(dir) { MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeDowngrade -> clientShared, - probeCopy -> meta.state)))(meta.p) + probeCopy -> clientShared)))(meta.p) // Manager states and functions: val nManagerStates = 0 // TODO: We could add a Shared state to avoid probing @@ -598,9 +598,10 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d releaseInvalidateAckMigratory, releaseInvalidateAck), probeInvalidateOthers -> Mux(clientSharedByTwo === meta.state, releaseInvalidateAckMigratory, releaseInvalidateAck), - probeDowngrade -> Mux(meta.state =/= clientInvalid, - releaseDowngradeAckHasCopy, releaseDowngradeAck), - probeCopy -> releaseCopyAck)) + probeDowngrade -> Mux(meta.state =/= clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck), + probeCopy -> Mux(meta.state =/= clientInvalid, + releaseDowngradeAckHasCopy, releaseDowngradeAck))) Mux(dirty, with_data, without_data) } @@ -631,18 +632,20 @@ class MigratoryCoherence(dir: DirectoryRepresentation) extends CoherencePolicy(d grantReadMigratory -> Mux(isWrite(cmd), clientMigratoryDirty, clientMigratoryClean)))))(meta.p) - def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = + def clientMetadataOnProbe(incoming: HasProbeType, meta: ClientMetadata) = { + val downgradeState = MuxLookup(meta.state, clientShared, Array( + clientExclusiveClean -> clientSharedByTwo, + clientExclusiveDirty -> clientSharedByTwo, + clientSharedByTwo -> clientShared, + clientMigratoryClean -> clientSharedByTwo, + clientMigratoryDirty -> clientInvalid)) ClientMetadata( MuxLookup(incoming.p_type, meta.state, Array( probeInvalidate -> clientInvalid, probeInvalidateOthers -> clientInvalid, - probeCopy -> meta.state, - probeDowngrade -> MuxLookup(meta.state, clientShared, Array( - clientExclusiveClean -> clientSharedByTwo, - clientExclusiveDirty -> clientSharedByTwo, - clientSharedByTwo -> clientShared, - clientMigratoryClean -> clientSharedByTwo, - clientMigratoryDirty -> clientInvalid)))))(meta.p) + probeDowngrade -> downgradeState, + probeCopy -> downgradeState)))(meta.p) + } // Manager states and functions: val nManagerStates = 0 // TODO: we could add some states to reduce the number of message types From fa8317fec1c3ad50449a80bd5e3d9179f021439c Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Mon, 18 Jul 2016 14:29:13 -0700 Subject: [PATCH 677/688] debug: add clock crossing primitives --- uncore/src/main/scala/devices/Debug.scala | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/uncore/src/main/scala/devices/Debug.scala b/uncore/src/main/scala/devices/Debug.scala index 9513e171..dc9468b1 100644 --- a/uncore/src/main/scala/devices/Debug.scala +++ b/uncore/src/main/scala/devices/Debug.scala @@ -983,3 +983,21 @@ class DebugModule ()(implicit val p:cde.Parameters) io.fullreset := CONTROLReg.fullreset } + +object AsyncDebugBusFrom { // OutsideClockDomain + def apply(from_clock: Clock, from_reset: Bool, source: DebugBusIO, depth: Int = 0, sync: Int = 2)(implicit p: Parameters): DebugBusIO = { + val sink = Wire(new DebugBusIO) + sink.req <> AsyncDecoupledFrom(from_clock, from_reset, source.req) + source.resp <> AsyncDecoupledTo(from_clock, from_reset, sink.resp) + sink + } +} + +object AsyncDebugBusTo { // OutsideClockDomain + def apply(to_clock: Clock, to_reset: Bool, source: DebugBusIO, depth: Int = 0, sync: Int = 2)(implicit p: Parameters): DebugBusIO = { + val sink = Wire(new DebugBusIO) + sink.req <> AsyncDecoupledTo(to_clock, to_reset, source.req) + source.resp <> AsyncDecoupledFrom(to_clock, to_reset, sink.resp) + sink + } +} From 86e31be8202f636df619630d7d444f03001b8b9b Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 20 Jul 2016 18:23:27 -0700 Subject: [PATCH 678/688] fix lockup from back to back releases with data --- uncore/src/main/scala/agents/Broadcast.scala | 3 ++- uncore/src/main/scala/agents/Bufferless.scala | 4 ++- uncore/src/main/scala/agents/Cache.scala | 3 ++- uncore/src/main/scala/agents/Trackers.scala | 25 ++++++++++++++++--- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index 86e02c5f..119570ce 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -160,7 +160,8 @@ class BufferedBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) !all_pending_done && !io.outer.grant.fire() && !io.inner.grant.fire() && - !vol_ignt_counter.pending + !vol_ignt_counter.pending && + !blockInnerRelease() innerRelease(block_vol_ignt = vol_ognt_counter.pending) diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index 7c67edc4..39276c2b 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -120,6 +120,7 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && !vol_ignt_counter.pending && + !(io.irel().hasData() && ognt_counter.pending) && (state =/= s_idle) innerRelease(block_vol_ignt = vol_ognt_counter.pending) @@ -131,7 +132,8 @@ class BufferlessBroadcastAcquireTracker(trackerId: Int)(implicit p: Parameters) // If there was a writeback, forward it outwards outerRelease( coh = outer_coh.onHit(M_XWR), - buffering = Bool(false)) + buffering = Bool(false), + block_orel = !irel_could_accept) // Send outer request for miss outerAcquire( diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 171555ff..52bff518 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -1050,7 +1050,8 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr io.irel().isVoluntary() && (state =/= s_idle) && !(state === s_busy && all_pending_done) && - !vol_ignt_counter.pending + !vol_ignt_counter.pending && + !blockInnerRelease() io.inner.release.ready := irel_can_merge || irel_same_xact diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index 5b8d3080..d695df8d 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -294,13 +294,28 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { val pending_orel_data = Reg(init=Bits(0, width = innerDataBeats)) val vol_ognt_counter = Wire(new TwoWayBeatCounterStatus) val pending_orel = pending_orel_send || pending_orel_data.orR || vol_ognt_counter.pending + val sending_orel = Reg(init = Bool(false)) + + // Block acceptance of inner releases if we have already started sending + // outer releases, but have not yet sent out the beat corresponding to the + // inner release. This function must be included in io.inner.release.ready + // if it is possible to start accepting a new inner release as the previous + // outer release is still being sent. DO NOT include this in the + // io.inner.release.ready if the releases are not buffered + // (i.e. io.inner.release and io.outer.release combinationally linked). + def blockInnerRelease(rel: ReleaseMetadata = io.irel()): Bool = { + val waiting_to_send = sending_orel && pending_orel_data(rel.addr_beat) + val sending_now = io.outer.release.fire() && rel.addr_beat === io.orel().addr_beat + rel.hasData() && (waiting_to_send || sending_now) + } def outerRelease( coh: ClientMetadata, buffering: Bool = Bool(true), data: UInt = io.irel().data, add_pending_data_bits: UInt = UInt(0), - add_pending_send_bit: Bool = Bool(false)) { + add_pending_send_bit: Bool = Bool(false), + block_orel: Bool = Bool(false)) { when (state =/= s_idle || io.alloc.irel.should) { pending_orel_data := (pending_orel_data | @@ -309,7 +324,11 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { dropPendingBitWhenBeatHasData(io.outer.release) } when (add_pending_send_bit) { pending_orel_send := Bool(true) } - when (io.outer.release.fire()) { pending_orel_send := Bool(false) } + when (io.outer.release.fire()) { + when (io.outer.release.bits.first()) { sending_orel := Bool(true) } + when (io.outer.release.bits.last()) { sending_orel := Bool(false) } + pending_orel_send := Bool(false) + } connectTwoWayBeatCounters( status = vol_ognt_counter, @@ -318,7 +337,7 @@ trait EmitsVoluntaryReleases extends HasVoluntaryReleaseMetadataBuffer { trackUp = (r: Release) => r.isVoluntary() && r.requiresAck(), trackDown = (g: Grant) => g.isVoluntary()) - io.outer.release.valid := Mux(buffering, + io.outer.release.valid := !block_orel && Mux(buffering, (state === s_busy) && Mux(io.orel().hasData(), pending_orel_data(vol_ognt_counter.up.idx), pending_orel_send), From 0a1cd647864de5108db97bd5c8b1beb555b953d4 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Jul 2016 13:45:20 -0700 Subject: [PATCH 679/688] fix number of builtin Acquire types --- uncore/src/main/scala/tilelink/Definitions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncore/src/main/scala/tilelink/Definitions.scala b/uncore/src/main/scala/tilelink/Definitions.scala index 892e0445..86e59983 100644 --- a/uncore/src/main/scala/tilelink/Definitions.scala +++ b/uncore/src/main/scala/tilelink/Definitions.scala @@ -342,7 +342,7 @@ class SecondaryMissInfo(implicit p: Parameters) extends TLBundle * @param union additional fields used for uncached types */ object Acquire { - val nBuiltInTypes = 5 + val nBuiltInTypes = 7 //TODO: Use Enum def getType = UInt("b000") // Get a single beat of data def getBlockType = UInt("b001") // Get a whole block of data From eb9e998c085589f60efb30f79ef5509b0a83b16a Mon Sep 17 00:00:00 2001 From: Megan Wachs Date: Mon, 18 Jul 2016 13:09:44 -0700 Subject: [PATCH 680/688] Add ManagerToClientStatelessBridge --- .../main/scala/agents/StatelessBridge.scala | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 uncore/src/main/scala/agents/StatelessBridge.scala diff --git a/uncore/src/main/scala/agents/StatelessBridge.scala b/uncore/src/main/scala/agents/StatelessBridge.scala new file mode 100644 index 00000000..efc7e2ab --- /dev/null +++ b/uncore/src/main/scala/agents/StatelessBridge.scala @@ -0,0 +1,61 @@ +// See LICENSE for license details. + +package uncore.agents + +import Chisel._ +import uncore.coherence._ +import uncore.tilelink._ +import uncore.constants._ +import uncore.devices._ +import cde.{Parameters, Field, Config} + +/** The ManagerToClientStateless Bridge does not maintain any state for the messages + * which pass through it. It simply passes the messages back and forth without any + * tracking or translation. + * + * This can reduce area and timing in very constrained situations: + * - The Manager and Client implement the same coherence protocol + * - There are no probe or finish messages. + * - The outer transaction ID is large enough to handle all possible inner + * transaction IDs, such that no remapping state must be maintained. + * + */ + +class ManagerToClientStatelessBridge(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { + + val icid = io.inner.acquire.bits.client_id.getWidth + val ixid = io.inner.acquire.bits.client_xact_id.getWidth + val oxid = io.outer.acquire.bits.client_xact_id.getWidth + + // Stateless Bridge is only usable in certain constrained situations. + // Sanity check its usage here. + // Additional requirements are that the inner and outer Coherence policies + // are the same (e.g. MEI != MESI), and that NTiles == 1, + // but this is not checked here. + require (p(NTiles) == 1) + + require(icid + ixid <= oxid) + require(icid == io.inner.release.bits.client_id.getWidth) + require(ixid == io.inner.release.bits.client_xact_id.getWidth) + require(oxid == io.outer.release.bits.client_xact_id.getWidth) + + io.outer.acquire.valid := io.inner.acquire.valid + io.inner.acquire.ready := io.outer.acquire.ready + io.outer.acquire.bits := io.inner.acquire.bits + io.outer.acquire.bits.client_xact_id := Cat(io.inner.acquire.bits.client_id, io.inner.acquire.bits.client_xact_id) + + io.outer.release.valid := io.inner.release.valid + io.inner.release.ready := io.outer.release.ready + io.outer.release.bits := io.inner.release.bits + io.outer.release.bits.client_xact_id := Cat(io.inner.release.bits.client_id, io.inner.release.bits.client_xact_id) + + io.inner.grant.valid := io.outer.grant.valid + io.outer.grant.ready := io.inner.grant.ready + io.inner.grant.bits := io.outer.grant.bits + io.inner.grant.bits.client_xact_id := io.outer.grant.bits.client_xact_id(ixid-1, 0) + io.inner.grant.bits.client_id := io.outer.grant.bits.client_xact_id(icid+ixid-1, ixid) + + io.inner.probe.valid := Bool(false) + io.inner.finish.ready := Bool(true) +} + From c38dff08550c820e039e40689c883bcfc30e3cc1 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Jul 2016 15:07:10 -0700 Subject: [PATCH 681/688] add some more warnings about the StatelessBridge --- uncore/src/main/scala/agents/StatelessBridge.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/uncore/src/main/scala/agents/StatelessBridge.scala b/uncore/src/main/scala/agents/StatelessBridge.scala index efc7e2ab..515b60c5 100644 --- a/uncore/src/main/scala/agents/StatelessBridge.scala +++ b/uncore/src/main/scala/agents/StatelessBridge.scala @@ -19,6 +19,15 @@ import cde.{Parameters, Field, Config} * - The outer transaction ID is large enough to handle all possible inner * transaction IDs, such that no remapping state must be maintained. * + * This bridge DOES NOT keep the uncached channel coherent with the cached + * channel. Uncached requests to blocks cached by the L1 will not probe the L1. + * As a result, uncached reads to cached blocks will get stale data until + * the L1 performs a voluntary writeback, and uncached writes to cached blocks + * will get lost, as the voluntary writeback from the L1 will overwrite the + * changes. If your tile relies on probing the L1 data cache in order to + * share data between the instruction cache and data cache (e.g. you are using + * a non-blocking L1 D$) or if the tile has uncached channels capable of + * writes (e.g. Hwacha and other RoCC accelerators), DO NOT USE THIS BRIDGE. */ class ManagerToClientStatelessBridge(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { From 12067a3b8d6ecb595d222093d640b57c82c6e802 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Jul 2016 15:07:25 -0700 Subject: [PATCH 682/688] make sure outer probe and finish lines are disconnected --- uncore/src/main/scala/agents/Agents.scala | 7 +++++++ uncore/src/main/scala/agents/Broadcast.scala | 2 ++ uncore/src/main/scala/agents/Bufferless.scala | 2 ++ uncore/src/main/scala/agents/Cache.scala | 2 ++ uncore/src/main/scala/agents/StatelessBridge.scala | 3 ++- 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/uncore/src/main/scala/agents/Agents.scala b/uncore/src/main/scala/agents/Agents.scala index 15bf0f7b..b7519633 100644 --- a/uncore/src/main/scala/agents/Agents.scala +++ b/uncore/src/main/scala/agents/Agents.scala @@ -151,4 +151,11 @@ abstract class HierarchicalCoherenceAgent(implicit p: Parameters) extends Cohere def innerTL = io.inner def outerTL = io.outer def incoherent = io.incoherent + + // TODO: Remove this function (and all its calls) when we support probing the L2 + def disconnectOuterProbeAndFinish() { + io.outer.probe.ready := Bool(false) + io.outer.finish.valid := Bool(false) + assert(!io.outer.probe.valid, "L2 agent got illegal probe") + } } diff --git a/uncore/src/main/scala/agents/Broadcast.scala b/uncore/src/main/scala/agents/Broadcast.scala index 119570ce..9845342e 100644 --- a/uncore/src/main/scala/agents/Broadcast.scala +++ b/uncore/src/main/scala/agents/Broadcast.scala @@ -54,6 +54,8 @@ class L2BroadcastHub(implicit p: Parameters) extends HierarchicalCoherenceAgent( doOutputArbitration(io.inner.grant, trackerList.map(_.io.inner.grant)) doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + disconnectOuterProbeAndFinish() } class BroadcastXactTracker(implicit p: Parameters) extends XactTracker()(p) { diff --git a/uncore/src/main/scala/agents/Bufferless.scala b/uncore/src/main/scala/agents/Bufferless.scala index 39276c2b..5371d74a 100644 --- a/uncore/src/main/scala/agents/Bufferless.scala +++ b/uncore/src/main/scala/agents/Bufferless.scala @@ -67,6 +67,8 @@ class BufferlessBroadcastHub(implicit p: Parameters) extends HierarchicalCoheren io.inner.grant.bits.addr_beat := io.outer.grant.bits.addr_beat doInputRouting(io.inner.finish, trackerList.map(_.io.inner.finish)) + + disconnectOuterProbeAndFinish() } class BufferlessBroadcastVoluntaryReleaseTracker(trackerId: Int)(implicit p: Parameters) diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 52bff518..5cfb819f 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -442,6 +442,8 @@ class L2HellaCacheBank(implicit p: Parameters) extends HierarchicalCoherenceAgen tshrfile.io.incoherent <> io.incoherent meta.io <> tshrfile.io.meta data.io <> tshrfile.io.data + + disconnectOuterProbeAndFinish() } class TSHRFileIO(implicit p: Parameters) extends HierarchicalTLIO()(p) diff --git a/uncore/src/main/scala/agents/StatelessBridge.scala b/uncore/src/main/scala/agents/StatelessBridge.scala index 515b60c5..aacc0f59 100644 --- a/uncore/src/main/scala/agents/StatelessBridge.scala +++ b/uncore/src/main/scala/agents/StatelessBridge.scala @@ -66,5 +66,6 @@ class ManagerToClientStatelessBridge(implicit p: Parameters) extends Hierarchica io.inner.probe.valid := Bool(false) io.inner.finish.ready := Bool(true) -} + disconnectOuterProbeAndFinish() +} From 9168f35971940c9c7319520bd73a5ad97f9825e3 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Thu, 21 Jul 2016 18:31:19 -0700 Subject: [PATCH 683/688] clean up the requirements in StatelessBridge * No need to check that release ID bits and acquire ID bits the same * Check that inner and outer coherence policies match --- .../main/scala/agents/StatelessBridge.scala | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/uncore/src/main/scala/agents/StatelessBridge.scala b/uncore/src/main/scala/agents/StatelessBridge.scala index aacc0f59..0ed818cf 100644 --- a/uncore/src/main/scala/agents/StatelessBridge.scala +++ b/uncore/src/main/scala/agents/StatelessBridge.scala @@ -31,22 +31,20 @@ import cde.{Parameters, Field, Config} */ class ManagerToClientStatelessBridge(implicit p: Parameters) extends HierarchicalCoherenceAgent()(p) { + val icid = io.inner.tlClientIdBits + val ixid = io.inner.tlClientXactIdBits + val oxid = io.outer.tlClientXactIdBits - val icid = io.inner.acquire.bits.client_id.getWidth - val ixid = io.inner.acquire.bits.client_xact_id.getWidth - val oxid = io.outer.acquire.bits.client_xact_id.getWidth + val innerCoh = io.inner.tlCoh.getClass + val outerCoh = io.outer.tlCoh.getClass // Stateless Bridge is only usable in certain constrained situations. // Sanity check its usage here. - // Additional requirements are that the inner and outer Coherence policies - // are the same (e.g. MEI != MESI), and that NTiles == 1, - // but this is not checked here. - require (p(NTiles) == 1) + require(io.inner.tlNCachingClients <= 1) require(icid + ixid <= oxid) - require(icid == io.inner.release.bits.client_id.getWidth) - require(ixid == io.inner.release.bits.client_xact_id.getWidth) - require(oxid == io.outer.release.bits.client_xact_id.getWidth) + require(innerCoh eq outerCoh, + s"Coherence policies do not match: inner is ${innerCoh.getSimpleName}, outer is ${outerCoh.getSimpleName}") io.outer.acquire.valid := io.inner.acquire.valid io.inner.acquire.ready := io.outer.acquire.ready From a52d418439f56f791e05626508fdb933961eb189 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 20 Jul 2016 13:58:49 -0700 Subject: [PATCH 684/688] fragmenter: support multi-beat get/put via fragmenting to single-beat operations --- .../src/main/scala/converters/Tilelink.scala | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/uncore/src/main/scala/converters/Tilelink.scala b/uncore/src/main/scala/converters/Tilelink.scala index a3c0349d..6cb15f91 100644 --- a/uncore/src/main/scala/converters/Tilelink.scala +++ b/uncore/src/main/scala/converters/Tilelink.scala @@ -538,3 +538,154 @@ class TileLinkIONarrower(innerTLId: String, outerTLId: String) sending_get := Bool(false) } } + +class TileLinkFragmenterSource(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = Decoupled(new Acquire).flip + val out = Decoupled(new Acquire) + val que = Decoupled(UInt(width = tlBeatAddrBits)) + } + + // Pipeline stage with acquire data; needed to ensure in.bits stay fixed when !in.ready + val acq_valid = RegInit(Bool(false)) + val acq_bits = Reg(new Acquire) + // The last beat of generate acquire to send + val acq_last_beat = Reg(UInt(width = tlBeatAddrBits)) + val acq_last = acq_bits.addr_beat === acq_last_beat + + // 'in' has the first beat? + val in_multi_put = io.in.bits.isBuiltInType(Acquire.putBlockType) + val in_multi_get = io.in.bits.isBuiltInType(Acquire.getBlockType) + val in_first_beat = !in_multi_put || io.in.bits.addr_beat === UInt(0) + + // Move stuff from acq to out whenever out is ready + io.out.valid := acq_valid + // When can acq accept a request? + val acq_ready = !acq_valid || (acq_last && io.out.ready) + // Move the first beat from in to acq only when both acq and que are ready + io.in.ready := (!in_first_beat || io.que.ready) && acq_ready + io.que.valid := (in_first_beat && io.in.valid) && acq_ready + + // in.fire moves data from in to acq and (optionally) que + // out.fire moves data from acq to out + + // Desired flow control results: + assert (!io.que.fire() || io.in.fire()) // 1. que.fire => in.fire + assert (!(io.in.fire() && in_first_beat) || io.que.fire()) // 2. in.fire && in_first_beat => que.fire + assert (!io.out.fire() || acq_valid) // 3. out.fire => acq_valid + assert (!io.in.fire() || (!acq_valid || (io.out.fire() && acq_last))) // 4. in.fire => !acq_valid || (out.fire && acq_last) + // Proofs: + // 1. que.fire => que.ready && in.valid && acq_ready => in.ready && in.valid + // 2. in.fire && in_first_beat => in.valid && acq_ready && [(!in_first_beat || que.ready) && in_first_beat] => + // in.valid && acq_ready && que.ready && in_first_beat => que.valid && que.ready + // 3. out.fire => out.valid => acq_valid + // 4. in.fire => acq_ready => !acq_valid || (acq_last && out.ready) => + // !acq_valid || (acq_valid && acq_last && out.ready) => !acq_valid || (acq_last && out.fire) + + val multi_size = SInt(-1, width = tlBeatAddrBits).asUInt // TL2: use in.bits.size()/beatBits-1 + val in_sizeMinus1 = Mux(in_multi_get || in_multi_put, multi_size, UInt(0)) + val in_insertSizeMinus1 = Mux(in_multi_get, multi_size, UInt(0)) + + when (io.in.fire()) { + // Theorem 4 makes this safe; we overwrite garbage, or replace the final acq + acq_valid := Bool(true) + acq_bits := io.in.bits + acq_last_beat := io.in.bits.addr_beat + in_insertSizeMinus1 + // Replace this with size truncation in TL2: + acq_bits.a_type := Mux(in_multi_put, Acquire.putType, Mux(in_multi_get, Acquire.getType, io.in.bits.a_type)) + } .elsewhen (io.out.fire()) { + acq_valid := !acq_last // false => !in.valid || (!que.ready && in_first_beat) + acq_bits.addr_beat := acq_bits.addr_beat + UInt(1) + // acq_last && out.fire => acq_last && out.ready && acq_valid => acq_ready + // Suppose in.valid, then !in.fire => !in.ready => !(!in_first_beat || que.ready) => !que.ready && in_first_beat + } + + // Safe by theorem 3 + io.out.bits := acq_bits + // Safe by theorem 1 + io.que.bits := in_sizeMinus1 +} + +class TileLinkFragmenterSink(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = Decoupled(new Grant).flip + val out = Decoupled(new Grant) + val que = Decoupled(UInt(width = tlBeatAddrBits)).flip + } + + val count_valid = RegInit(Bool(false)) + val multi_op = Reg(Bool()) + val count_bits = Reg(UInt(width = tlBeatAddrBits)) + val last = count_bits === UInt(0) + + val in_put = io.in.bits.isBuiltInType(Grant.putAckType) + val in_get = io.in.bits.isBuiltInType(Grant.getDataBeatType) + val deliver = last || in_get + + // Accept the input, discarding the non-final put grant + io.in.ready := count_valid && (io.out.ready || !deliver) + // Output the grant whenever we want delivery + io.out.valid := count_valid && io.in.valid && deliver + // Take a new number whenever we deliver the last beat + io.que.ready := !count_valid || (io.in.valid && io.out.ready && last) + + // Desired flow control results: + assert (!io.out.fire() || (count_valid && io.in.fire())) // 1. out.fire => in.fire && count_valid + assert (!(io.in.fire() && deliver) || io.out.fire()) // 2. in.fire && deliver => out.fire + assert (!(io.out.fire() && last) || io.que.ready) // 3. out.fire && last => que.ready + assert (!io.que.fire() || (!count_valid || io.out.fire())) // 4. que.fire => !count_valid || (out.fire && last) + // Proofs: + // 1. out.fire => out.ready && (count_valid && in.valid && deliver) => (count_valid && out.ready) && in.valid => in.fire + // 2. in.fire && deliver => in.valid && count_valid && [(out.ready || !deliver) && deliver] => + // in.valid && count_valid && deliver && out.ready => out.fire + // 3. out.fire && last => out.valid && out.ready && last => in.valid && out.ready && last => que.ready + // 4. que.fire => que.valid && (!count_valid || (in.valid && out.ready && last)) + // => !count_valid || (count_valid && in.valid && out.ready && [last => deliver]) + // => !count_valid || (out.valid && out.ready && last) + + when (io.que.fire()) { + // Theorem 4 makes this safe; we overwrite garbage or last output + count_valid := Bool(true) + count_bits := io.que.bits + multi_op := io.que.bits =/= UInt(0) + } .elsewhen (io.in.fire()) { + count_valid := !last // false => !que.valid + count_bits := count_bits - UInt(1) + // Proof: in.fire && [last => deliver] =2=> out.fire && last =3=> que.ready + // !que.fire && que.ready => !que.valid + } + + // Safe by Theorem 1 + io.out.bits := io.in.bits + io.out.bits.g_type := Mux(multi_op, Mux(in_get, Grant.getDataBlockType, Grant.putAckType), io.in.bits.g_type) +} + +class TileLinkFragmenter(depth: Int = 1)(implicit p: Parameters) extends TLModule()(p) { + val io = new Bundle { + val in = new ClientUncachedTileLinkIO().flip + val out = new ClientUncachedTileLinkIO + } + + // TL2: + // supportsAcquire = false + // modify all outward managers to supportsMultibeat = true + // assert: all managers must behaveFIFO (not inspect duplicated id field) + + val source = Module(new TileLinkFragmenterSource) + val sink = Module(new TileLinkFragmenterSink) + sink.io.que <> Queue(source.io.que, depth) + + source.io.in <> io.in.acquire + io.out.acquire <> source.io.out + sink.io.in <> io.out.grant + io.in.grant <> sink.io.out +} + +object TileLinkFragmenter { + // Pass the source/client to fragment + def apply(source: ClientUncachedTileLinkIO, depth: Int = 1)(implicit p: Parameters): ClientUncachedTileLinkIO = { + val fragmenter = Module(new TileLinkFragmenter(depth)) + fragmenter.io.in <> source + fragmenter.io.out + } +} From 11ec5b2cf4087048342eeb90e0c5e5490cc5a6f7 Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Wed, 20 Jul 2016 16:05:28 -0700 Subject: [PATCH 685/688] bram: don't deal with multibeat; rely on the fragmenter --- uncore/src/main/scala/devices/Bram.scala | 97 +++++++++--------------- 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/uncore/src/main/scala/devices/Bram.scala b/uncore/src/main/scala/devices/Bram.scala index e472b697..1b8c5194 100644 --- a/uncore/src/main/scala/devices/Bram.scala +++ b/uncore/src/main/scala/devices/Bram.scala @@ -11,78 +11,53 @@ class BRAMSlave(depth: Int)(implicit val p: Parameters) extends Module with HasTileLinkParameters { val io = new ClientUncachedTileLinkIO().flip - val bram = SeqMem(depth, Bits(width = tlDataBits)) + // For TL2: + // supportsAcquire = false + // supportsMultibeat = false + // supportsHint = false + // supportsAtomic = false - val fire_acq = io.acquire.fire() - val fire_gnt = io.grant.fire() + // Timing-wise, we assume the input is coming out of registers + // since you probably needed a TileLinkFragmenter infront of us - val state_getblk = Reg(init = Bool(false)) - val state_putblk = Reg(init = Bool(false)) - val state_init = !(state_getblk || state_putblk) + // Thus, only one pipeline stage: the grant result + val g_valid = RegInit(Bool(false)) + val g_bits = Reg(new Grant) - private def last(acq: AcquireMetadata) = - (acq.addr_beat === UInt(tlDataBeats-1)) + // Just pass the pipeline straight through + io.grant.valid := g_valid + io.grant.bits := g_bits + io.acquire.ready := !g_valid || io.grant.ready - val s0_acq = io.acquire.bits - val s0_last = last(s0_acq) + val acq_get = io.acquire.bits.isBuiltInType(Acquire.getType) + val acq_put = io.acquire.bits.isBuiltInType(Acquire.putType) + val acq_addr = Cat(io.acquire.bits.addr_block, io.acquire.bits.addr_beat) - val s1_acq = RegEnable(s0_acq, fire_acq) - val s1_last = last(s1_acq) + val bram = Mem(depth, Bits(width = tlDataBits)) - val (is_get :: is_getblk :: is_put :: is_putblk :: Nil) = - Seq(Acquire.getType, Acquire.getBlockType, - Acquire.putType, Acquire.putBlockType).map(s0_acq.isBuiltInType _) + val ren = acq_get && io.acquire.fire() + val wen = acq_put && io.acquire.fire() - val is_read = is_get || is_getblk - val is_write = is_put || is_putblk - val ren_getblk = state_getblk && !s1_last + when (io.grant.fire()) { + g_valid := Bool(false) + } - val s0_valid = (fire_acq && (!is_putblk || s0_last)) || ren_getblk - val s1_valid = RegNext(s0_valid, Bool(false)) + when (io.acquire.fire()) { + g_valid := Bool(true) + g_bits := Grant( + is_builtin_type = Bool(true), + g_type = io.acquire.bits.getBuiltInGrantType(), + client_xact_id = io.acquire.bits.client_xact_id, + manager_xact_id = UInt(0), + addr_beat = io.acquire.bits.addr_beat, + data = UInt(0)) + } - val ren = (fire_acq && is_read) || ren_getblk - val wen = (fire_acq && is_write) - - val s0_addr = Cat(s0_acq.addr_block, s0_acq.addr_beat) - val s1_addr_beat = s1_acq.addr_beat + Mux(io.grant.ready, UInt(1), UInt(0)) - val s1_addr = Cat(s1_acq.addr_block, s1_addr_beat) - - val raddr = Mux(state_getblk, s1_addr, s0_addr) - val waddr = s0_addr - - val rdata = bram.read(raddr, ren) - val wdata = s0_acq.data - val wmask = s0_acq.wmask() when (wen) { - bram.write(waddr, wdata) - assert(wmask.andR, "BRAMSlave: partial write masks not supported") + bram.write(acq_addr, io.acquire.bits.data) + assert(io.acquire.bits.wmask().andR, "BRAMSlave: partial write masks not supported") } - - val stall = io.grant.valid && !io.grant.ready - io.acquire.ready := state_init && !stall - - when (fire_acq) { - state_getblk := is_getblk - state_putblk := is_putblk && s0_last - } - - when (state_getblk && fire_gnt) { - s1_acq.addr_beat := s1_addr_beat - state_getblk := !s1_last - } - - when (state_putblk && fire_gnt) { - state_putblk := Bool(false) - } - - io.grant.valid := s1_valid - io.grant.bits := Grant( - is_builtin_type = Bool(true), - g_type = s1_acq.getBuiltInGrantType(), - client_xact_id = s1_acq.client_xact_id, - manager_xact_id = UInt(0), - addr_beat = s1_acq.addr_beat, - data = rdata) + io.grant.bits.data := RegEnable(bram.read(acq_addr), ren) } class HastiRAM(depth: Int)(implicit p: Parameters) extends HastiModule()(p) { From 82bbbf908df9f7ace069b39c327d6a92f435de92 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Tue, 26 Jul 2016 12:31:08 -0700 Subject: [PATCH 686/688] Fix L2 Writeback deadlock issue The deadlock condition occurs when the acquire tracker attempts to request a writeback while the writeback unit is still busy and a voluntary release for the block to be written back is coming in. The voluntary release cannot be accepted because it conflicts with the acquire tracker. The acquire tracker can't merge the voluntary release because it is waiting to send the writeback. The writeback can't progress because the release it is waiting on is behind the voluntary release. The solution to this is to break the atomicity guarantee between the acquire tracker and the writeback unit. This allows the voluntary release tracker to take the voluntary release before the writeback unit accepts the conflicting request. This causes a potential race condition for the metadata array. The solution to this is to have the writeback unit re-read the metadata after accepting a request. --- uncore/src/main/scala/agents/Cache.scala | 126 ++++++++++++++------ uncore/src/main/scala/agents/Trackers.scala | 19 ++- 2 files changed, 101 insertions(+), 44 deletions(-) diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index 5cfb819f..fc67c391 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -188,12 +188,26 @@ trait HasOuterCacheParameters extends HasCacheParameters with HasCoherenceAgentP val idxLSB = cacheIdBits val idxMSB = idxLSB + idxBits - 1 val tagLSB = idxLSB + idxBits - def inSameSet(block: HasCacheBlockAddress, addr: UInt): Bool = { - block.addr_block(idxMSB,idxLSB) === addr(idxMSB,idxLSB) - } - def haveSameTag(block: HasCacheBlockAddress, addr: UInt): Bool = { - block.addr_block >> UInt(tagLSB) === addr >> UInt(tagLSB) - } + val tagMSB = tagLSB + tagBits - 1 + + def inSameSet(block_a: HasCacheBlockAddress, block_b: HasCacheBlockAddress): Bool = + inSameSet(block_a, block_b.addr_block) + + def inSameSet(block: HasCacheBlockAddress, addr: UInt): Bool = + inSet(block, addr(idxMSB, idxLSB)) + + def inSet(block: HasCacheBlockAddress, idx: UInt): Bool = + block.addr_block(idxMSB,idxLSB) === idx + + def haveSameTag(block: HasCacheBlockAddress, addr: UInt): Bool = + hasTag(block, addr(tagMSB, tagLSB)) + + def hasTag(block: HasCacheBlockAddress, tag: UInt): Bool = + block.addr_block(tagMSB, tagLSB) === tag + + def isSameBlock(block: HasCacheBlockAddress, tag: UInt, idx: UInt) = + hasTag(block, tag) && inSet(block, idx) + //val blockAddrBits = p(TLBlockAddrBits) val refillCyclesPerBeat = outerDataBits/rowBits val refillCycles = refillCyclesPerBeat*outerDataBeats @@ -211,10 +225,13 @@ abstract class L2HellaCacheModule(implicit val p: Parameters) extends Module with HasOuterCacheParameters { def doInternalOutputArbitration[T <: Data : ClassTag]( out: DecoupledIO[T], - ins: Seq[DecoupledIO[T]]) { + ins: Seq[DecoupledIO[T]], + block_transfer: T => Bool = (t: T) => Bool(false)) { val arb = Module(new RRArbiter(out.bits, ins.size)) - out <> arb.io.out - arb.io.in <> ins + out.valid := arb.io.out.valid && !block_transfer(arb.io.out.bits) + out.bits := arb.io.out.bits + arb.io.out.ready := out.ready && !block_transfer(arb.io.out.bits) + arb.io.in <> ins } def doInternalInputRouting[T <: Bundle with HasL2Id](in: ValidIO[T], outs: Seq[ValidIO[T]]) { @@ -297,6 +314,9 @@ class L2MetaRWIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) with HasL2MetaReadIO with HasL2MetaWriteIO +class L2MetaReadOnlyIO(implicit p: Parameters) extends L2HellaCacheBundle()(p) + with HasL2MetaReadIO + trait HasL2MetaRWIO extends HasOuterCacheParameters { val meta = new L2MetaRWIO } @@ -462,11 +482,24 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) (nReleaseTransactors until nTransactors).map(id => Module(new CacheAcquireTracker(id))) val trackerList = irelTrackerList ++ iacqTrackerList - + + // Don't allow a writeback request to go through if we are taking + // a voluntary release for the same block. + // The writeback can go forward once the voluntary release is handled + def writebackConflictsWithVolRelease(wb: L2WritebackReq): Bool = + irelTrackerList + .map(tracker => + !tracker.io.alloc.idle && + isSameBlock(tracker.io.alloc, wb.tag, wb.idx)) + .reduce(_ || _) || + (io.inner.release.valid && + isSameBlock(io.inner.release.bits, wb.tag, wb.idx)) + // WritebackUnit evicts data from L2, including invalidating L1s val wb = Module(new L2WritebackUnit(nTransactors)) val trackerAndWbIOs = trackerList.map(_.io) :+ wb.io - doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req)) + doInternalOutputArbitration(wb.io.wb.req, trackerList.map(_.io.wb.req), + block_transfer = writebackConflictsWithVolRelease _) doInternalInputRouting(wb.io.wb.resp, trackerList.map(_.io.wb.resp)) // Propagate incoherence flags @@ -476,7 +509,7 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) val irel_vs_iacq_conflict = io.inner.acquire.valid && io.inner.release.valid && - inSameSet(io.inner.acquire.bits, io.inner.release.bits.addr_block) + inSameSet(io.inner.acquire.bits, io.inner.release.bits) doInputRoutingWithAllocation( in = io.inner.acquire, outs = trackerList.map(_.io.inner.acquire), @@ -508,11 +541,11 @@ class TSHRFile(implicit p: Parameters) extends L2HellaCacheModule()(p) io.outer <> outer_arb.io.out // Wire local memory arrays - doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read)) + doInternalOutputArbitration(io.meta.read, trackerList.map(_.io.meta.read) :+ wb.io.meta.read) doInternalOutputArbitration(io.meta.write, trackerList.map(_.io.meta.write)) doInternalOutputArbitration(io.data.read, trackerList.map(_.io.data.read) :+ wb.io.data.read) doInternalOutputArbitration(io.data.write, trackerList.map(_.io.data.write)) - doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp)) + doInternalInputRouting(io.meta.resp, trackerList.map(_.io.meta.resp) :+ wb.io.meta.resp) doInternalInputRouting(io.data.resp, trackerList.map(_.io.data.resp) :+ wb.io.data.resp) } @@ -684,17 +717,18 @@ trait HasCoherenceMetadataBuffer extends HasOuterCacheParameters } } - def metaRead(port: HasL2MetaReadIO, next_state: UInt) { + def metaRead(port: HasL2MetaReadIO, next_state: UInt, way_en_known: Bool = Bool(false)) { port.read.valid := state === s_meta_read port.read.bits.id := UInt(trackerId) port.read.bits.idx := xact_addr_idx port.read.bits.tag := xact_addr_tag + port.read.bits.way_en := Mux(way_en_known, xact_way_en, ~UInt(0, nWays)) when(state === s_meta_read && port.read.ready) { state := s_meta_resp } when(state === s_meta_resp && port.resp.valid) { xact_old_meta := port.resp.bits.meta - xact_way_en := port.resp.bits.way_en + when (!way_en_known) { xact_way_en := port.resp.bits.way_en } state := next_state } } @@ -716,7 +750,6 @@ trait TriggersWritebacks extends HasCoherenceMetadataBuffer { wb.req.bits.id := UInt(trackerId) wb.req.bits.idx := xact_addr_idx wb.req.bits.tag := xact_old_meta.tag - wb.req.bits.coh := xact_old_meta.coh wb.req.bits.way_en := xact_way_en when(state === s_wb_req && wb.req.ready) { state := s_wb_resp } @@ -808,14 +841,12 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) val pending_coh_on_miss = HierarchicalMetadata.onReset - - // Setup IOs used for routing in the parent - val before_wb_alloc = state isOneOf (s_meta_read, s_meta_resp, s_wb_req) + val before_wb_req = state.isOneOf(s_meta_read, s_meta_resp) routeInParent( iacqMatches = inSameSet(_, xact_addr_block), - irelMatches = (irel: HasCacheBlockAddress) => - Mux(before_wb_alloc, inSameSet(irel, xact_addr_block), exactAddrMatch(irel)), + irelMatches = (irel: HasCacheBlockAddress) => + Mux(before_wb_req, inSameSet(irel, xact_addr_block), exactAddrMatch(irel)), iacqCanAlloc = Bool(true)) // TileLink allows for Gets-under-Get @@ -868,7 +899,7 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) val needs_inner_probes = tag_match && coh.inner.requiresProbes(xact_iacq) val should_update_meta = !tag_match && xact_allocate || is_hit && pending_coh_on_hit =/= coh - def full_representation = io.meta.resp.bits.meta.coh.inner.full() + def full_representation = coh.inner.full() metaRead( io.meta, @@ -996,7 +1027,9 @@ class CacheAcquireTracker(trackerId: Int)(implicit p: Parameters) quiesce(Mux(pending_meta_write, s_meta_write, s_idle)) { clearWmaskBuffer() } } -class L2WritebackReq(implicit p: Parameters) extends L2Metadata()(p) with HasL2Id { +class L2WritebackReq(implicit p: Parameters) + extends L2HellaCacheBundle()(p) with HasL2Id { + val tag = Bits(width = tagBits) val idx = Bits(width = idxBits) val way_en = Bits(width = nWays) } @@ -1012,9 +1045,10 @@ trait HasL2WritebackIO extends HasOuterCacheParameters { val wb = new L2WritebackIO() } -class L2WritebackUnitIO(implicit p: Parameters) extends HierarchicalXactTrackerIO()(p) - with HasL2DataRWIO { +class L2WritebackUnitIO(implicit p: Parameters) + extends HierarchicalXactTrackerIO()(p) with HasL2DataRWIO { val wb = new L2WritebackIO().flip() + val meta = new L2MetaReadOnlyIO } class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTracker()(p) @@ -1039,6 +1073,18 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr // Start the writeback sub-transaction io.wb.req.ready := state === s_idle + val coh = io.meta.resp.bits.meta.coh + val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() + val needs_outer_release = coh.outer.requiresVoluntaryWriteback() + def full_representation = coh.inner.full() + + // Even though we already read the metadata in the acquire tracker that + // sent the writeback request, we have to read it again in the writeback + // unit, since it may have been updated in the meantime. + metaRead(io.meta, + next_state = Mux(needs_inner_probes, s_inner_probe, s_busy), + way_en_known = Bool(true)) + // Track which clients yet need to be probed and make Probe message innerProbe( inner_coh.makeProbeForVoluntaryWriteback(curr_probe_dst, xact_addr_block), @@ -1050,7 +1096,7 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr def irel_can_merge = io.irel().conflicts(xact_addr_block) && io.irel().isVoluntary() && - (state =/= s_idle) && + !state.isOneOf(s_idle, s_meta_read, s_meta_resp) && !(state === s_busy && all_pending_done) && !vol_ignt_counter.pending && !blockInnerRelease() @@ -1062,18 +1108,16 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr mergeDataInner(io.inner.release) // If a release didn't write back data, have to read it from data array - readDataArray(drop_pending_bit = dropPendingBitWhenBeatHasData(io.inner.release)) - - val coh = io.wb.req.bits.coh - val needs_inner_probes = coh.inner.requiresProbesOnVoluntaryWriteback() - val needs_outer_release = coh.outer.requiresVoluntaryWriteback() + readDataArray( + drop_pending_bit = dropPendingBitWhenBeatHasData(io.inner.release)) // Once the data is buffered we can write it back to outer memory outerRelease( coh = outer_coh, data = data_buffer(vol_ognt_counter.up.idx), add_pending_data_bits = addPendingBitInternal(io.data.resp), - add_pending_send_bit = io.wb.req.fire() && needs_outer_release) + add_pending_send_bit = io.meta.resp.valid && needs_outer_release) + // Respond to the initiating transaction handler signalling completion of the writeback io.wb.resp.valid := state === s_busy && all_pending_done @@ -1081,17 +1125,21 @@ class L2WritebackUnit(val trackerId: Int)(implicit p: Parameters) extends XactTr quiesce() {} - def full_representation = io.wb.req.bits.coh.inner.full() // State machine updates and transaction handler metadata intialization when(state === s_idle && io.wb.req.valid) { xact_id := io.wb.req.bits.id xact_way_en := io.wb.req.bits.way_en xact_addr_block := (if (cacheIdBits == 0) Cat(io.wb.req.bits.tag, io.wb.req.bits.idx) else Cat(io.wb.req.bits.tag, io.wb.req.bits.idx, UInt(cacheId, cacheIdBits))) - when(needs_inner_probes) { initializeProbes() } - pending_reads := Mux(needs_outer_release, ~UInt(0, width = innerDataBeats), UInt(0)) - pending_resps := UInt(0) - pending_coh := coh - state := Mux(needs_inner_probes, s_inner_probe, s_busy) + state := s_meta_read } + + when (state === s_meta_resp && io.meta.resp.valid) { + pending_reads := Fill(innerDataBeats, needs_outer_release) + pending_coh := coh + when(needs_inner_probes) { initializeProbes() } + } + + assert(!io.meta.resp.valid || io.meta.resp.bits.tag_match, + "L2 requested Writeback for block not present in cache") } diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index d695df8d..c2a7d301 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -7,6 +7,7 @@ import uncore.coherence._ import uncore.tilelink._ import uncore.util._ import uncore.Util._ +import junctions._ import cde.{Field, Parameters} import scala.math.max @@ -18,12 +19,18 @@ class TrackerAllocation extends Bundle { val should = Bool(INPUT) } +class TrackerAllocationIO(implicit val p: Parameters) + extends ParameterizedBundle()(p) + with HasCacheBlockAddress { + val iacq = new TrackerAllocation + val irel = new TrackerAllocation + val oprb = new TrackerAllocation + val idle = Bool(OUTPUT) +} + trait HasTrackerAllocationIO extends Bundle { - val alloc = new Bundle { - val iacq = new TrackerAllocation - val irel = new TrackerAllocation - val oprb = new TrackerAllocation - } + implicit val p: Parameters + val alloc = new TrackerAllocationIO } class ManagerXactTrackerIO(implicit p: Parameters) extends ManagerTLIO()(p) @@ -420,6 +427,8 @@ trait RoutesInParent extends HasBlockAddressBuffer io.alloc.iacq.can := state === s_idle && iacqCanAlloc io.alloc.irel.can := state === s_idle && irelCanAlloc io.alloc.oprb.can := state === s_idle && oprbCanAlloc + io.alloc.addr_block := xact_addr_block + io.alloc.idle := state === s_idle } } From 15d1aa934655e69e4c2c28ee0bda389bf49a1e23 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 27 Jul 2016 16:47:22 -0700 Subject: [PATCH 687/688] make sure TrackerAllocationIO addr_block has correct direction set --- uncore/src/main/scala/agents/Trackers.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/agents/Trackers.scala b/uncore/src/main/scala/agents/Trackers.scala index c2a7d301..80d7a409 100644 --- a/uncore/src/main/scala/agents/Trackers.scala +++ b/uncore/src/main/scala/agents/Trackers.scala @@ -26,6 +26,7 @@ class TrackerAllocationIO(implicit val p: Parameters) val irel = new TrackerAllocation val oprb = new TrackerAllocation val idle = Bool(OUTPUT) + override val addr_block = UInt(OUTPUT, tlBlockAddrBits) } trait HasTrackerAllocationIO extends Bundle { From fbcc7317cfa4dc7a532acac7bc9066f673eaf7aa Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Wed, 27 Jul 2016 18:39:33 -0700 Subject: [PATCH 688/688] make sure PseudoLRU is given power of 2 ways --- uncore/src/main/scala/agents/Cache.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/uncore/src/main/scala/agents/Cache.scala b/uncore/src/main/scala/agents/Cache.scala index fc67c391..80f8f8dd 100644 --- a/uncore/src/main/scala/agents/Cache.scala +++ b/uncore/src/main/scala/agents/Cache.scala @@ -82,6 +82,7 @@ class SeqRandom(n_ways: Int) extends SeqReplacementPolicy { class PseudoLRU(n: Int) { + require(isPow2(n)) val state_reg = Reg(Bits(width = n)) def access(way: UInt) { state_reg := get_next_state(state_reg,way)