Allow the number of memory channels to be picked at runtime

We're building a chip with 8 memory channels.  Since this will require a
complicated test setup we want to also be able to bring up the chip with fewer
memory channels.  This commit adds a SCR that controls the number of active
memory channels on a chip.  Toggling this SCR will scramble memory and drop
Nasti messages, so it's only possible to change while the chip is booting.

By default this just adds a 1-bit SCR, which essentially no extra logic.

When multiple memory channel configurations are enabled at elaboration time, a
NastiMemoryInterconnect is generated for each channel configuration.  The
number of outstanding misses is increased to coorespond to the maximum number
of banks per memory channel (added as a parameter), which I believe is
necessary to avoid deadlock in the memory system.

A configuration is added that supports 8 memory channels but has only 1 enabled
by default.

src/main/scala/Configs.scala

@@ -90,7 +90,7 @@ class DefaultConfig extends Config (
       case MIFTagBits => // Bits needed at the L2 agent
                          log2Up(site(NAcquireTransactors)+2) +
                          // Bits added by NASTI interconnect
-                         max(log2Up(site(NBanksPerMemoryChannel)),
+                         max(log2Up(site(MaxBanksPerMemoryChannel)),
                             (if (site(UseDma)) 3 else 2))
       case MIFDataBits => 64
       case MIFAddrBits => site(PAddrBits) - site(CacheBlockOffsetBits)
@@ -216,7 +216,9 @@ class DefaultConfig extends Config (
       case NTiles => Knob("NTILES")
       case NMemoryChannels => Dump("N_MEM_CHANNELS", 1)
       case NBanksPerMemoryChannel => Knob("NBANKS_PER_MEM_CHANNEL")
-      case NOutstandingMemReqsPerChannel => site(NBanksPerMemoryChannel)*(site(NAcquireTransactors)+2)
+      case MemoryChannelMuxConfigs => Dump("MEMORY_CHANNEL_MUX_CONFIGS", List( site(NMemoryChannels) ))
+      case MaxBanksPerMemoryChannel => site(NBanksPerMemoryChannel) * site(NMemoryChannels) / site(MemoryChannelMuxConfigs).sortWith{_ < _}(0)
+      case NOutstandingMemReqsPerChannel => site(MaxBanksPerMemoryChannel)*(site(NAcquireTransactors)+2)
       case BankIdLSB => 0
       case CacheBlockBytes => Dump("CACHE_BLOCK_BYTES", 64)
       case CacheBlockOffsetBits => log2Up(here(CacheBlockBytes))
@@ -273,6 +275,11 @@ class With4MemoryChannels extends Config(
     case NMemoryChannels => Dump("N_MEM_CHANNELS", 4)
   }
 )
+class With8MemoryChannels extends Config(
+  (pname,site,here) => pname match {
+    case NMemoryChannels => Dump("N_MEM_CHANNELS", 8)
+  }
+)
 
 class WithL2Cache extends Config(
   (pname,site,here) => pname match {
@@ -430,3 +437,12 @@ class SmallL2Config extends Config(
 class SingleChannelBenchmarkConfig extends Config(new WithL2Capacity256 ++ new DefaultL2Config)
 class DualChannelBenchmarkConfig extends Config(new With2MemoryChannels ++ new SingleChannelBenchmarkConfig)
 class QuadChannelBenchmarkConfig extends Config(new With4MemoryChannels ++ new SingleChannelBenchmarkConfig)
+class OctoChannelBenchmarkConfig extends Config(new With8MemoryChannels ++ new SingleChannelBenchmarkConfig)
+
+class WithOneOrMaxChannels extends Config(
+  (pname, site, here) => pname match {
+    case MemoryChannelMuxConfigs => Dump("MEMORY_CHANNEL_MUX_CONFIGS", List(1, site(NMemoryChannels)))
+  }
+)
+
+class OneOrEightChannelBenchmarkConfig extends Config(new WithOneOrMaxChannels ++ new With8MemoryChannels ++ new SingleChannelBenchmarkConfig)

src/main/scala/RocketChip.scala

@@ -17,6 +17,10 @@ case object NTiles extends Field[Int]
 case object NMemoryChannels extends Field[Int]
 /** Number of banks per memory channel */
 case object NBanksPerMemoryChannel extends Field[Int]
+/** Maximum number of banks per memory channel, when configurable */
+case object MaxBanksPerMemoryChannel extends Field[Int]
+/** Dynamic memory channel configurations */
+case object MemoryChannelMuxConfigs extends Field[List[Int]]
 /** Least significant bit of address used for bank partitioning */
 case object BankIdLSB extends Field[Int]
 /** Number of outstanding memory requests */
@@ -56,6 +60,7 @@ trait HasTopLevelParameters {
   lazy val scrAddrBits = log2Up(nSCR)
   lazy val scrDataBits = 64
   lazy val scrDataBytes = scrDataBits / 8
+  lazy val memoryChannelMuxConfigs = p(MemoryChannelMuxConfigs)
   //require(lsb + log2Up(nBanks) < mifAddrBits)
 }
 
@@ -167,6 +172,14 @@ class Uncore(implicit val p: Parameters) extends Module
   scrFile.io.smi <> scrArb.io.out
   // scrFile.io.scr <> (... your SCR connections ...)
 
+  // Configures the enabled memory channels.  This can't be changed while the
+  // chip is actively using memory, as it both drops Nasti messages and garbles
+  // all of memory.
+  val memory_channel_mux_select = scrFile.io.scr.attach(
+    Reg(UInt(width = log2Up(memoryChannelMuxConfigs.size))),
+    "MEMORY_CHANNEL_MUX_SELECT")
+  outmemsys.io.memory_channel_mux_select := memory_channel_mux_select
+
   val deviceTree = Module(new NastiROM(p(DeviceTree).toSeq))
   deviceTree.io <> outmemsys.io.deviceTree
 
@@ -195,6 +208,7 @@ class OuterMemorySystem(implicit val p: Parameters) extends Module with HasTopLe
     val mem = Vec(nMemChannels, new NastiIO)
     val mem_backup = new MemSerializedIO(htifW)
     val mem_backup_en = Bool(INPUT)
+    val memory_channel_mux_select = UInt(INPUT, log2Up(memoryChannelMuxConfigs.size))
     val csr = Vec(nTiles, new SmiIO(xLen, csrAddrBits))
     val scr = new SmiIO(xLen, scrAddrBits)
     val deviceTree = new NastiIO
@@ -253,7 +267,20 @@ class OuterMemorySystem(implicit val p: Parameters) extends Module with HasTopLe
   }
 
   val mmio_ic = Module(new NastiRecursiveInterconnect(nMasters, nSlaves, addrMap, mmioBase))
-  val mem_ic = Module(new NastiMemoryInterconnect(nBanksPerMemChannel, nMemChannels))
+
+  val channelConfigs = p(MemoryChannelMuxConfigs)
+  Predef.assert(channelConfigs.sortWith(_ > _)(0) == nMemChannels,
+                "More memory channels elaborated than can be enabled")
+  val mem_ic =
+    if (channelConfigs.size == 1) {
+      val ic = Module(new NastiMemoryInterconnect(nBanksPerMemChannel, nMemChannels))
+      ic
+    } else {
+      val nBanks = nBanksPerMemChannel * nMemChannels
+      val ic = Module(new NastiMemorySelector(nBanks, nMemChannels, channelConfigs))
+      ic.io.select := io.memory_channel_mux_select
+      ic
+    }
 
   for ((bank, i) <- managerEndpoints.zipWithIndex) {
     val unwrap = Module(new ClientTileLinkIOUnwrapper()(outerTLParams))