Allow the number of memory channels to be picked at runtime

We're building a chip with 8 memory channels.  Since this will require a
complicated test setup we want to also be able to bring up the chip with fewer
memory channels.  This commit adds a SCR that controls the number of active
memory channels on a chip.  Toggling this SCR will scramble memory and drop
Nasti messages, so it's only possible to change while the chip is booting.

By default this just adds a 1-bit SCR, which essentially no extra logic.

When multiple memory channel configurations are enabled at elaboration time, a
NastiMemoryInterconnect is generated for each channel configuration.  The
number of outstanding misses is increased to coorespond to the maximum number
of banks per memory channel (added as a parameter), which I believe is
necessary to avoid deadlock in the memory system.

A configuration is added that supports 8 memory channels but has only 1 enabled
by default.

context-dependent-environments

@@ -1 +1 @@
-Subproject commit 62cf74d84e1bfd456ff967c321fd612d94f015be
+Subproject commit 867131718f7544268b5934c866ddf750f8cfa2bd

csrc/htif_emulator.h

@@ -7,12 +7,20 @@
 
 class htif_emulator_t : public htif_pthread_t
 {
+ int memory_channel_mux_select;
+
  public:
   htif_emulator_t(uint32_t memsz_mb, const std::vector<std::string>& args)
-    : htif_pthread_t(args)
+    : htif_pthread_t(args),
+      memory_channel_mux_select(0)
   {
     this->_memsz_mb = memsz_mb;
-  }
+
+    for (const auto& arg: args) {
+      if (!strncmp(arg.c_str(), "+memory_channel_mux_select=", 27))
+        memory_channel_mux_select = atoi(arg.c_str()+27);
+    }
+ }
 
   void set_clock_divisor(int divisor, int hold_cycles)
   {
@@ -22,6 +30,7 @@ class htif_emulator_t : public htif_pthread_t
   void start()
   {
     set_clock_divisor(5, 2);
+    write_cr(-1, UNCORE_SCR__MEMORY_CHANNEL_MUX_SELECT, memory_channel_mux_select);
     htif_pthread_t::start();
   }
 

csrc/vcs_main.rocketTestHarness.cc

@@ -14,11 +14,24 @@ extern "C" {
 
 extern int vcs_main(int argc, char** argv);
 
+static const int MEMORY_CHANNEL_MUX_CONFIGS[] = {
+#ifdef MEMORY_CHANNEL_MUX_CONFIGS__0
+  MEMORY_CHANNEL_MUX_CONFIGS__0,
+#endif
+#ifdef MEMORY_CHANNEL_MUX_CONFIGS__1
+  MEMORY_CHANNEL_MUX_CONFIGS__1,
+#endif
+#ifdef MEMORY_CHANNEL_MUX_CONFIGS__2
+#error "Add a preprocessor repeat macro"
+#endif
+};
+
 static htif_emulator_t* htif;
 static unsigned htif_bytes = HTIF_WIDTH / 8;
 static mm_t* mm[N_MEM_CHANNELS];
 static const char* loadmem;
 static bool dramsim = false;
+static int memory_channel_mux_select = 0;
 
 void htif_fini(vc_handle failure)
 {
@@ -37,21 +50,25 @@ int main(int argc, char** argv)
       dramsim = true;
     else if (!strncmp(argv[i], "+loadmem=", 9))
       loadmem = argv[i]+9;
+    else if (!strncmp(argv[i], "+memory_channel_mux_select=", 27))
+      memory_channel_mux_select = atoi(argv[i]+27);
   }
 
+  int enabled_mem_channels = MEMORY_CHANNEL_MUX_CONFIGS[memory_channel_mux_select];
+
   htif = new htif_emulator_t(memsz_mb,
           std::vector<std::string>(argv + 1, argv + argc));
 
   for (int i=0; i<N_MEM_CHANNELS; i++) {
     mm[i] = dramsim ? (mm_t*)(new mm_dramsim2_t) : (mm_t*)(new mm_magic_t);
-    mm[i]->init(MEM_SIZE / N_MEM_CHANNELS, MEM_DATA_BITS / 8, CACHE_BLOCK_BYTES);
+    mm[i]->init(MEM_SIZE / enabled_mem_channels, MEM_DATA_BITS / 8, CACHE_BLOCK_BYTES);
   }
 
   if (loadmem) {
     void *mems[N_MEM_CHANNELS];
     for (int i = 0; i < N_MEM_CHANNELS; i++)
       mems[i] = mm[i]->get_data();
-    load_mem(mems, loadmem, CACHE_BLOCK_BYTES, N_MEM_CHANNELS);
+    load_mem(mems, loadmem, CACHE_BLOCK_BYTES, enabled_mem_channels);
   }
 
   vcs_main(argc, argv);

junctions

@@ -1 +1 @@
-Subproject commit 287eca2386e1cd0386ecb6072c7522c1ede5ea4d
+Subproject commit 5e9160b48a725294c22f4c84072c1a06f9295a29

src/main/scala/Configs.scala

@@ -90,7 +90,7 @@ class DefaultConfig extends Config (
       case MIFTagBits => // Bits needed at the L2 agent
                          log2Up(site(NAcquireTransactors)+2) +
                          // Bits added by NASTI interconnect
-                         max(log2Up(site(NBanksPerMemoryChannel)),
+                         max(log2Up(site(MaxBanksPerMemoryChannel)),
                             (if (site(UseDma)) 3 else 2))
       case MIFDataBits => 64
       case MIFAddrBits => site(PAddrBits) - site(CacheBlockOffsetBits)
@@ -216,7 +216,9 @@ class DefaultConfig extends Config (
       case NTiles => Knob("NTILES")
       case NMemoryChannels => Dump("N_MEM_CHANNELS", 1)
       case NBanksPerMemoryChannel => Knob("NBANKS_PER_MEM_CHANNEL")
-      case NOutstandingMemReqsPerChannel => site(NBanksPerMemoryChannel)*(site(NAcquireTransactors)+2)
+      case MemoryChannelMuxConfigs => Dump("MEMORY_CHANNEL_MUX_CONFIGS", List( site(NMemoryChannels) ))
+      case MaxBanksPerMemoryChannel => site(NBanksPerMemoryChannel) * site(NMemoryChannels) / site(MemoryChannelMuxConfigs).sortWith{_ < _}(0)
+      case NOutstandingMemReqsPerChannel => site(MaxBanksPerMemoryChannel)*(site(NAcquireTransactors)+2)
       case BankIdLSB => 0
       case CacheBlockBytes => Dump("CACHE_BLOCK_BYTES", 64)
       case CacheBlockOffsetBits => log2Up(here(CacheBlockBytes))
@@ -273,6 +275,11 @@ class With4MemoryChannels extends Config(
     case NMemoryChannels => Dump("N_MEM_CHANNELS", 4)
   }
 )
+class With8MemoryChannels extends Config(
+  (pname,site,here) => pname match {
+    case NMemoryChannels => Dump("N_MEM_CHANNELS", 8)
+  }
+)
 
 class WithL2Cache extends Config(
   (pname,site,here) => pname match {
@@ -430,3 +437,12 @@ class SmallL2Config extends Config(
 class SingleChannelBenchmarkConfig extends Config(new WithL2Capacity256 ++ new DefaultL2Config)
 class DualChannelBenchmarkConfig extends Config(new With2MemoryChannels ++ new SingleChannelBenchmarkConfig)
 class QuadChannelBenchmarkConfig extends Config(new With4MemoryChannels ++ new SingleChannelBenchmarkConfig)
+class OctoChannelBenchmarkConfig extends Config(new With8MemoryChannels ++ new SingleChannelBenchmarkConfig)
+
+class WithOneOrMaxChannels extends Config(
+  (pname, site, here) => pname match {
+    case MemoryChannelMuxConfigs => Dump("MEMORY_CHANNEL_MUX_CONFIGS", List(1, site(NMemoryChannels)))
+  }
+)
+
+class OneOrEightChannelBenchmarkConfig extends Config(new WithOneOrMaxChannels ++ new With8MemoryChannels ++ new SingleChannelBenchmarkConfig)

src/main/scala/RocketChip.scala

@@ -17,6 +17,10 @@ case object NTiles extends Field[Int]
 case object NMemoryChannels extends Field[Int]
 /** Number of banks per memory channel */
 case object NBanksPerMemoryChannel extends Field[Int]
+/** Maximum number of banks per memory channel, when configurable */
+case object MaxBanksPerMemoryChannel extends Field[Int]
+/** Dynamic memory channel configurations */
+case object MemoryChannelMuxConfigs extends Field[List[Int]]
 /** Least significant bit of address used for bank partitioning */
 case object BankIdLSB extends Field[Int]
 /** Number of outstanding memory requests */
@@ -56,6 +60,7 @@ trait HasTopLevelParameters {
   lazy val scrAddrBits = log2Up(nSCR)
   lazy val scrDataBits = 64
   lazy val scrDataBytes = scrDataBits / 8
+  lazy val memoryChannelMuxConfigs = p(MemoryChannelMuxConfigs)
   //require(lsb + log2Up(nBanks) < mifAddrBits)
 }
 
@@ -167,6 +172,14 @@ class Uncore(implicit val p: Parameters) extends Module
   scrFile.io.smi <> scrArb.io.out
   // scrFile.io.scr <> (... your SCR connections ...)
 
+  // Configures the enabled memory channels.  This can't be changed while the
+  // chip is actively using memory, as it both drops Nasti messages and garbles
+  // all of memory.
+  val memory_channel_mux_select = scrFile.io.scr.attach(
+    Reg(UInt(width = log2Up(memoryChannelMuxConfigs.size))),
+    "MEMORY_CHANNEL_MUX_SELECT")
+  outmemsys.io.memory_channel_mux_select := memory_channel_mux_select
+
   val deviceTree = Module(new NastiROM(p(DeviceTree).toSeq))
   deviceTree.io <> outmemsys.io.deviceTree
 
@@ -195,6 +208,7 @@ class OuterMemorySystem(implicit val p: Parameters) extends Module with HasTopLe
     val mem = Vec(nMemChannels, new NastiIO)
     val mem_backup = new MemSerializedIO(htifW)
     val mem_backup_en = Bool(INPUT)
+    val memory_channel_mux_select = UInt(INPUT, log2Up(memoryChannelMuxConfigs.size))
     val csr = Vec(nTiles, new SmiIO(xLen, csrAddrBits))
     val scr = new SmiIO(xLen, scrAddrBits)
     val deviceTree = new NastiIO
@@ -253,7 +267,20 @@ class OuterMemorySystem(implicit val p: Parameters) extends Module with HasTopLe
   }
 
   val mmio_ic = Module(new NastiRecursiveInterconnect(nMasters, nSlaves, addrMap, mmioBase))
-  val mem_ic = Module(new NastiMemoryInterconnect(nBanksPerMemChannel, nMemChannels))
+
+  val channelConfigs = p(MemoryChannelMuxConfigs)
+  Predef.assert(channelConfigs.sortWith(_ > _)(0) == nMemChannels,
+                "More memory channels elaborated than can be enabled")
+  val mem_ic =
+    if (channelConfigs.size == 1) {
+      val ic = Module(new NastiMemoryInterconnect(nBanksPerMemChannel, nMemChannels))
+      ic
+    } else {
+      val nBanks = nBanksPerMemChannel * nMemChannels
+      val ic = Module(new NastiMemorySelector(nBanks, nMemChannels, channelConfigs))
+      ic.io.select := io.memory_channel_mux_select
+      ic
+    }
 
   for ((bank, i) <- managerEndpoints.zipWithIndex) {
     val unwrap = Module(new ClientTileLinkIOUnwrapper()(outerTLParams))