only run the important (high coverage) tests in regression suite

.travis.yml

@@ -49,8 +49,7 @@ script:
   - make vsim-verilog -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION
   - make fsim-verilog -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION
   - make emulator-ndebug -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION
-  - make emulator-asm-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION
+  - make emulator-regression-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION
-  - make emulator-bmark-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION -j1
 
 before_cache:
   - ls -tr regression/install | tail -n+2 | sed s@^@regression/install/@ | xargs rm -rf

regression/Makefile

@@ -44,7 +44,7 @@ $(error Set SUITE to the regression suite you want to run)
 endif
 
 ifeq ($(SUITE),RocketSuite)
-CONFIGS=DefaultConfig DefaultL2Config DefaultBufferlessConfig TinyConfig RoccExampleConfig
+CONFIGS=DefaultConfig DefaultL2Config DefaultBufferlessConfig TinyConfig
 endif
 
 ifeq ($(SUITE),GroundtestSuite)
@@ -65,12 +65,14 @@ EMU_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-debug.sta
 EMU_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-ndebug.stamp)
 EMU_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-asm-tests.stamp)
 EMU_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-bmark-tests.stamp)
+EMU_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-regression-tests.stamp)
 EMU_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-torture-$(TORTURE_CONFIG).stamp)
 
 emulator-debug: $(EMU_DEBUG_STAMPS)
 emulator-ndebug: $(EMU_NDEBUG_STAMPS)
 emulator-asm-tests: $(EMU_ASM_TEST_STAMPS)
 emulator-bmark-tests: $(EMU_BMARK_TEST_STAMPS)
+emulator-regression-tests: $(EMU_REGRESSION_TEST_STAMPS)
 emulator-torture: $(EMU_TORTURE_STAMPS)
 
 VSIM_VERILOG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-verilog.stamp)
@@ -78,6 +80,7 @@ VSIM_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-debug.stamp)
 VSIM_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-ndebug.stamp)
 VSIM_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-asm-tests.stamp)
 VSIM_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-bmark-tests.stamp)
+VSIM_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-regression-tests.stamp)
 VSIM_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-torture-$(TORTURE_CONFIG).stamp)
 
 vsim-verilog: $(VSIM_VERILOG_STAMPS)
@@ -85,6 +88,7 @@ vsim-debug: $(VSIM_DEBUG_STAMPS)
 vsim-ndebug: $(VSIM_NDEBUG_STAMPS)
 vsim-asm-tests: $(VSIM_ASM_TEST_STAMPS)
 vsim-bmark-tests: $(VSIM_BMARK_TEST_STAMPS)
+vsim-regression-tests: $(VSIM_REGRESSION_TEST_STAMPS)
 vsim-torture: $(VSIM_TORTURE_STAMPS)
 
 FSIM_VERILOG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-verilog.stamp)
@@ -92,6 +96,7 @@ FSIM_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-debug.stamp)
 FSIM_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-ndebug.stamp)
 FSIM_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-asm-tests.stamp)
 FSIM_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-bmark-tests.stamp)
+FSIM_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-regression-tests.stamp)
 FSIM_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-torture-$(TORTURE_CONFIG).stamp)
 
 fsim-verilog: $(FSIM_VERILOG_STAMPS)
@@ -99,6 +104,7 @@ fsim-debug: $(FSIM_DEBUG_STAMPS)
 fsim-ndebug: $(FSIM_NDEBUG_STAMPS)
 fsim-asm-tests: $(FSIM_ASM_TEST_STAMPS)
 fsim-bmark-tests: $(FSIM_BMARK_TEST_STAMPS)
+fsim-regression-tests: $(FSIM_REGRESSION_TEST_STAMPS)
 fsim-torture: $(FSIM_TORTURE_STAMPS)
 
 submodule_names = chisel2 chisel3 context-dependent-environments dramsim2 firrtl groundtest hardfloat junctions rocket torture uncore $(ROCKETCHIP_ADDONS)
@@ -179,6 +185,11 @@ stamps/%/emulator-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/inst
 	$(MAKE) -C $(abspath $(TOP))/emulator CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast
 	date > $@
 
+stamps/%/emulator-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp
+	mkdir -p $(dir $@)
+	$(MAKE) -C $(abspath $(TOP))/emulator CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast
+	date > $@
+
 stamps/%/vsim-asm-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp
 	mkdir -p $(dir $@)
 	$(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-asm-tests-fast
@@ -189,6 +200,11 @@ stamps/%/vsim-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.
 	$(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast
 	date > $@
 
+stamps/%/vsim-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp
+	mkdir -p $(dir $@)
+	$(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast
+	date > $@
+
 stamps/%/fsim-asm-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp
 	mkdir -p $(dir $@)
 	$(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-asm-tests-fast
@@ -199,6 +215,11 @@ stamps/%/fsim-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.
 	$(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast
 	date > $@
 
+stamps/%/fsim-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp
+	mkdir -p $(dir $@)
+	$(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast
+	date > $@
+
 # The torture tests run subtly differently on the different targets, so they
 # don't have pattern rules like everything else does.
 stamps/%/vsim-torture-$(TORTURE_CONFIG).stamp: stamps/%/vsim-debug.stamp stamps/%/vsim-ndebug.stamp

src/main/scala/Configs.scala

@@ -13,7 +13,7 @@ import rocket._
 import rocket.Util._
 import groundtest._
 import scala.math.max
-import scala.collection.mutable.ListBuffer
+import scala.collection.mutable.{LinkedHashSet, ListBuffer}
 import DefaultTestSuites._
 import cde.{Parameters, Config, Dump, Knob, CDEMatchError}
 
@@ -321,6 +321,32 @@ class BaseConfig extends Config (
       case GlobalAddrMap => globalAddrMap
       case EnableL2Logging => false
       case ExportGroundTestStatus => false
+      case RegressionTestNames => LinkedHashSet(
+        "rv64ud-v-fcvt",
+        "rv64ud-p-fdiv",
+        "rv64ud-v-fadd",
+        "rv64uf-v-fadd",
+        "rv64um-v-mul",
+        "rv64mi-p-breakpoint",
+        "rv64uc-v-rvc",
+        "rv64ud-v-structural",
+        "rv64si-p-wfi",
+        "rv64um-v-divw",
+        "rv64ua-v-lrsc",
+        "rv64ui-v-fence_i",
+        "rv64ud-v-fcvt_w",
+        "rv64uf-v-fmin",
+        "rv64ui-v-sb",
+        "rv64ua-v-amomax_d",
+        "rv64ud-v-move",
+        "rv64ud-v-fclass",
+        "rv64ua-v-amoand_d",
+        "rv64ua-v-amoxor_d",
+        "rv64si-p-sbreak",
+        "rv64ud-v-fmadd",
+        "rv64uf-v-ldst",
+        "rv64um-v-mulh",
+        "rv64si-p-dirty")
       case _ => throw new CDEMatchError
   }},
   knobValues = {
@@ -447,6 +473,13 @@ class WithRV32 extends Config(
     case UseUser => false
     case UseAtomics => false
     case UseFPU => false
+    case RegressionTestNames => LinkedHashSet(
+      "rv32mi-p-ma_addr",
+      "rv32mi-p-csr",
+      "rv32ui-p-sh",
+      "rv32ui-p-lh",
+      "rv32mi-p-sbreak",
+      "rv32ui-p-sll")
     case _ => throw new CDEMatchError
   }
 )

src/main/scala/TestConfigs.scala

@@ -57,6 +57,7 @@ class WithGroundTest extends Config(
     case UseFPU => false
     case UseAtomics => false
     case UseCompressed => false
+    case RegressionTestNames => LinkedHashSet("rv64ui-p-simple")
     case _ => throw new CDEMatchError
   })
 
@@ -190,6 +191,7 @@ class WithUnitTest extends Config(
     case UseFPU => false
     case UseAtomics => false
     case UseCompressed => false
+    case RegressionTestNames => LinkedHashSet("rv64ui-p-simple")
     case _ => throw new CDEMatchError
   })
 

src/main/scala/Testing.scala

@@ -4,7 +4,9 @@ package rocketchip
 
 import Chisel._
 import scala.collection.mutable.{LinkedHashSet,LinkedHashMap}
-import cde.{Parameters, ParameterDump, Config}
+import cde.{Parameters, ParameterDump, Config, Field}
+
+case object RegressionTestNames extends Field[LinkedHashSet[String]]
 
 abstract class RocketTestSuite {
   val dir: String
@@ -41,15 +43,24 @@ class BenchmarkTestSuite(makePrefix: String, val dir: String, val names: LinkedH
   override def toString = s"$makeTargetName = \\\n" + names.map(n => s"\t$n.riscv").mkString(" \\\n") + postScript
 }
 
+class RegressionTestSuite(val names: LinkedHashSet[String]) extends RocketTestSuite {
+  val envName = ""
+  val dir = "$(RISCV)/riscv64-unknown-elf/share/riscv-tests/isa"
+  val makeTargetName = "regression-tests"
+  override def toString = s"$makeTargetName = \\\n" + names.mkString(" \\\n")
+}
+
 object TestGeneration {
   import scala.collection.mutable.HashMap
   val asmSuites = new LinkedHashMap[String,AssemblyTestSuite]()
-  val bmarkSuites = new  HashMap[String,BenchmarkTestSuite]()
+  val bmarkSuites = new LinkedHashMap[String,BenchmarkTestSuite]()
+  val regressionSuites = new LinkedHashMap[String,RegressionTestSuite]()
 
   def addSuite(s: RocketTestSuite) {
     s match {
       case a: AssemblyTestSuite => asmSuites += (a.makeTargetName -> a)
       case b: BenchmarkTestSuite => bmarkSuites += (b.makeTargetName -> b)
+      case r: RegressionTestSuite => regressionSuites += (r.makeTargetName -> r)
     }
   }
   
@@ -86,7 +97,8 @@ run-$kind-tests-fast: $$(addprefix $$(output_dir)/, $$(addsuffix .run, $targets)
     f.write(
       List(
         gen("asm", asmSuites.values.toSeq),
-        gen("bmark", bmarkSuites.values.toSeq)
+        gen("bmark", bmarkSuites.values.toSeq),
+        gen("regression", regressionSuites.values.toSeq)
       ).mkString("\n"))
     f.close
   }
@@ -205,6 +217,8 @@ object TestGenerator extends App {
   chiselMain.run(args.drop(3), gen)
   //Driver.elaborate(gen, configName = configClassName)
 
+  TestGeneration.addSuite(new RegressionTestSuite(paramsFromConfig(RegressionTestNames)))
+
   TestGeneration.generateMakefrag(topModuleName, configClassName)
   TestBenchGeneration.generateVerilogFragment(
     topModuleName, configClassName, paramsFromConfig)