From 63b814fcd701661758274db0f229cef07c585d43 Mon Sep 17 00:00:00 2001 From: Howard Mao Date: Mon, 1 Aug 2016 19:07:03 -0700 Subject: [PATCH] only run the important (high coverage) tests in regression suite --- .travis.yml | 3 +-- regression/Makefile | 23 ++++++++++++++++++++- src/main/scala/Configs.scala | 35 +++++++++++++++++++++++++++++++- src/main/scala/TestConfigs.scala | 2 ++ src/main/scala/Testing.scala | 20 +++++++++++++++--- 5 files changed, 76 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 92b3e394..eb419490 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,8 +49,7 @@ script: - make vsim-verilog -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION - make fsim-verilog -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION - make emulator-ndebug -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION - - make emulator-asm-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION - - make emulator-bmark-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION -j1 + - make emulator-regression-tests -C regression SUITE=$SUITE TORTURE_CONFIG=default CHISEL_VERSION=$CHISEL_VERSION before_cache: - ls -tr regression/install | tail -n+2 | sed s@^@regression/install/@ | xargs rm -rf diff --git a/regression/Makefile b/regression/Makefile index 60d9bbba..f9d7f8e2 100644 --- a/regression/Makefile +++ b/regression/Makefile @@ -44,7 +44,7 @@ $(error Set SUITE to the regression suite you want to run) endif ifeq ($(SUITE),RocketSuite) -CONFIGS=DefaultConfig DefaultL2Config DefaultBufferlessConfig TinyConfig RoccExampleConfig +CONFIGS=DefaultConfig DefaultL2Config DefaultBufferlessConfig TinyConfig endif ifeq ($(SUITE),GroundtestSuite) @@ -65,12 +65,14 @@ EMU_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-debug.sta EMU_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-ndebug.stamp) EMU_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-asm-tests.stamp) EMU_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-bmark-tests.stamp) +EMU_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-regression-tests.stamp) EMU_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/emulator-torture-$(TORTURE_CONFIG).stamp) emulator-debug: $(EMU_DEBUG_STAMPS) emulator-ndebug: $(EMU_NDEBUG_STAMPS) emulator-asm-tests: $(EMU_ASM_TEST_STAMPS) emulator-bmark-tests: $(EMU_BMARK_TEST_STAMPS) +emulator-regression-tests: $(EMU_REGRESSION_TEST_STAMPS) emulator-torture: $(EMU_TORTURE_STAMPS) VSIM_VERILOG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-verilog.stamp) @@ -78,6 +80,7 @@ VSIM_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-debug.stamp) VSIM_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-ndebug.stamp) VSIM_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-asm-tests.stamp) VSIM_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-bmark-tests.stamp) +VSIM_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-regression-tests.stamp) VSIM_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/vsim-torture-$(TORTURE_CONFIG).stamp) vsim-verilog: $(VSIM_VERILOG_STAMPS) @@ -85,6 +88,7 @@ vsim-debug: $(VSIM_DEBUG_STAMPS) vsim-ndebug: $(VSIM_NDEBUG_STAMPS) vsim-asm-tests: $(VSIM_ASM_TEST_STAMPS) vsim-bmark-tests: $(VSIM_BMARK_TEST_STAMPS) +vsim-regression-tests: $(VSIM_REGRESSION_TEST_STAMPS) vsim-torture: $(VSIM_TORTURE_STAMPS) FSIM_VERILOG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-verilog.stamp) @@ -92,6 +96,7 @@ FSIM_DEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-debug.stamp) FSIM_NDEBUG_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-ndebug.stamp) FSIM_ASM_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-asm-tests.stamp) FSIM_BMARK_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-bmark-tests.stamp) +FSIM_REGRESSION_TEST_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-regression-tests.stamp) FSIM_TORTURE_STAMPS=$(foreach config,$(CONFIGS),stamps/$(config)/fsim-torture-$(TORTURE_CONFIG).stamp) fsim-verilog: $(FSIM_VERILOG_STAMPS) @@ -99,6 +104,7 @@ fsim-debug: $(FSIM_DEBUG_STAMPS) fsim-ndebug: $(FSIM_NDEBUG_STAMPS) fsim-asm-tests: $(FSIM_ASM_TEST_STAMPS) fsim-bmark-tests: $(FSIM_BMARK_TEST_STAMPS) +fsim-regression-tests: $(FSIM_REGRESSION_TEST_STAMPS) fsim-torture: $(FSIM_TORTURE_STAMPS) submodule_names = chisel2 chisel3 context-dependent-environments dramsim2 firrtl groundtest hardfloat junctions rocket torture uncore $(ROCKETCHIP_ADDONS) @@ -179,6 +185,11 @@ stamps/%/emulator-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/inst $(MAKE) -C $(abspath $(TOP))/emulator CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast date > $@ +stamps/%/emulator-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp + mkdir -p $(dir $@) + $(MAKE) -C $(abspath $(TOP))/emulator CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast + date > $@ + stamps/%/vsim-asm-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp mkdir -p $(dir $@) $(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-asm-tests-fast @@ -189,6 +200,11 @@ stamps/%/vsim-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install. $(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast date > $@ +stamps/%/vsim-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp + mkdir -p $(dir $@) + $(MAKE) -C $(abspath $(TOP))/vsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast + date > $@ + stamps/%/fsim-asm-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp mkdir -p $(dir $@) $(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-asm-tests-fast @@ -199,6 +215,11 @@ stamps/%/fsim-bmark-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install. $(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-bmark-tests-fast date > $@ +stamps/%/fsim-regression-tests.stamp: stamps/other-submodules.stamp $(RISCV)/install.stamp + mkdir -p $(dir $@) + $(MAKE) -C $(abspath $(TOP))/fsim CONFIG=$* RISCV=$(abspath $(RISCV)) CHISEL_VERSION=$(CHISEL_VERSION) run-regression-tests-fast + date > $@ + # The torture tests run subtly differently on the different targets, so they # don't have pattern rules like everything else does. stamps/%/vsim-torture-$(TORTURE_CONFIG).stamp: stamps/%/vsim-debug.stamp stamps/%/vsim-ndebug.stamp diff --git a/src/main/scala/Configs.scala b/src/main/scala/Configs.scala index 197acde9..1cfded2d 100644 --- a/src/main/scala/Configs.scala +++ b/src/main/scala/Configs.scala @@ -13,7 +13,7 @@ import rocket._ import rocket.Util._ import groundtest._ import scala.math.max -import scala.collection.mutable.ListBuffer +import scala.collection.mutable.{LinkedHashSet, ListBuffer} import DefaultTestSuites._ import cde.{Parameters, Config, Dump, Knob, CDEMatchError} @@ -321,6 +321,32 @@ class BaseConfig extends Config ( case GlobalAddrMap => globalAddrMap case EnableL2Logging => false case ExportGroundTestStatus => false + case RegressionTestNames => LinkedHashSet( + "rv64ud-v-fcvt", + "rv64ud-p-fdiv", + "rv64ud-v-fadd", + "rv64uf-v-fadd", + "rv64um-v-mul", + "rv64mi-p-breakpoint", + "rv64uc-v-rvc", + "rv64ud-v-structural", + "rv64si-p-wfi", + "rv64um-v-divw", + "rv64ua-v-lrsc", + "rv64ui-v-fence_i", + "rv64ud-v-fcvt_w", + "rv64uf-v-fmin", + "rv64ui-v-sb", + "rv64ua-v-amomax_d", + "rv64ud-v-move", + "rv64ud-v-fclass", + "rv64ua-v-amoand_d", + "rv64ua-v-amoxor_d", + "rv64si-p-sbreak", + "rv64ud-v-fmadd", + "rv64uf-v-ldst", + "rv64um-v-mulh", + "rv64si-p-dirty") case _ => throw new CDEMatchError }}, knobValues = { @@ -447,6 +473,13 @@ class WithRV32 extends Config( case UseUser => false case UseAtomics => false case UseFPU => false + case RegressionTestNames => LinkedHashSet( + "rv32mi-p-ma_addr", + "rv32mi-p-csr", + "rv32ui-p-sh", + "rv32ui-p-lh", + "rv32mi-p-sbreak", + "rv32ui-p-sll") case _ => throw new CDEMatchError } ) diff --git a/src/main/scala/TestConfigs.scala b/src/main/scala/TestConfigs.scala index 755433bb..c1b606e2 100644 --- a/src/main/scala/TestConfigs.scala +++ b/src/main/scala/TestConfigs.scala @@ -57,6 +57,7 @@ class WithGroundTest extends Config( case UseFPU => false case UseAtomics => false case UseCompressed => false + case RegressionTestNames => LinkedHashSet("rv64ui-p-simple") case _ => throw new CDEMatchError }) @@ -190,6 +191,7 @@ class WithUnitTest extends Config( case UseFPU => false case UseAtomics => false case UseCompressed => false + case RegressionTestNames => LinkedHashSet("rv64ui-p-simple") case _ => throw new CDEMatchError }) diff --git a/src/main/scala/Testing.scala b/src/main/scala/Testing.scala index 613edfe2..2c017780 100644 --- a/src/main/scala/Testing.scala +++ b/src/main/scala/Testing.scala @@ -4,7 +4,9 @@ package rocketchip import Chisel._ import scala.collection.mutable.{LinkedHashSet,LinkedHashMap} -import cde.{Parameters, ParameterDump, Config} +import cde.{Parameters, ParameterDump, Config, Field} + +case object RegressionTestNames extends Field[LinkedHashSet[String]] abstract class RocketTestSuite { val dir: String @@ -41,15 +43,24 @@ class BenchmarkTestSuite(makePrefix: String, val dir: String, val names: LinkedH override def toString = s"$makeTargetName = \\\n" + names.map(n => s"\t$n.riscv").mkString(" \\\n") + postScript } +class RegressionTestSuite(val names: LinkedHashSet[String]) extends RocketTestSuite { + val envName = "" + val dir = "$(RISCV)/riscv64-unknown-elf/share/riscv-tests/isa" + val makeTargetName = "regression-tests" + override def toString = s"$makeTargetName = \\\n" + names.mkString(" \\\n") +} + object TestGeneration { import scala.collection.mutable.HashMap val asmSuites = new LinkedHashMap[String,AssemblyTestSuite]() - val bmarkSuites = new HashMap[String,BenchmarkTestSuite]() + val bmarkSuites = new LinkedHashMap[String,BenchmarkTestSuite]() + val regressionSuites = new LinkedHashMap[String,RegressionTestSuite]() def addSuite(s: RocketTestSuite) { s match { case a: AssemblyTestSuite => asmSuites += (a.makeTargetName -> a) case b: BenchmarkTestSuite => bmarkSuites += (b.makeTargetName -> b) + case r: RegressionTestSuite => regressionSuites += (r.makeTargetName -> r) } } @@ -86,7 +97,8 @@ run-$kind-tests-fast: $$(addprefix $$(output_dir)/, $$(addsuffix .run, $targets) f.write( List( gen("asm", asmSuites.values.toSeq), - gen("bmark", bmarkSuites.values.toSeq) + gen("bmark", bmarkSuites.values.toSeq), + gen("regression", regressionSuites.values.toSeq) ).mkString("\n")) f.close } @@ -205,6 +217,8 @@ object TestGenerator extends App { chiselMain.run(args.drop(3), gen) //Driver.elaborate(gen, configName = configClassName) + TestGeneration.addSuite(new RegressionTestSuite(paramsFromConfig(RegressionTestNames))) + TestGeneration.generateMakefrag(topModuleName, configClassName) TestBenchGeneration.generateVerilogFragment( topModuleName, configClassName, paramsFromConfig)