diff --git a/groundtest/README.md b/groundtest/README.md index 300417bd..b5ca6042 100644 --- a/groundtest/README.md +++ b/groundtest/README.md @@ -51,8 +51,9 @@ Putting the generator and the checker together, we can automatically search for The [tracegen+check.sh](https://github.com/ucb-bar/rocket-chip/blob/master/scripts/tracegen%2Bcheck.sh) script (included in rocket-chip/scripts) provides an automated way to run a number of randomized tests. The number of tests, initial seed, and other parameters can be set via environment variables or the command line, see the script for more details. -The examples that follow assume that the rocket-chip/scripts directory is -in your PATH and that rocket-chip is your current working directory. +The examples that follow assume that the `groundtest/scripts` +directory is in your `PATH` and that `rocket-chip/emulator` is your +current working directory. ``` > make CONFIG=TraceGenConfig @@ -72,7 +73,6 @@ Load-external rate: 47% To run a single test with a specified random seed: ``` -> cd emulator > make CONFIG=TraceGenConfig > tracegen.py ./emulator-Top-TraceGenConfig 1 > trace.log > toaxe.py trace.log > trace.axe diff --git a/groundtest/scripts/toaxe.py b/groundtest/scripts/toaxe.py new file mode 100755 index 00000000..bd0daee2 --- /dev/null +++ b/groundtest/scripts/toaxe.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# ------- +# Outline +# ------- + +# This script takes memory-subsystem traces produced by the groundtest +# trace generator (see tracegen.scala) and puts them into a format +# that can be validated by the axe tool (see +# https://github.com/CTSRD-CHERI/axe). + +import sys +import re +import sets + +def main(): + + if len(sys.argv) < 2: + sys.stderr.write("Usage: toaxe.py TRACE-FILE [STATS-OUT-FILE]\n") + sys.exit(-1) + + if sys.argv[1] == "-": + f = sys.stdin + else: + f = open(sys.argv[1], 'r') + if f == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + statsFile = None + if len(sys.argv) > 2: + statsFile = open(sys.argv[2], 'a') + + lineCount = 0 + def error(msg): + sys.stderr.write("Error at line " + str(lineCount) + ": " + msg + "\n") + sys.exit(-1) + + # Mapping from address to axe address + addrMap = {} + nextAddr = 0 + + # Mapping from (thread id, tag id) to axe operation id + tagMap = {} + + # Mapping from thread id to operation id + fenceReq = {} + + # Mapping from thread id to operation id + loadReserve = {} + + # Array of axe operations + ops = [] + + # Statistics + scCount = 0 # Number of store-conditionals + scSuccessCount = 0 # Number of store-conditionals that succeeded + loadCount = 0 # Number of loads + loadExtCount = 0 # Number of loads of value written by another core + + # The previous write to each address by each thread + prevWrite = {} + + for line in f: + # Parse thread id and command + m = re.search(' *([0-9]+) *: *([^ ]*) (.*)', line) + if m == None: error("Expected: ':' ") + tid, cmd, line = m.group(1), m.group(2), m.group(3) + + if cmd == 'fence-req': + # Parse time + m = re.search(' *@ *([0-9]+)', line) + if m == None: error ("expected timestamp") + # Insert placeholder containing request time + ops.append(str(m.group(1))) + fenceReq[tid] = len(ops)-1 + elif cmd == 'fence-resp': + # Insert 'sync' operation + if not (tid in fenceReq) or fenceReq[tid] == None: + error("fence-resp without fence-req on thread " + tid) + startTime = ops[fenceReq[tid]] + op = str(tid) + ": sync @ " + startTime + # Add end-time + m = re.search(' *@ *([0-9]+)', line) + if m != None: op = op + ":" + str(m.group(1)) + ops[fenceReq[tid]] = (op,) + fenceReq[tid] = None + elif cmd == 'load-req' or cmd == 'load-reserve-req': + # Parse address, tag, and time + m = re.search(' *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected
# @") + # Update address map + if not (m.group(1) in addrMap): + addrMap[m.group(1)] = nextAddr + nextAddr = nextAddr+1 + # Insert place-holder + ops.append((cmd, None, addrMap[m.group(1)], m.group(3), None)) + tagMap[(tid, m.group(2))] = len(ops)-1 + if cmd == 'load-reserve-req': + loadReserve[tid] = len(ops)-1 + elif cmd == 'store-req' or cmd == 'store-cond-req' or cmd == 'swap-req': + # Parse value, address, tag, and time + m = re.search(' *([0-9]+) *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected
# @") + # Update address map + if not (m.group(2) in addrMap): + addrMap[m.group(2)] = nextAddr + nextAddr = nextAddr+1 + # Insert place-holder + lr = loadReserve[tid] if tid in loadReserve else None + ops.append((cmd, m.group(1), addrMap[m.group(2)], m.group(4), lr)) + tagMap[(tid, m.group(3))] = len(ops)-1 + if cmd == 'store-cond-req': loadReserve[tid] = None + prevWrite[(tid, addrMap[m.group(2)])] = m.group(1) + elif cmd == 'resp': + # Parse value and timestamp + m = re.search(' *([0-9]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected # @") + # Find corresponding response + tag = m.group(2) + if not ((tid, tag) in tagMap) or tagMap[(tid, tag)] == None: + error("resp without associated req with tag " + tag + + " on thread " + tid) + opId = tagMap[(tid, tag)] + (c, val, addr, start, lr) = ops[opId] + if c == 'load-req' or c == 'load-reserve-req': + loadCount = loadCount+1 + if prevWrite.get((tid, addr), None) != m.group(1): + loadExtCount = loadExtCount+1 + if c == 'load-req': + op = tid + ": M[" + str(addr) + '] == ' + m.group(1) + ' @ ' + op += start + ':' + m.group(3) + ops[opId] = (op,) + elif c == 'store-req': + op = tid + ": M[" + str(addr) + '] := ' + val + ' @ ' + op += start + ':' # + m.group(3) + ops[opId] = (op,) + elif c == 'load-reserve-req': + ops[opId] = (m.group(1), start, m.group(3)) + elif c == 'store-cond-req': + if lr == None: error("store conditional without load-reserve") + (loadVal, loadStart, loadFin) = ops[lr] + scCount = scCount + 1 + if int(m.group(1)) != 0: + # SC fail + op = tid + ": M[" + str(addr) + "] == " + loadVal + op += " @ " + loadStart + ":" + loadFin + else: + # SC success + scSuccessCount = scSuccessCount + 1 + op = tid + ": { M[" + str(addr) + "] == " + loadVal + "; " + op += "M[" + str(addr) + "] := " + val + "} @ " + op += loadStart + ":" + loadFin + ops[lr] = (op,) + ops[opId] = None + elif c == 'swap-req': + op = tid + ": { M[" + str(addr) + '] == ' + m.group(1) + op += '; M[' + str(addr) + '] := ' + val + op += '} @ ' + start + ':' # + m.group(3) + ops[opId] = (op,) + else: + error("Unknown command '" + cmd + "'") + + lineCount = lineCount+1 + + # Print statistics + if (scCount > 0): + scSuccessRate = str(scSuccessCount/float(scCount))[0:6] + print("# LRSC_Success_Rate=" + scSuccessRate) + if statsFile != None: + statsFile.write("LRSC_Success_Rate=" + scSuccessRate + "\n") + + if (loadCount > 0): + loadExtRate = str(loadExtCount/float(loadCount))[0:6] + print("# Load_External_Rate=" + loadExtRate) + if statsFile != None: + statsFile.write("Load_External_Rate=" + loadExtRate + "\n") + + if statsFile != None: + statsFile.close() + + # Print address map in comments + for addr in addrMap: + print ("# &M[" + str(addrMap[addr]) + "] == " + addr) + + # Print axe trace + for op in ops: + if op != None and isinstance(op, tuple) and len(op) == 1: + print op[0] + +try: + main() +except KeyboardInterrupt: + sys.exit(-1) diff --git a/groundtest/scripts/tracegen+check.sh b/groundtest/scripts/tracegen+check.sh new file mode 100755 index 00000000..5b012845 --- /dev/null +++ b/groundtest/scripts/tracegen+check.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# This script can be used to test the memory consistency of rocket +# chip in simulation when compiled with the 'TraceGenConfig' +# configuation. + +############################################################################### + +# Parameters +# ========== + +# Arguments are taken from environment variables where available. +# Elsewhere, defaults values are chosen. + +START_SEED=${START_SEED-0} +NUM_TESTS=${NUM_TESTS-100} +EMU=${EMU-emulator-Top-TraceGenConfig} +TRACE_GEN=${TRACE_GEN-tracegen.py} +TO_AXE=${TO_AXE-toaxe.py} +AXE=${AXE-axe} +MODEL=${MODEL-WMO} +LOG_DIR=${LOG_DIR-tracegen-log} +TRACE_STATS=${TRACE_STATS-tracestats.py} + +############################################################################### + +# Inferred parameters +# =================== + +END_SEED=`expr \( $START_SEED + $NUM_TESTS \) - 1` +LOG=$LOG_DIR +PATH=$PATH:. + +# Sanity check +# ============ + +if [ ! `command -v $EMU` ]; then + echo Can\'t find emulator: \'$EMU\' + exit -1 +fi + +if [ ! `command -v $TO_AXE` ]; then + echo Please add \'toaxe.py\' to your PATH + exit -1 +fi + +if [ ! `command -v $TRACE_GEN` ]; then + echo Please add \'tracegen.py\' to your PATH + exit -1 +fi + +if [ ! `command -v $AXE` ]; then + echo Please add \'axe\' to your PATH + exit -1 +fi + +if [ ! `command -v $TRACE_STATS` ]; then + echo Please add \'tracestats.py\' to your PATH + exit -1 +fi + +if [ "$MODEL" != SC -a \ + "$MODEL" != TSO -a \ + "$MODEL" != PSO -a \ + "$MODEL" != WMO -a \ + "$MODEL" != POW ]; then + echo Unknown consistency model \'$MODEL\' + exit -1 +fi + +# Setup log directory +# =================== + +if [ ! -d $LOG ]; then + echo Creating log directory: $LOG + mkdir $LOG +fi + +rm -f $LOG/errors.txt +rm -f $LOG/stats.txt + +# Test loop +# ========= + +echo Testing against $MODEL model: + +for (( I = $START_SEED; I <= $END_SEED; I++ )); do + SPACE=`expr $I \% 10` + if [ $SPACE -eq 0 ]; then + echo -n " " + fi + + NEWLINE=`expr $I \% 50` + if [ $NEWLINE -eq 0 ]; then + printf "\n%8i: " $I + fi + + # Generate trace + $TRACE_GEN $EMU $I > $LOG/trace.txt + if [ ! $? -eq 0 ]; then + echo -e "\n\nError: emulator returned non-zero exit code" + echo See $LOG/trace.txt for details + exit -1 + fi + + # Convert to axe format + $TO_AXE $LOG/trace.txt $LOG/stats.txt 2>> $LOG/errors.txt > $LOG/trace.axe + if [ ! $? -eq 0 ]; then + echo -e "\n\nError during trace generation with seed $I" + echo "See $LOG/errors.txt" + exit -1 + else + # Check trace + OUTCOME=`$AXE check $MODEL $LOG/trace.axe 2>> $LOG/errors.txt` + if [ "$OUTCOME" == "OK" ]; then + echo -n . + else + if [ "$OUTCOME" == "NO" ]; then + echo -e "\n\nFailed $MODEL with seed $I" + echo "See $LOG/trace.txt and $LOG/trace.axe for counterexample" + exit -1 + else + echo -e "\n\nError during trace generation with seed $I" + echo "See $LOG/errors.txt for details" + exit -1 + fi + fi + fi +done + +echo -e "\n\nOK, passed $NUM_TESTS tests" +$TRACE_STATS $LOG/stats.txt diff --git a/groundtest/scripts/tracegen.py b/groundtest/scripts/tracegen.py new file mode 100755 index 00000000..45dfb672 --- /dev/null +++ b/groundtest/scripts/tracegen.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# ------- +# Outline +# ------- + +# Usage: +# +# tracegen.py EMULATOR SEED +# +# This script generates a trace using the given emulator (built +# with CONFIG=TraceGenConfig). It waits until all cores have +# completed trace generation before terminating the emulator. + +import sys +import subprocess +import re + +def main(): + if len(sys.argv) != 3: + sys.stderr.write("Usage: tracegen.py EMULATOR SEED\n") + sys.exit(-1) + + p = subprocess.Popen([sys.argv[1], "+verbose", "-s" + sys.argv[2]], + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if p == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + numFinished = 0 + while True: + line = p.stderr.readline() + if line[0:9] == "FINISHED ": + total = int(line[9:-1]) + numFinished = numFinished + 1 + if numFinished == total: + break + else: + print line, + + p.terminate() + +try: + main() +except KeyboardInterrupt: + sys.exit(-1) diff --git a/groundtest/scripts/tracestats.py b/groundtest/scripts/tracestats.py new file mode 100755 index 00000000..ddf683e0 --- /dev/null +++ b/groundtest/scripts/tracestats.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# ------- +# Outline +# ------- + +# Usage: +# +# tracegen-stats.py STATS-FILE +# +# This script produces some statistics about the traces generated +# using tracegen.py. + +import sys +import subprocess +import re + +def main(): + if len(sys.argv) != 2: + sys.stderr.write("Usage: tracegen-stats.py STATS-FILE\n") + sys.exit(-1) + + f = open(sys.argv[1], 'r') + if f == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + lrscSuccessSum = 0.0 + lrscSuccessCount = 0 + loadExtRateSum = 0.0 + loadExtRateCount = 0 + for line in f: + if line[0:18] == "LRSC_Success_Rate=": + val = float(line[18:-1]) + lrscSuccessSum = lrscSuccessSum + val + lrscSuccessCount = lrscSuccessCount + 1 + + if line[0:19] == "Load_External_Rate=": + val = float(line[19:-1]) + loadExtRateSum = loadExtRateSum + val + loadExtRateCount = loadExtRateCount + 1 + + if lrscSuccessCount > 0: + lrscSuccessAvg = lrscSuccessSum / float(lrscSuccessCount) + lrscSuccessRate = str(int(100.0*lrscSuccessAvg)) + "%" + print "LR/SC success rate:", lrscSuccessRate + else: + print "LR/SC success rate: none performed" + + if loadExtRateCount > 0: + loadExtRateAvg = loadExtRateSum / float(loadExtRateCount) + loadExtRate = str(int(100.0*loadExtRateAvg)) + "%" + print "Load-external rate:", loadExtRate + else: + print "Load-external rate: none performed" + +try: + main() +except KeyboardInterrupt: + sys.exit(-1)