From 6da45e7f26e459ca7fce104c606d1de3684c99eb Mon Sep 17 00:00:00 2001 From: Matthew Naylor Date: Fri, 18 Mar 2016 12:24:12 +0000 Subject: [PATCH] Trace generator: updates and additions to the scripts directory. (1) Introduce tracegen.py, a script that invokes the emulator (built with TraceGenConfig), sending a SIGTERM once all cores are finished. (2) Update toaxe.py to gather some statistics about the trace. (3) Introduce tracestats.py, which displays the stats in a useful way. (4) Introduce tracegen+check.py, a top-level script that generates traces, checks them, and emits stats. If this commit is pulled, it should be done after pulling my latest groundtest commit. --- groundtest | 2 +- scripts/toaxe.py | 299 +++++++++++++++++++++----------------- scripts/tracegen+check.sh | 148 +++++++++++++++++++ scripts/tracegen.py | 62 ++++++++ scripts/tracestats.py | 76 ++++++++++ 5 files changed, 454 insertions(+), 133 deletions(-) create mode 100755 scripts/tracegen+check.sh create mode 100755 scripts/tracegen.py create mode 100755 scripts/tracestats.py diff --git a/groundtest b/groundtest index 82a6f060..e711f798 160000 --- a/groundtest +++ b/groundtest @@ -1 +1 @@ -Subproject commit 82a6f060db93d71ff6d2d44bde9ff8e4e1bf4bef +Subproject commit e711f7988fe5be7881406eb310501cc0a7efb098 diff --git a/scripts/toaxe.py b/scripts/toaxe.py index fa75f059..10b15dc6 100755 --- a/scripts/toaxe.py +++ b/scripts/toaxe.py @@ -27,142 +27,177 @@ import sys import re +import sets -if len(sys.argv) != 2: - print "Usage: toaxe.py [FILE]" - sys.exit() +def main(): -if sys.argv[1] == "-": - f = sys.stdin -else: - f = open(sys.argv[1], 'r') - if f == None: - print "File not found: ", sys.argv[1] - sys.exit() + if len(sys.argv) < 2: + sys.stderr.write("Usage: toaxe.py TRACE-FILE [STATS-OUT-FILE]\n") + sys.exit(-1) -lineCount = 0 -def error(msg): - print "Error at line ", lineCount, ": ", msg - sys.exit() - -# Mapping from address to axe address -addrMap = {} -nextAddr = 0 - -# Mapping from (thread id, tag id) to axe operation id -tagMap = {} - -# Mapping from thread id to operation id -fenceReq = {} - -# Mapping from thread id to operation id -loadReserve = {} - -# Array of axe operations -ops = [] - -for line in f: - # Exit loop at end of trace - if line[0:9] == 'Completed': break - - # Parse thread id and command - m = re.search(' *([0-9]+) *: *([^ ]*) (.*)', line) - if m == None: error("Expected: ':' ") - tid, cmd, line = m.group(1), m.group(2), m.group(3) - - if cmd == 'fence-req': - # Parse time - m = re.search(' *@ *([0-9]+)', line) - if m == None: error ("expected timestamp") - # Insert placeholder containing request time - ops.append(str(m.group(1))) - fenceReq[tid] = len(ops)-1 - elif cmd == 'fence-resp': - # Insert 'sync' operation - if not (tid in fenceReq) or fenceReq[tid] == None: - error("fence-resp without fence-req on thread " + tid) - startTime = ops[fenceReq[tid]] - op = str(tid) + ": sync @ " + startTime - # Add end-time - m = re.search(' *@ *([0-9]+)', line) - if m != None: op = op + ":" + str(m.group(1)) - ops[fenceReq[tid]] = (op,) - fenceReq[tid] = None - elif cmd == 'load-req' or cmd == 'load-reserve-req': - # Parse address, tag, and time - m = re.search(' *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) - if m == None: error("expected
# @") - # Update address map - if not (m.group(1) in addrMap): - addrMap[m.group(1)] = nextAddr - nextAddr = nextAddr+1 - # Insert place-holder - ops.append((cmd, None, addrMap[m.group(1)], m.group(3), None)) - tagMap[(tid, m.group(2))] = len(ops)-1 - if cmd == 'load-reserve-req': - loadReserve[tid] = len(ops)-1 - elif cmd == 'store-req' or cmd == 'store-cond-req' or cmd == 'swap-req': - # Parse value, address, tag, and time - m = re.search(' *([0-9]+) *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) - if m == None: error("expected
# @") - # Update address map - if not (m.group(2) in addrMap): - addrMap[m.group(2)] = nextAddr - nextAddr = nextAddr+1 - # Insert place-holder - lr = loadReserve[tid] if tid in loadReserve else None - ops.append((cmd, m.group(1), addrMap[m.group(2)], m.group(4), lr)) - tagMap[(tid, m.group(3))] = len(ops)-1 - if cmd == 'store-cond-req': loadReserve[tid] = None - elif cmd == 'resp': - # Parse value and timestamp - m = re.search(' *([0-9]+) *# *([0-9]+) *@ *([0-9]+)', line) - if m == None: error("expected # @") - # Find corresponding response - tag = m.group(2) - if not ((tid, tag) in tagMap) or tagMap[(tid, tag)] == None: - error("resp without associated req with tag " + tag + " on thread " + tid) - opId = tagMap[(tid, tag)] - (c, val, addr, start, lr) = ops[opId] - if c == 'load-req': - op = tid + ": M[" + str(addr) + '] == ' + m.group(1) + ' @ ' - op += start + ':' + m.group(3) - ops[opId] = (op,) - elif c == 'store-req': - op = tid + ": M[" + str(addr) + '] := ' + val + ' @ ' - op += start + ':' # + m.group(3) - ops[opId] = (op,) - elif c == 'load-reserve-req': - ops[opId] = (m.group(1), start, m.group(3)) - elif c == 'store-cond-req': - if lr == None: error("store conditional without load-reserve") - (loadVal, loadStart, loadFin) = ops[lr] - if int(m.group(1)) != 0: - # SC fail - op = tid + ": M[" + str(addr) + "] == " + loadVal - op += " @ " + loadStart + ":" + loadFin - else: - # SC success - op = tid + ": { M[" + str(addr) + "] == " + loadVal + "; " - op += "M[" + str(addr) + "] := " + val + "} @ " - op += loadStart + ":" # + m.group(3) - ops[lr] = (op,) - ops[opId] = None - elif c == 'swap-req': - op = tid + ": { M[" + str(addr) + '] == ' + m.group(1) - op += '; M[' + str(addr) + '] := ' + val - op += '} @ ' + start + ':' # + m.group(3) - ops[opId] = (op,) + if sys.argv[1] == "-": + f = sys.stdin else: - error("Unknown command '" + cmd + "'") + f = open(sys.argv[1], 'r') + if f == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + statsFile = None + if len(sys.argv) > 2: + statsFile = open(sys.argv[2], 'a') + + lineCount = 0 + def error(msg): + sys.stderr.write("Error at line " + str(lineCount) + ": " + msg + "\n") + sys.exit(-1) + + # Mapping from address to axe address + addrMap = {} + nextAddr = 0 + + # Mapping from (thread id, tag id) to axe operation id + tagMap = {} + + # Mapping from thread id to operation id + fenceReq = {} + + # Mapping from thread id to operation id + loadReserve = {} + + # Array of axe operations + ops = [] + + # Statistics + scCount = 0 # Number of store-conditionals + scSuccessCount = 0 # Number of store-conditionals that succeeded + loadCount = 0 # Number of loads + loadExtCount = 0 # Number of loads of value written by another core + + # The previous write to each address by each thread + prevWrite = {} + + for line in f: + # Parse thread id and command + m = re.search(' *([0-9]+) *: *([^ ]*) (.*)', line) + if m == None: error("Expected: ':' ") + tid, cmd, line = m.group(1), m.group(2), m.group(3) + + if cmd == 'fence-req': + # Parse time + m = re.search(' *@ *([0-9]+)', line) + if m == None: error ("expected timestamp") + # Insert placeholder containing request time + ops.append(str(m.group(1))) + fenceReq[tid] = len(ops)-1 + elif cmd == 'fence-resp': + # Insert 'sync' operation + if not (tid in fenceReq) or fenceReq[tid] == None: + error("fence-resp without fence-req on thread " + tid) + startTime = ops[fenceReq[tid]] + op = str(tid) + ": sync @ " + startTime + # Add end-time + m = re.search(' *@ *([0-9]+)', line) + if m != None: op = op + ":" + str(m.group(1)) + ops[fenceReq[tid]] = (op,) + fenceReq[tid] = None + elif cmd == 'load-req' or cmd == 'load-reserve-req': + # Parse address, tag, and time + m = re.search(' *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected
# @") + # Update address map + if not (m.group(1) in addrMap): + addrMap[m.group(1)] = nextAddr + nextAddr = nextAddr+1 + # Insert place-holder + ops.append((cmd, None, addrMap[m.group(1)], m.group(3), None)) + tagMap[(tid, m.group(2))] = len(ops)-1 + if cmd == 'load-reserve-req': + loadReserve[tid] = len(ops)-1 + elif cmd == 'store-req' or cmd == 'store-cond-req' or cmd == 'swap-req': + # Parse value, address, tag, and time + m = re.search(' *([0-9]+) *([0-9a-fx]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected
# @") + # Update address map + if not (m.group(2) in addrMap): + addrMap[m.group(2)] = nextAddr + nextAddr = nextAddr+1 + # Insert place-holder + lr = loadReserve[tid] if tid in loadReserve else None + ops.append((cmd, m.group(1), addrMap[m.group(2)], m.group(4), lr)) + tagMap[(tid, m.group(3))] = len(ops)-1 + if cmd == 'store-cond-req': loadReserve[tid] = None + prevWrite[(tid, addrMap[m.group(2)])] = m.group(1) + elif cmd == 'resp': + # Parse value and timestamp + m = re.search(' *([0-9]+) *# *([0-9]+) *@ *([0-9]+)', line) + if m == None: error("expected # @") + # Find corresponding response + tag = m.group(2) + if not ((tid, tag) in tagMap) or tagMap[(tid, tag)] == None: + error("resp without associated req with tag " + tag + + " on thread " + tid) + opId = tagMap[(tid, tag)] + (c, val, addr, start, lr) = ops[opId] + if c == 'load-req' or c == 'load-reserve-req': + loadCount = loadCount+1 + if prevWrite.get((tid, addr), None) != m.group(1): + loadExtCount = loadExtCount+1 + if c == 'load-req': + op = tid + ": M[" + str(addr) + '] == ' + m.group(1) + ' @ ' + op += start + ':' + m.group(3) + ops[opId] = (op,) + elif c == 'store-req': + op = tid + ": M[" + str(addr) + '] := ' + val + ' @ ' + op += start + ':' # + m.group(3) + ops[opId] = (op,) + elif c == 'load-reserve-req': + ops[opId] = (m.group(1), start, m.group(3)) + elif c == 'store-cond-req': + if lr == None: error("store conditional without load-reserve") + (loadVal, loadStart, loadFin) = ops[lr] + scCount = scCount + 1 + if int(m.group(1)) != 0: + # SC fail + op = tid + ": M[" + str(addr) + "] == " + loadVal + op += " @ " + loadStart + ":" + loadFin + else: + # SC success + scSuccessCount = scSuccessCount + 1 + op = tid + ": { M[" + str(addr) + "] == " + loadVal + "; " + op += "M[" + str(addr) + "] := " + val + "} @ " + op += loadStart + ":" + loadFin + ops[lr] = (op,) + ops[opId] = None + elif c == 'swap-req': + op = tid + ": { M[" + str(addr) + '] == ' + m.group(1) + op += '; M[' + str(addr) + '] := ' + val + op += '} @ ' + start + ':' # + m.group(3) + ops[opId] = (op,) + else: + error("Unknown command '" + cmd + "'") - lineCount = lineCount+1 + lineCount = lineCount+1 -# Print address map in comments -for addr in addrMap: - print ("# &M[" + str(addrMap[addr]) + "] == " + addr) + # Print statistics + scSuccessRate = str(scSuccessCount/float(scCount))[0:6] + loadExtRate = str(loadExtCount/float(loadCount))[0:6] + print("# LRSC_Success_Rate=" + scSuccessRate) + if statsFile != None: + statsFile.write("LRSC_Success_Rate=" + scSuccessRate + "\n") + statsFile.write("Load_External_Rate=" + loadExtRate + "\n") + statsFile.close() -# Print axe trace -for op in ops: - if op != None and isinstance(op, tuple) and len(op) == 1: - print op[0] + # Print address map in comments + for addr in addrMap: + print ("# &M[" + str(addrMap[addr]) + "] == " + addr) + + # Print axe trace + for op in ops: + if op != None and isinstance(op, tuple) and len(op) == 1: + print op[0] + +try: + main() +except KeyboardInterrupt: + sys.exit(-1) diff --git a/scripts/tracegen+check.sh b/scripts/tracegen+check.sh new file mode 100755 index 00000000..5b012845 --- /dev/null +++ b/scripts/tracegen+check.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# This script can be used to test the memory consistency of rocket +# chip in simulation when compiled with the 'TraceGenConfig' +# configuation. + +############################################################################### + +# Parameters +# ========== + +# Arguments are taken from environment variables where available. +# Elsewhere, defaults values are chosen. + +START_SEED=${START_SEED-0} +NUM_TESTS=${NUM_TESTS-100} +EMU=${EMU-emulator-Top-TraceGenConfig} +TRACE_GEN=${TRACE_GEN-tracegen.py} +TO_AXE=${TO_AXE-toaxe.py} +AXE=${AXE-axe} +MODEL=${MODEL-WMO} +LOG_DIR=${LOG_DIR-tracegen-log} +TRACE_STATS=${TRACE_STATS-tracestats.py} + +############################################################################### + +# Inferred parameters +# =================== + +END_SEED=`expr \( $START_SEED + $NUM_TESTS \) - 1` +LOG=$LOG_DIR +PATH=$PATH:. + +# Sanity check +# ============ + +if [ ! `command -v $EMU` ]; then + echo Can\'t find emulator: \'$EMU\' + exit -1 +fi + +if [ ! `command -v $TO_AXE` ]; then + echo Please add \'toaxe.py\' to your PATH + exit -1 +fi + +if [ ! `command -v $TRACE_GEN` ]; then + echo Please add \'tracegen.py\' to your PATH + exit -1 +fi + +if [ ! `command -v $AXE` ]; then + echo Please add \'axe\' to your PATH + exit -1 +fi + +if [ ! `command -v $TRACE_STATS` ]; then + echo Please add \'tracestats.py\' to your PATH + exit -1 +fi + +if [ "$MODEL" != SC -a \ + "$MODEL" != TSO -a \ + "$MODEL" != PSO -a \ + "$MODEL" != WMO -a \ + "$MODEL" != POW ]; then + echo Unknown consistency model \'$MODEL\' + exit -1 +fi + +# Setup log directory +# =================== + +if [ ! -d $LOG ]; then + echo Creating log directory: $LOG + mkdir $LOG +fi + +rm -f $LOG/errors.txt +rm -f $LOG/stats.txt + +# Test loop +# ========= + +echo Testing against $MODEL model: + +for (( I = $START_SEED; I <= $END_SEED; I++ )); do + SPACE=`expr $I \% 10` + if [ $SPACE -eq 0 ]; then + echo -n " " + fi + + NEWLINE=`expr $I \% 50` + if [ $NEWLINE -eq 0 ]; then + printf "\n%8i: " $I + fi + + # Generate trace + $TRACE_GEN $EMU $I > $LOG/trace.txt + if [ ! $? -eq 0 ]; then + echo -e "\n\nError: emulator returned non-zero exit code" + echo See $LOG/trace.txt for details + exit -1 + fi + + # Convert to axe format + $TO_AXE $LOG/trace.txt $LOG/stats.txt 2>> $LOG/errors.txt > $LOG/trace.axe + if [ ! $? -eq 0 ]; then + echo -e "\n\nError during trace generation with seed $I" + echo "See $LOG/errors.txt" + exit -1 + else + # Check trace + OUTCOME=`$AXE check $MODEL $LOG/trace.axe 2>> $LOG/errors.txt` + if [ "$OUTCOME" == "OK" ]; then + echo -n . + else + if [ "$OUTCOME" == "NO" ]; then + echo -e "\n\nFailed $MODEL with seed $I" + echo "See $LOG/trace.txt and $LOG/trace.axe for counterexample" + exit -1 + else + echo -e "\n\nError during trace generation with seed $I" + echo "See $LOG/errors.txt for details" + exit -1 + fi + fi + fi +done + +echo -e "\n\nOK, passed $NUM_TESTS tests" +$TRACE_STATS $LOG/stats.txt diff --git a/scripts/tracegen.py b/scripts/tracegen.py new file mode 100755 index 00000000..45dfb672 --- /dev/null +++ b/scripts/tracegen.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# ------- +# Outline +# ------- + +# Usage: +# +# tracegen.py EMULATOR SEED +# +# This script generates a trace using the given emulator (built +# with CONFIG=TraceGenConfig). It waits until all cores have +# completed trace generation before terminating the emulator. + +import sys +import subprocess +import re + +def main(): + if len(sys.argv) != 3: + sys.stderr.write("Usage: tracegen.py EMULATOR SEED\n") + sys.exit(-1) + + p = subprocess.Popen([sys.argv[1], "+verbose", "-s" + sys.argv[2]], + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + if p == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + numFinished = 0 + while True: + line = p.stderr.readline() + if line[0:9] == "FINISHED ": + total = int(line[9:-1]) + numFinished = numFinished + 1 + if numFinished == total: + break + else: + print line, + + p.terminate() + +try: + main() +except KeyboardInterrupt: + sys.exit(-1) diff --git a/scripts/tracestats.py b/scripts/tracestats.py new file mode 100755 index 00000000..ddf683e0 --- /dev/null +++ b/scripts/tracestats.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +# This file was originally written by Matthew Naylor, University of +# Cambridge. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 +# ("CTSRD"), as part of the DARPA CRASH research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory under DARPA/AFRL contract FA8750-11-C-0249 +# ("MRC2"), as part of the DARPA MRC research programme. +# +# This software was partly developed by the University of Cambridge +# Computer Laboratory as part of the Rigorous Engineering of +# Mainstream Systems (REMS) project, funded by EPSRC grant +# EP/K008528/1. + +# ------- +# Outline +# ------- + +# Usage: +# +# tracegen-stats.py STATS-FILE +# +# This script produces some statistics about the traces generated +# using tracegen.py. + +import sys +import subprocess +import re + +def main(): + if len(sys.argv) != 2: + sys.stderr.write("Usage: tracegen-stats.py STATS-FILE\n") + sys.exit(-1) + + f = open(sys.argv[1], 'r') + if f == None: + sys.stderr.write("File not found: " + sys.argv[1] + "\n") + sys.exit(-1) + + lrscSuccessSum = 0.0 + lrscSuccessCount = 0 + loadExtRateSum = 0.0 + loadExtRateCount = 0 + for line in f: + if line[0:18] == "LRSC_Success_Rate=": + val = float(line[18:-1]) + lrscSuccessSum = lrscSuccessSum + val + lrscSuccessCount = lrscSuccessCount + 1 + + if line[0:19] == "Load_External_Rate=": + val = float(line[19:-1]) + loadExtRateSum = loadExtRateSum + val + loadExtRateCount = loadExtRateCount + 1 + + if lrscSuccessCount > 0: + lrscSuccessAvg = lrscSuccessSum / float(lrscSuccessCount) + lrscSuccessRate = str(int(100.0*lrscSuccessAvg)) + "%" + print "LR/SC success rate:", lrscSuccessRate + else: + print "LR/SC success rate: none performed" + + if loadExtRateCount > 0: + loadExtRateAvg = loadExtRateSum / float(loadExtRateCount) + loadExtRate = str(int(100.0*loadExtRateAvg)) + "%" + print "Load-external rate:", loadExtRate + else: + print "Load-external rate: none performed" + +try: + main() +except KeyboardInterrupt: + sys.exit(-1)