2016-11-28 01:16:37 +01:00
// See LICENSE.SiFive for license details.
2016-09-21 01:49:57 +02:00
2017-07-07 19:48:16 +02:00
package freechips.rocketchip.tilelink
2016-09-21 01:49:57 +02:00
import Chisel._
2017-07-07 19:48:16 +02:00
import freechips.rocketchip.config.Parameters
import freechips.rocketchip.diplomacy._
import freechips.rocketchip.util._
2016-09-21 01:49:57 +02:00
import scala.math. { min , max }
// Ensures that all downstream RW managers support Atomic operationss.
// If !passthrough, intercept all Atomics. Otherwise, only intercept those unsupported downstream.
2016-12-02 02:46:52 +01:00
class TLAtomicAutomata ( logical : Boolean = true , arithmetic : Boolean = true , concurrency : Int = 1 , passthrough : Boolean = true ) ( implicit p : Parameters ) extends LazyModule
2016-09-21 01:49:57 +02:00
{
require ( concurrency >= 1 )
val node = TLAdapterNode (
2017-01-30 00:17:52 +01:00
managerFn = { case mp => mp . copy ( managers = mp . managers . map { m =>
2016-09-21 01:49:57 +02:00
val ourSupport = TransferSizes ( 1 , mp . beatBytes )
def widen ( x : TransferSizes ) = if ( passthrough && x . min <= 2 * mp . beatBytes ) TransferSizes ( 1 , max ( mp . beatBytes , x . max ) ) else ourSupport
val canDoit = m . supportsPutFull . contains ( ourSupport ) && m . supportsGet . contains ( ourSupport )
// Blow up if there are devices to which we cannot add Atomics, because their R|W are too inflexible
2017-07-23 17:31:04 +02:00
require ( ! m . supportsPutFull || ! m . supportsGet || canDoit , s" ${ m . name } has $ourSupport , needed PutFull( ${ m . supportsPutFull } ) or Get( ${ m . supportsGet } ) " )
2016-09-21 01:49:57 +02:00
m . copy (
supportsArithmetic = if ( ! arithmetic || ! canDoit ) m . supportsArithmetic else widen ( m . supportsArithmetic ) ,
supportsLogical = if ( ! logical || ! canDoit ) m . supportsLogical else widen ( m . supportsLogical ) )
} ) } )
lazy val module = new LazyModuleImp ( this ) {
2017-09-14 03:06:03 +02:00
( node . in zip node . out ) foreach { case ( ( in , edgeIn ) , ( out , edgeOut ) ) =>
2017-01-30 00:17:52 +01:00
val managers = edgeOut . manager . managers
val beatBytes = edgeOut . manager . beatBytes
2017-11-15 02:49:10 +01:00
// This is necessary (though not sufficient) for correctness:
edgeOut . manager . findTreeViolation ( ) match {
case None => ( )
case Some ( node ) => require ( edgeOut . manager . isTree , s" AtomicAutomata can only be placed infront of a tree of diplomatic nodes ( ${ node . name } has parents ${ node . inputs . map ( _ . _1 . name ) } ) " )
}
// You also need to know that the slaves don't have an internal masters that can get between the read and write
2017-01-30 00:17:52 +01:00
// To which managers are we adding atomic support?
val ourSupport = TransferSizes ( 1 , edgeOut . manager . beatBytes )
val managersNeedingHelp = managers . filter { m =>
m . supportsPutFull . contains ( ourSupport ) &&
m . supportsGet . contains ( ourSupport ) &&
( ( logical && ! m . supportsLogical . contains ( ourSupport ) ) ||
( arithmetic && ! m . supportsArithmetic . contains ( ourSupport ) ) ||
! passthrough ) // we will do atomics for everyone we can
2016-09-21 01:49:57 +02:00
}
2017-01-30 00:17:52 +01:00
// We cannot add atomcis to a non-FIFO manager
managersNeedingHelp foreach { m => require ( m . fifoId . isDefined ) }
// We need to preserve FIFO semantics across FIFO domains, not managers
// Suppose you have Put(42) Atomic(+1) both inflight; valid results: 42 or 43
// If we allow Put(42) Get() Put(+1) concurrent; valid results: 42 43 OR undef
// Making non-FIFO work requires waiting for all Acks to come back (=> use FIFOFixer)
val domainsNeedingHelp = managersNeedingHelp . map ( _ . fifoId . get ) . distinct
// Don't overprovision the CAM
val camSize = min ( domainsNeedingHelp . size , concurrency )
// Compact the fifoIds to only those we care about
2017-06-01 23:59:53 +02:00
def camFifoId ( m : TLManagerParameters ) = m . fifoId . map ( id => max ( 0 , domainsNeedingHelp . indexOf ( id ) ) ) . getOrElse ( 0 )
2017-01-30 00:17:52 +01:00
// CAM entry state machine
val FREE = UInt ( 0 ) // unused waiting on Atomic from A
val GET = UInt ( 3 ) // Get sent down A waiting on AccessDataAck from D
val AMO = UInt ( 2 ) // AccessDataAck sent up D waiting for A availability
val ACK = UInt ( 1 ) // Put sent down A waiting for PutAck from D
val params = TLAtomicAutomata . CAMParams ( out . a . bits . params , domainsNeedingHelp . size )
// Do we need to do anything at all?
if ( camSize > 0 ) {
val initval = Wire ( new TLAtomicAutomata . CAM_S ( params ) )
initval . state : = FREE
val cam_s = RegInit ( Vec . fill ( camSize ) ( initval ) )
val cam_a = Reg ( Vec ( camSize , new TLAtomicAutomata . CAM_A ( params ) ) )
val cam_d = Reg ( Vec ( camSize , new TLAtomicAutomata . CAM_D ( params ) ) )
val cam_free = cam_s . map ( _ . state === FREE )
val cam_amo = cam_s . map ( _ . state === AMO )
val cam_abusy = cam_s . map ( e => e . state === GET || e . state === AMO ) // A is blocked
val cam_dmatch = cam_s . map ( e => e . state =/= FREE ) // D should inspect these entries
// Can the manager already handle this message?
2017-06-01 23:59:53 +02:00
val a_address = edgeIn . address ( in . a . bits )
2017-01-30 00:17:52 +01:00
val a_size = edgeIn . size ( in . a . bits )
2017-06-01 23:59:53 +02:00
val a_canLogical = Bool ( passthrough ) && edgeOut . manager . supportsLogicalFast ( a_address , a_size )
val a_canArithmetic = Bool ( passthrough ) && edgeOut . manager . supportsArithmeticFast ( a_address , a_size )
2017-01-30 00:17:52 +01:00
val a_isLogical = in . a . bits . opcode === TLMessages . LogicalData
val a_isArithmetic = in . a . bits . opcode === TLMessages . ArithmeticData
val a_isSupported = Mux ( a_isLogical , a_canLogical , Mux ( a_isArithmetic , a_canArithmetic , Bool ( true ) ) )
// Must we do a Put?
val a_cam_any_put = cam_amo . reduce ( _ || _ )
val a_cam_por_put = cam_amo . scanLeft ( Bool ( false ) ) ( _ || _ ) . init
val a_cam_sel_put = ( cam_amo zip a_cam_por_put ) map { case ( a , b ) => a && ! b }
val a_cam_a = PriorityMux ( cam_amo , cam_a )
val a_cam_d = PriorityMux ( cam_amo , cam_d )
val a_a = a_cam_a . bits . data
val a_d = a_cam_d . data
// Does the A request conflict with an inflight AMO?
2017-06-01 23:59:53 +02:00
val a_fifoId = edgeOut . manager . fastProperty ( a_address , camFifoId _ , ( i : Int ) => UInt ( i ) )
2017-01-30 00:17:52 +01:00
val a_cam_busy = ( cam_abusy zip cam_a . map ( _ . fifoId === a_fifoId ) ) map { case ( a , b ) => a && b } reduce ( _ || _ )
// (Where) are we are allocating in the CAM?
val a_cam_any_free = cam_free . reduce ( _ || _ )
val a_cam_por_free = cam_free . scanLeft ( Bool ( false ) ) ( _ || _ ) . init
val a_cam_sel_free = ( cam_free zip a_cam_por_free ) map { case ( a , b ) => a && ! b }
// Logical AMO
val indexes = Seq . tabulate ( beatBytes * 8 ) { i => Cat ( a_a ( i , i ) , a_d ( i , i ) ) }
val logic_out = Cat ( indexes . map ( x => a_cam_a . lut ( x ) . asUInt ) . reverse )
// Arithmetic AMO
val unsigned = a_cam_a . bits . param ( 1 )
val take_max = a_cam_a . bits . param ( 0 )
val adder = a_cam_a . bits . param ( 2 )
val mask = a_cam_a . bits . mask
val signSel = ~ ( ~ mask | ( mask >> 1 ) )
val signbits_a = Cat ( Seq . tabulate ( beatBytes ) { i => a_a ( 8 * i + 7 , 8 * i + 7 ) } . reverse )
val signbits_d = Cat ( Seq . tabulate ( beatBytes ) { i => a_d ( 8 * i + 7 , 8 * i + 7 ) } . reverse )
// Move the selected sign bit into the first byte position it will extend
val signbit_a = ( ( signbits_a & signSel ) << 1 ) ( beatBytes - 1 , 0 )
val signbit_d = ( ( signbits_d & signSel ) << 1 ) ( beatBytes - 1 , 0 )
val signext_a = FillInterleaved ( 8 , leftOR ( signbit_a ) )
val signext_d = FillInterleaved ( 8 , leftOR ( signbit_d ) )
// NOTE: sign-extension does not change the relative ordering in EITHER unsigned or signed arithmetic
val wide_mask = FillInterleaved ( 8 , mask )
val a_a_ext = ( a_a & wide_mask ) | signext_a
val a_d_ext = ( a_d & wide_mask ) | signext_d
val a_d_inv = Mux ( adder , a_d_ext , ~ a_d_ext )
val adder_out = a_a_ext + a_d_inv
val h = 8 * beatBytes - 1 // now sign-extended; use biggest bit
val a_bigger_uneq = unsigned === a_a_ext ( h ) // result if high bits are unequal
val a_bigger = Mux ( a_a_ext ( h ) === a_d_ext ( h ) , ! adder_out ( h ) , a_bigger_uneq )
val pick_a = take_max === a_bigger
val arith_out = Mux ( adder , adder_out , Mux ( pick_a , a_a , a_d ) )
// AMO result data
val amo_data =
if ( ! logical ) arith_out else
if ( ! arithmetic ) logic_out else
Mux ( a_cam_a . bits . opcode ( 0 ) , logic_out , arith_out )
// Potentially mutate the message from inner
val source_i = Wire ( in . a )
val a_allow = ! a_cam_busy && ( a_isSupported || a_cam_any_free )
in . a . ready : = source_i . ready && a_allow
source_i . valid : = in . a . valid && a_allow
source_i . bits : = in . a . bits
when ( ! a_isSupported ) { // minimal mux difference
source_i . bits . opcode : = TLMessages . Get
source_i . bits . param : = UInt ( 0 )
}
2016-09-21 01:49:57 +02:00
2017-01-30 00:17:52 +01:00
// Potentially take the message from the CAM
val source_c = Wire ( in . a )
source_c . valid : = a_cam_any_put
source_c . bits : = edgeOut . Put ( a_cam_a . bits . source , edgeIn . address ( a_cam_a . bits ) , a_cam_a . bits . size , amo_data ) . _2
// Finishing an AMO from the CAM has highest priority
TLArbiter ( TLArbiter . lowestIndexFirst ) ( out . a , ( UInt ( 0 ) , source_c ) , ( edgeOut . numBeats1 ( in . a . bits ) , source_i ) )
// Capture the A state into the CAM
when ( source_i . fire ( ) && ! a_isSupported ) {
( a_cam_sel_free zip cam_a ) foreach { case ( en , r ) =>
when ( en ) {
r . fifoId : = a_fifoId
r . bits : = in . a . bits
r . lut : = MuxLookup ( in . a . bits . param ( 1 , 0 ) , UInt ( 0 , width = 4 ) , Array (
TLAtomics . AND -> UInt ( 0x8 ) ,
TLAtomics . OR -> UInt ( 0xe ) ,
TLAtomics . XOR -> UInt ( 0x6 ) ,
TLAtomics . SWAP -> UInt ( 0xc ) ) )
}
2016-09-21 01:49:57 +02:00
}
2017-01-30 00:17:52 +01:00
( a_cam_sel_free zip cam_s ) foreach { case ( en , r ) =>
when ( en ) {
r . state : = GET
}
2016-09-21 01:49:57 +02:00
}
}
2017-01-30 00:17:52 +01:00
// Advance the put state
when ( source_c . fire ( ) ) {
( a_cam_sel_put zip cam_s ) foreach { case ( en , r ) =>
when ( en ) {
r . state : = ACK
}
2016-09-21 01:49:57 +02:00
}
}
2017-01-30 00:17:52 +01:00
// We need to deal with a potential D response in the same cycle as the A request
2017-07-26 21:52:29 +02:00
val d_first = edgeOut . first ( out . d )
2017-01-30 00:17:52 +01:00
val d_cam_sel_raw = cam_a . map ( _ . bits . source === in . d . bits . source )
val d_cam_sel_match = ( d_cam_sel_raw zip cam_dmatch ) map { case ( a , b ) => a && b }
val d_cam_data = Mux1H ( d_cam_sel_match , cam_d . map ( _ . data ) )
2017-11-06 20:31:23 +01:00
val d_cam_error = Mux1H ( d_cam_sel_match , cam_d . map ( _ . error ) )
2017-01-30 00:17:52 +01:00
val d_cam_sel_bypass = if ( edgeOut . manager . minLatency > 0 ) Bool ( false ) else
out . d . bits . source === in . a . bits . source && in . a . valid && ! a_isSupported
val d_cam_sel = ( a_cam_sel_free zip d_cam_sel_match ) map { case ( a , d ) => Mux ( d_cam_sel_bypass , a , d ) }
val d_cam_sel_any = d_cam_sel_bypass || d_cam_sel_match . reduce ( _ || _ )
val d_ackd = out . d . bits . opcode === TLMessages . AccessAckData
val d_ack = out . d . bits . opcode === TLMessages . AccessAck
2017-07-26 21:52:29 +02:00
when ( out . d . fire ( ) && d_first ) {
2017-01-30 00:17:52 +01:00
( d_cam_sel zip cam_d ) foreach { case ( en , r ) =>
when ( en && d_ackd ) {
r . data : = out . d . bits . data
2017-11-06 20:31:23 +01:00
r . error : = out . d . bits . error
2017-01-30 00:17:52 +01:00
}
2016-09-21 01:49:57 +02:00
}
2017-01-30 00:17:52 +01:00
( d_cam_sel zip cam_s ) foreach { case ( en , r ) =>
when ( en ) {
// Note: it is important that this comes AFTER the := GET, so we can go FREE=>GET=>AMO in one cycle
r . state : = Mux ( d_ackd , AMO , FREE )
}
2016-09-21 01:49:57 +02:00
}
}
2017-07-26 21:52:29 +02:00
val d_drop = d_first && d_ackd && d_cam_sel_any
val d_replace = d_first && d_ack && d_cam_sel_match . reduce ( _ || _ )
2016-09-21 01:49:57 +02:00
2017-01-30 00:17:52 +01:00
in . d . valid : = out . d . valid && ! d_drop
out . d . ready : = in . d . ready || d_drop
2016-09-21 01:49:57 +02:00
2017-01-30 00:17:52 +01:00
in . d . bits : = out . d . bits
when ( d_replace ) { // minimal muxes
in . d . bits . opcode : = TLMessages . AccessAckData
in . d . bits . data : = d_cam_data
2017-11-06 20:31:23 +01:00
in . d . bits . error : = d_cam_error || out . d . bits . error
2017-01-30 00:17:52 +01:00
}
} else {
out . a . valid : = in . a . valid
in . a . ready : = out . a . ready
out . a . bits : = in . a . bits
in . d . valid : = out . d . valid
out . d . ready : = in . d . ready
in . d . bits : = out . d . bits
2016-09-21 01:49:57 +02:00
}
2017-01-30 00:17:52 +01:00
if ( edgeOut . manager . anySupportAcquireB && edgeIn . client . anySupportProbe ) {
in . b . valid : = out . b . valid
out . b . ready : = in . b . ready
in . b . bits : = out . b . bits
out . c . valid : = in . c . valid
in . c . ready : = out . c . ready
out . c . bits : = in . c . bits
out . e . valid : = in . e . valid
in . e . ready : = out . e . ready
out . e . bits : = in . e . bits
} else {
in . b . valid : = Bool ( false )
in . c . ready : = Bool ( true )
in . e . ready : = Bool ( true )
out . b . ready : = Bool ( true )
out . c . valid : = Bool ( false )
out . e . valid : = Bool ( false )
}
2016-09-21 01:49:57 +02:00
}
}
}
object TLAtomicAutomata
{
2017-10-27 09:45:21 +02:00
def apply ( logical : Boolean = true , arithmetic : Boolean = true , concurrency : Int = 1 , passthrough : Boolean = true ) ( implicit p : Parameters ) : TLNode =
LazyModule ( new TLAtomicAutomata ( logical , arithmetic , concurrency , passthrough ) ) . node
2017-01-30 00:17:52 +01:00
case class CAMParams ( a : TLBundleParameters , domainsNeedingHelp : Int )
class CAM_S ( params : CAMParams ) extends GenericParameterizedBundle ( params ) {
val state = UInt ( width = 2 )
}
class CAM_A ( params : CAMParams ) extends GenericParameterizedBundle ( params ) {
val bits = new TLBundleA ( params . a )
val fifoId = UInt ( width = log2Up ( params . domainsNeedingHelp ) )
val lut = UInt ( width = 4 )
}
class CAM_D ( params : CAMParams ) extends GenericParameterizedBundle ( params ) {
2017-11-06 20:31:23 +01:00
val data = UInt ( width = params . a . dataBits )
val error = Bool ( )
2017-01-30 00:17:52 +01:00
}
2016-09-21 01:49:57 +02:00
}
2016-09-29 00:11:05 +02:00
/* * Synthesizeable unit tests */
2017-07-07 19:48:16 +02:00
import freechips.rocketchip.unittest._
2016-09-29 00:11:05 +02:00
2017-05-17 20:56:01 +02:00
class TLRAMAtomicAutomata ( txns : Int ) ( implicit p : Parameters ) extends LazyModule {
val fuzz = LazyModule ( new TLFuzzer ( txns ) )
2017-04-13 20:51:10 +02:00
val model = LazyModule ( new TLRAMModel ( "AtomicAutomata" ) )
2016-09-29 00:11:05 +02:00
val ram = LazyModule ( new TLRAM ( AddressSet ( 0x0 , 0x3ff ) ) )
2017-11-06 21:05:44 +01:00
// Confirm that the AtomicAutomata combines read + write errors
import TLMessages._
val test = new RequestPattern ( { a : TLBundleA =>
val doesA = a . opcode === ArithmeticData || a . opcode === LogicalData
val doesR = a . opcode === Get || doesA
val doesW = a . opcode === PutFullData || a . opcode === PutPartialData || doesA
( doesR && RequestPattern . overlaps ( Seq ( AddressSet ( 0x08 , ~ 0x08 ) ) ) ( a ) ) ||
( doesW && RequestPattern . overlaps ( Seq ( AddressSet ( 0x10 , ~ 0x10 ) ) ) ( a ) )
} )
2017-10-27 10:13:19 +02:00
( ram . node
2017-11-06 21:05:44 +01:00
: = TLErrorEvaluator ( test )
2017-10-27 10:13:19 +02:00
: = TLFragmenter ( 4 , 256 )
: = TLDelayer ( 0.1 )
: = TLAtomicAutomata ( )
: = TLDelayer ( 0.1 )
2017-11-06 21:05:44 +01:00
: = TLErrorEvaluator ( test , testOn = true , testOff = true )
2017-10-27 10:13:19 +02:00
: = model . node
: = fuzz . node )
2016-09-29 00:11:05 +02:00
2017-09-14 03:06:03 +02:00
lazy val module = new LazyModuleImp ( this ) with UnitTestModule {
2016-09-29 00:11:05 +02:00
io . finished : = fuzz . module . io . finished
}
}
2017-05-17 20:56:01 +02:00
class TLRAMAtomicAutomataTest ( txns : Int = 5000 , timeout : Int = 500000 ) ( implicit p : Parameters ) extends UnitTest ( timeout ) {
io . finished : = Module ( LazyModule ( new TLRAMAtomicAutomata ( txns ) ) . module ) . io . finished
2016-09-29 00:11:05 +02:00
}