Fix LRSC starvation bug by punching Finish messages out to caching clients via a new TileLinkNetworkPort.

uncore/src/main/scala/converters.scala

@@ -77,12 +77,12 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e
     }
     val q = Module(new FinishQueue(outstanding))
     q.io.enq.valid := io.grant.fire() && g.requiresAck() && (!g.hasMultibeatData() || done)
-    q.io.enq.bits.fin := g.makeFinish()
+    q.io.enq.bits := g.makeFinish()
-    q.io.enq.bits.dst := io.grant.bits.header.src
+    q.io.enq.bits.client_id := io.grant.bits.header.src
 
     io.finish.bits.header.src := UInt(srcId)
-    io.finish.bits.header.dst := q.io.deq.bits.dst
+    io.finish.bits.header.dst := q.io.deq.bits.client_id
-    io.finish.bits.payload := q.io.deq.bits.fin
+    io.finish.bits.payload := q.io.deq.bits
     io.finish.valid := q.io.deq.valid
     q.io.deq.ready := io.finish.ready
 
@@ -93,19 +93,15 @@ class FinishUnit(srcId: Int = 0, outstanding: Int = 2)(implicit p: Parameters) e
   }
 }
 
-class FinishQueueEntry(implicit p: Parameters) extends TLBundle()(p) {
+class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishToDst()(p), entries)
-    val fin = new Finish
-    val dst = UInt(width = log2Up(p(LNEndpoints)))
-}
-
-class FinishQueue(entries: Int)(implicit p: Parameters) extends Queue(new FinishQueueEntry()(p), entries)
 
 /** A port to convert [[uncore.ClientTileLinkIO]].flip into [[uncore.TileLinkIO]]
   *
   * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages,
   * calculating header.dst and filling in header.src.
   * Strips headers from [[uncore.Probe Probes]].
-  * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers.
+  * Passes [[uncore.GrantFromSrc]] and accepts [[uncore.FinishFromDst]] in response,
+  * setting up the headers for each.
   *
   * @param clientId network port id of this agent
   * @param addrConvert how a physical address maps to a destination manager port id
@@ -117,24 +113,64 @@ class ClientTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt)
     val network = new TileLinkIO
   }
 
+  val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert)
+  val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert)
+  val fin_with_header = ClientTileLinkHeaderCreator(io.client.finish, clientId)
+  val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe)
+  val gnt_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.grant)
+
+  io.network.acquire <> acq_with_header
+  io.network.release <> rel_with_header
+  io.network.finish <> fin_with_header
+  io.client.probe <> prb_without_header
+  io.client.grant.bits.client_id := io.network.grant.bits.header.src
+  io.client.grant <> gnt_without_header
+}
+
+/** A port to convert [[uncore.ClientUncachedTileLinkIO]].flip into [[uncore.TileLinkIO]]
+  *
+  * Creates network headers for [[uncore.Acquire]] and [[uncore.Release]] messages,
+  * calculating header.dst and filling in header.src.
+  * Responds to [[uncore.Grant]] by automatically issuing [[uncore.Finish]] to the granting managers.
+  *
+  * @param clientId network port id of this agent
+  * @param addrConvert how a physical address maps to a destination manager port id
+  */
+class ClientUncachedTileLinkNetworkPort(clientId: Int, addrConvert: UInt => UInt)
+                               (implicit p: Parameters) extends TLModule()(p) {
+  val io = new Bundle {
+    val client = new ClientUncachedTileLinkIO().flip
+    val network = new TileLinkIO
+  }
+
   val finisher = Module(new FinishUnit(clientId))
   finisher.io.grant <> io.network.grant
   io.network.finish <> finisher.io.finish
 
   val acq_with_header = ClientTileLinkHeaderCreator(io.client.acquire, clientId, addrConvert)
-  val rel_with_header = ClientTileLinkHeaderCreator(io.client.release, clientId, addrConvert)
-  val prb_without_header = DecoupledLogicalNetworkIOUnwrapper(io.network.probe)
   val gnt_without_header = finisher.io.refill
 
   io.network.acquire.bits := acq_with_header.bits
   io.network.acquire.valid := acq_with_header.valid && finisher.io.ready
   acq_with_header.ready := io.network.acquire.ready && finisher.io.ready
-  io.network.release <> rel_with_header
-  io.client.probe <> prb_without_header
   io.client.grant <> gnt_without_header
+  io.network.probe.ready :=  Bool(false)
+  io.network.release.valid := Bool(false)
 }
 
 object ClientTileLinkHeaderCreator {
+  def apply[T <: ClientToManagerChannel with HasClientId](
+        in: DecoupledIO[T],
+        clientId: Int)
+      (implicit p: Parameters): DecoupledIO[LogicalNetworkIO[T]] = {
+    val out = Wire(new DecoupledIO(new LogicalNetworkIO(in.bits)))
+    out.bits.payload := in.bits
+    out.bits.header.src := UInt(clientId)
+    out.bits.header.dst := in.bits.client_id
+    out.valid := in.valid
+    in.ready := out.ready
+    out
+  }
   def apply[T <: ClientToManagerChannel with HasCacheBlockAddress](
         in: DecoupledIO[T],
         clientId: Int,
@@ -1098,5 +1134,5 @@ class MMIOTileLinkManager(implicit p: Parameters)
   io.inner.grant.bits.client_id := gnt_xact.client_id
   io.inner.grant.bits.client_xact_id := gnt_xact.client_xact_id
   io.inner.grant.bits.manager_xact_id := io.ognt().client_xact_id
-  io.inner.finish.ready := xact_pending(io.inner.finish.bits.manager_xact_id)
+  io.inner.finish.ready := Bool(true)
 }

uncore/src/main/scala/interconnect.scala

@@ -5,6 +5,191 @@ import junctions._
 import scala.collection.mutable.ArraySeq
 import cde.{Parameters, Field}
 
+
+/** PortedTileLinkNetworks combine a TileLink protocol with a particular physical
+  * network implementation.
+  *
+  * Specifically, they provide mappings between ClientTileLinkIO/ 
+  * ManagerTileLinkIO channels and LogicalNetwork ports (i.e. generic
+  * TileLinkIO with networking headers). Channels coming into the network have
+  * appropriate networking headers appended and outgoing channels have their
+  * headers stripped.
+  *
+  * @constructor base class constructor for Ported TileLink NoC
+  * @param addrToManagerId a mapping from a physical address to the network
+  *        id of a coherence manager
+  * @param sharerToClientId a mapping from the id of a particular coherent
+  *        client (as determined by e.g. the directory) and the network id
+  *        of that client
+  * @param clientDepths the depths of the queue that should be used to buffer
+  *        each channel on the client side of the network
+  * @param managerDepths the depths of the queue that should be used to buffer
+  *        each channel on the manager side of the network
+  */
+abstract class PortedTileLinkNetwork(
+      addrToManagerId: UInt => UInt,
+      sharerToClientId: UInt => UInt,
+      clientDepths: TileLinkDepths, 
+      managerDepths: TileLinkDepths)
+    (implicit p: Parameters) extends TLModule()(p) {
+  val nClients = tlNClients
+  val nManagers = tlNManagers
+  val io = new Bundle {
+    val clients_cached = Vec(tlNCachingClients, new ClientTileLinkIO).flip
+    val clients_uncached = Vec(tlNCachelessClients, new ClientUncachedTileLinkIO).flip
+    val managers = Vec(nManagers, new ManagerTileLinkIO).flip
+  }
+
+  val clients = (io.clients_cached ++ io.clients_uncached).zipWithIndex.map { 
+    case (io, idx) => {
+      val qs = Module(new TileLinkEnqueuer(clientDepths))
+      io match {
+        case c: ClientTileLinkIO => {
+          val port = Module(new ClientTileLinkNetworkPort(idx, addrToManagerId))
+          port.io.client <> c
+          qs.io.client <> port.io.network
+          qs.io.manager 
+        }
+        case u: ClientUncachedTileLinkIO => {
+          val port = Module(new ClientUncachedTileLinkNetworkPort(idx, addrToManagerId))
+          port.io.client <> u
+          qs.io.client <> port.io.network
+          qs.io.manager 
+        }
+      }
+    }
+  }
+
+  val managers = io.managers.zipWithIndex.map {
+    case (m, i) => {
+      val port = Module(new ManagerTileLinkNetworkPort(i, sharerToClientId))
+      val qs = Module(new TileLinkEnqueuer(managerDepths))
+      port.io.manager <> m
+      port.io.network <> qs.io.manager
+      qs.io.client
+    }
+  }
+}
+
+/** A simple arbiter for each channel that also deals with header-based routing.
+  * Assumes a single manager agent. */
+class PortedTileLinkArbiter(
+      sharerToClientId: UInt => UInt = (u: UInt) => u,
+      clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), 
+      managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0))
+    (implicit p: Parameters)
+      extends PortedTileLinkNetwork(u => UInt(0), sharerToClientId, clientDepths, managerDepths)(p)
+        with TileLinkArbiterLike
+        with PassesId {
+  val arbN = nClients
+  require(nManagers == 1)
+  if(arbN > 1) {
+    hookupClientSource(clients.map(_.acquire), managers.head.acquire)
+    hookupClientSource(clients.map(_.release), managers.head.release)
+    hookupFinish(clients.map(_.finish), managers.head.finish)
+    hookupManagerSourceWithHeader(clients.map(_.probe), managers.head.probe)
+    hookupManagerSourceWithHeader(clients.map(_.grant), managers.head.grant)
+  } else {
+    managers.head <> clients.head
+  }
+}
+
+/** Provides a separate physical crossbar for each channel. Assumes multiple manager
+  * agents. Managers are assigned to higher physical network port ids than
+  * clients, and translations between logical network id and physical crossbar
+  * port id are done automatically.
+  */
+class PortedTileLinkCrossbar(
+      addrToManagerId: UInt => UInt = u => UInt(0),
+      sharerToClientId: UInt => UInt = u => u,
+      clientDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0), 
+      managerDepths: TileLinkDepths = TileLinkDepths(0,0,0,0,0))
+    (implicit p: Parameters)
+      extends PortedTileLinkNetwork(addrToManagerId, sharerToClientId, clientDepths, managerDepths)(p) {
+  val n = p(LNEndpoints)
+  val count = tlDataBeats
+  // Actually instantiate the particular networks required for TileLink
+  val acqNet = Module(new BasicCrossbar(n, new Acquire, count, Some((a: PhysicalNetworkIO[Acquire]) => a.payload.hasMultibeatData())))
+  val relNet = Module(new BasicCrossbar(n, new Release, count, Some((r: PhysicalNetworkIO[Release]) => r.payload.hasMultibeatData())))
+  val prbNet = Module(new BasicCrossbar(n, new Probe))
+  val gntNet = Module(new BasicCrossbar(n, new Grant, count, Some((g: PhysicalNetworkIO[Grant]) => g.payload.hasMultibeatData())))
+  val ackNet = Module(new BasicCrossbar(n, new Finish))
+
+  // Aliases for the various network IO bundle types
+  type PNIO[T <: Data] = DecoupledIO[PhysicalNetworkIO[T]]
+  type LNIO[T <: Data] = DecoupledIO[LogicalNetworkIO[T]]
+  type FromCrossbar[T <: Data] = PNIO[T] => LNIO[T]
+  type ToCrossbar[T <: Data] = LNIO[T] => PNIO[T]
+
+  // Shims for converting between logical network IOs and physical network IOs
+  def crossbarToManagerShim[T <: Data](in: PNIO[T]): LNIO[T] = {
+    val out = DefaultFromPhysicalShim(in)
+    out.bits.header.src := in.bits.header.src - UInt(nManagers)
+    out
+  }
+  def crossbarToClientShim[T <: Data](in: PNIO[T]): LNIO[T] = {
+    val out = DefaultFromPhysicalShim(in)
+    out.bits.header.dst := in.bits.header.dst - UInt(nManagers)
+    out
+  }
+  def managerToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = {
+    val out = DefaultToPhysicalShim(n, in)
+    out.bits.header.dst := in.bits.header.dst + UInt(nManagers)
+    out
+  }
+  def clientToCrossbarShim[T <: Data](in: LNIO[T]): PNIO[T] = {
+    val out = DefaultToPhysicalShim(n, in)
+    out.bits.header.src := in.bits.header.src + UInt(nManagers)
+    out
+  }
+
+  // Make an individual connection between virtual and physical ports using
+  // a particular shim. Also pin the unused Decoupled control signal low.
+  def doDecoupledInputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: ToCrossbar[T]) = {
+    val s = shim(log_io)
+    phys_in.valid := s.valid
+    phys_in.bits := s.bits
+    s.ready := phys_in.ready
+    phys_out.ready := Bool(false)
+  }
+
+  def doDecoupledOutputHookup[T <: Data](phys_in: PNIO[T], phys_out: PNIO[T], log_io: LNIO[T], shim: FromCrossbar[T]) = {
+    val s = shim(phys_out)
+    log_io.valid := s.valid
+    log_io.bits := s.bits
+    s.ready := log_io.ready
+    phys_in.valid := Bool(false)
+  }
+
+  //Hookup all instances of a particular subbundle of TileLink
+  def doDecoupledHookups[T <: Data](physIO: BasicCrossbarIO[T], getLogIO: TileLinkIO => LNIO[T]) = {
+    physIO.in.head.bits.payload match {
+      case c: ClientToManagerChannel => {
+        managers.zipWithIndex.map { case (i, id) => 
+          doDecoupledOutputHookup(physIO.in(id), physIO.out(id), getLogIO(i), crossbarToManagerShim[T])
+        }
+        clients.zipWithIndex.map { case (i, id) =>
+          doDecoupledInputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), clientToCrossbarShim[T])
+        }
+      }
+      case m: ManagerToClientChannel => {
+        managers.zipWithIndex.map { case (i, id) =>
+          doDecoupledInputHookup(physIO.in(id), physIO.out(id), getLogIO(i), managerToCrossbarShim[T])
+        }
+        clients.zipWithIndex.map { case (i, id) =>
+          doDecoupledOutputHookup(physIO.in(id+nManagers), physIO.out(id+nManagers), getLogIO(i), crossbarToClientShim[T])
+        }
+      }
+    }
+  }
+
+  doDecoupledHookups(acqNet.io, (tl: TileLinkIO) => tl.acquire)
+  doDecoupledHookups(relNet.io, (tl: TileLinkIO) => tl.release)
+  doDecoupledHookups(prbNet.io, (tl: TileLinkIO) => tl.probe)
+  doDecoupledHookups(gntNet.io, (tl: TileLinkIO) => tl.grant)
+  doDecoupledHookups(ackNet.io, (tl: TileLinkIO) => tl.finish)
+}
+
 class ClientUncachedTileLinkIORouter(
     nOuter: Int, routeSel: UInt => UInt)(implicit p: Parameters)
     extends TLModule {

uncore/src/main/scala/tilelink.scala

@@ -785,6 +785,17 @@ class Grant(implicit p: Parameters) extends GrantMetadata
 class GrantToDst(implicit p: Parameters) extends Grant
   with HasClientId
 
+/** [[uncore.Grant]] with an extra field stating its destination */
+class GrantFromSrc(implicit p: Parameters) extends Grant
+    with HasClientId {
+  override def makeFinish(dummy: Int = 0): FinishToDst = {
+    val f = Wire(new FinishToDst)
+    f.manager_xact_id := this.manager_xact_id
+    f.client_id := this.client_id
+    f
+  }
+}
+
 /** [[uncore.GrantMetadata]] with an extra field containing an entire cache block */
 class BufferedGrant(implicit p: Parameters) extends GrantMetadata
   with HasTileLinkBlock
@@ -868,6 +879,10 @@ class Finish(implicit p: Parameters) extends ClientToManagerChannel()(p)
   def hasMultibeatData(dummy: Int = 0) = Bool(false)
 }
 
+/** [[uncore.Finish]] with an extra field stating its destination */
+class FinishToDst(implicit p: Parameters) extends Finish
+  with HasClientId
+
 /** Complete IO definition for incoherent TileLink, including networking headers */
 class UncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) {
   val acquire   = new DecoupledIO(new LogicalNetworkIO(new Acquire))
@@ -901,9 +916,12 @@ class ClientUncachedTileLinkIO(implicit p: Parameters) extends TLBundle()(p) {
 /** This version of TileLinkIO does not contain network headers. 
   * It is intended for use within client agents.
   */
-class ClientTileLinkIO(implicit p: Parameters) extends ClientUncachedTileLinkIO()(p) {
+class ClientTileLinkIO(implicit p: Parameters) extends TLBundle()(p) {
+  val acquire   = new DecoupledIO(new Acquire)
   val probe     = new DecoupledIO(new Probe).flip
   val release   = new DecoupledIO(new Release)
+  val grant     = new DecoupledIO(new GrantFromSrc).flip
+  val finish    = new DecoupledIO(new FinishToDst)
 }
 
 /** This version of TileLinkIO does not contain network headers, but