Split Rocket + Shuttle Clock domains

src/CanHaveClockTap.scala

@@ -0,0 +1,27 @@
+package edu.berkeley.cs.iris
+
+import chisel3._
+
+import org.chipsalliance.cde.config.{Parameters, Field, Config}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.util._
+import freechips.rocketchip.tile._
+import freechips.rocketchip.prci._
+
+case object ClockTapKey extends Field[Boolean](true)
+
+trait CanHaveClockTap { this: BaseSubsystem =>
+  require(!p(SubsystemDriveClockGroupsFromIO), "Subsystem must not drive clocks from IO")
+  val clockTapNode = Option.when(p(ClockTapKey)) {
+    val clockTap = ClockSinkNode(Seq(ClockSinkParameters(name=Some("clock_tap"))))
+    clockTap := ClockGroup() := allClockGroupsNode
+    clockTap
+  }
+  val clockTapIO = clockTapNode.map { node => InModuleBody {
+    val clock_tap = IO(Output(Clock()))
+    clock_tap := node.in.head._1.clock
+    clock_tap
+  }}
+}

src/Iris.scala

@@ -45,6 +45,7 @@ class JTAGChipIO(hasReset: Boolean) extends Bundle {
 
 class IrisSystem(implicit p: Parameters)
     extends edu.berkeley.cs.chippy.ChippySystem
+    with CanHaveClockTap
     with testchipip.soc.CanHaveChipletRouting
     with testchipip.soc.CanHaveSubsystemInjectors // Enables the subsystem injector API
     with testchipip.soc.CanHaveSwitchableOffchipBus // Enables optional off-chip-bus with interface-switch
@@ -55,10 +56,17 @@ class IrisSystem(implicit p: Parameters)
 
 class IrisTop(implicit p: Parameters) extends LazyModule with BindingScope {
   val system = LazyModule(new IrisSystem)
+
+  val tlbus = system.asInstanceOf[BaseSubsystem].locateTLBusWrapper(system.prciParams.slaveWhere)
+  val baseAddress = system.prciParams.baseAddress
+  val clockDivider = system.prci_ctrl_domain { LazyModule(new TLClockDivider (baseAddress + 0x20000, tlbus.beatBytes, enable=true)) }
+
   val clockGroupsSourceNode = ClockGroupSourceNode(
     Seq(ClockGroupSourceParameters())
   )
-  system.chiptopClockGroupsNode := clockGroupsSourceNode
+  tlbus.coupleTo("clock-div-ctrl")    { clockDivider.tlNode := TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := TLBuffer() := _ }  // Off-chip clock divider
+  system.chiptopClockGroupsNode := clockDivider.clockNode := clockGroupsSourceNode
+
   val debugClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
   debugClockSinkNode := system
     .locateTLBusWrapper(p(ExportDebug).slaveWhere)
@@ -123,6 +131,9 @@ class IrisTop(implicit p: Parameters) extends LazyModule with BindingScope {
     // Tie off interupts and chip ID
     system.module.interrupts := DontCare
 
+    val clock_tap = IO(Output(Clock()))
+    clock_tap := system.clockTapIO.get.getWrappedValue
+
     val serial_tl = IO(
       new DecoupledExternalSyncPhitIO(p(SerialTLKey)(0).phyParams.phitWidth)
     )
@@ -242,6 +253,8 @@ class WithIrisBusTweaks
   */
 class WithIrisRocketCore
     extends Config(
+      // Decouple Rocket Core from uncore freq
+      new freechips.rocketchip.rocket.WithAsynchronousCDCs(4, 3) ++
       // ICache
       new freechips.rocketchip.rocket.WithL1ICacheWays(2) ++
         new freechips.rocketchip.rocket.WithL1ICacheSets(128) ++
@@ -347,7 +360,8 @@ class WithIrisUncore(sim: Boolean = false)
               "cbus",
               "system_bus",
               "fbus",
-              "pbus"
+              "pbus",
+              "d2d"
             ),
             Nil
           )
@@ -468,15 +482,14 @@ class IrisConfig(sim: Boolean = false)
         // ==================================
         // Shuttle Tile + Saturn Cores
         // ==================================
-        // new shuttle.common.WithAsynchronousShuttleTiles(3, 3, location=InCluster(0)) ++ // Add async crossings between RocketTile and uncore
+        new shuttle.common.WithAsynchronousShuttleTiles(4, 3) ++ // Add async crossings between RocketTile and uncore
         new saturn.shuttle.WithShuttleVectorUnit(
           256,
           128,
           VectorParams.opuMxParams
         ) ++
         new shuttle.common.WithShuttleTileBeatBytes(16) ++
         new shuttle.common.WithTCM(size = 128L << 10, banks = 2) ++
-        new shuttle.common.WithShuttleTileBoundaryBuffers() ++
         // ICache
         new shuttle.common.WithL1ICacheWays(2) ++
         new shuttle.common.WithL1ICacheSets(64) ++

src/TLClockDivider.scala

@@ -0,0 +1,86 @@
+package edu.berkeley.cs.iris
+
+import chisel3._
+
+import org.chipsalliance.cde.config.Parameters
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.regmapper._
+import freechips.rocketchip.util._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.util.ElaborationArtefacts
+
+import testchipip.clocking._
+
+// This module adds a TileLink memory-mapped clock divider to the clock graph
+// The output clock/reset pairs from this module should be synchronized later
+// If enable is unset, this will not divide the clock
+// DO NOT unset enable for VLSI, or prototyping flows. The disable feature is a work around for
+// some RTL simulators which do not simulate the reset synchronization properly
+class TLClockDivider(address: BigInt, beatBytes: Int, divBits: Int = 8, enable: Boolean = true)(implicit p: Parameters) extends LazyModule {
+  val device = new SimpleDevice(s"clk-div-ctrl", Nil)
+  val clockNode = ClockGroupIdentityNode()
+  val tlNode = TLRegisterNode(Seq(AddressSet(address, 4096-1)), device, "reg/control", beatBytes=beatBytes)
+
+  if (!enable) println(Console.RED + s"""
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+WARNING:
+
+YOU ARE USING THE TLCLOCKDIVIDER IN
+"DISABLED" MODE. THIS SHOULD ONLY BE DONE
+FOR RTL SIMULATION
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+""" + Console.RESET)
+
+  lazy val module = new LazyModuleImp(this) {
+    require (clockNode.out.size == 1)
+    val sources = clockNode.in.head._1.member.data.toSeq
+    val sinks = clockNode.out.head._1.member.elements.toSeq
+    require (sources.size == sinks.size)
+    val nSinks = sinks.size
+    // The implicit clock of this module is the clock of the tilelink bus
+    // busReset is sync'd to that clock, and will be asserted longer than the
+    // resets coming in through the clockNode, since the busReset is derived from
+    // the clockNode resets in downstream PRCI nodes
+    val busReset = reset
+
+    val regs = (0 until nSinks) .map { i =>
+      val sinkName = sinks(i)._1
+      val asyncReset = sources(i).reset
+      val reg = Module(new AsyncResetRegVec(w=divBits, init=0))
+
+      println(s"${(address+i*4).toString(16)}: Clock domain $sinkName divider")
+      val divider = Module(new ClockDivideOrPass(divBits, depth = 3, genClockGate = p(ClockGateImpl)))
+      divider.io.clockIn := sources(i).clock
+      // busReset is expected to be high for a long time, since reset will take a while to propagate
+      // to the TL bus. While reset is propagating, make sure we propagate a fast, undivided clock
+      // by setting divisor=0. The divisor signal into the ClockDividerOrPass is synchronized internally
+      divider.io.divisor := Mux(busReset.asBool, 0.U, reg.io.q)
+      divider.io.resetAsync := ResetStretcher(sources(i).clock, asyncReset, 20).asAsyncReset
+
+      if (enable) {
+        sinks(i)._2.clock := divider.io.clockOut
+
+        // Note this is not synchronized to the output clock, which takes time to appear
+        // so this is still asyncreset
+        // Stretch the reset for 40 cycles, to give enough time to reset any downstream
+        // digital logic
+        sinks(i)._2.reset := ResetStretcher(sources(i).clock, asyncReset, 40).asAsyncReset
+      } else {
+        // WARNING: THIS IS FOR RTL SIMULATION ONLY
+        sinks(i)._2.clock := sources(i).clock
+        sinks(i)._2.reset := sources(i).reset
+      }
+
+      reg
+    }
+
+    tlNode.regmap((0 until nSinks).map { i =>
+      i * 4 -> Seq(RegField.rwReg(divBits, regs(i).io))
+    }: _*)
+  }
+}