From 2f40de86cdd4b7a51696990e9c385c90eaa903fa Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Sat, 6 Jun 2026 11:53:59 -0700 Subject: [PATCH 1/4] adding RoCEv2 targets --- firmware/Makefile | 28 +++ firmware/releases.yaml | 15 +- firmware/submodules/surf | 2 +- .../RoCEv2_10GbeRudpKcu105Example/Makefile | 6 + .../hdl/RoCEv2_10GbeRudpKcu105Example.vhd | 231 ++++++++++++++++++ .../RoCEv2_10GbeRudpKcu105Example/ruckus.tcl | 10 + .../vivado/promgen.tcl | 13 + .../RoCEv2_1GbeRudpKcu105Example/Makefile | 6 + .../hdl/RoCEv2_1GbeRudpKcu105Example.vhd | 230 +++++++++++++++++ .../RoCEv2_1GbeRudpKcu105Example/ruckus.tcl | 10 + .../vivado/promgen.tcl | 13 + .../RoCEv2_Rj45RudpKcu105Example/Makefile | 6 + .../hdl/RoCEv2_Rj45RudpKcu105Example.vhd | 229 +++++++++++++++++ .../RoCEv2_Rj45RudpKcu105Example/ruckus.tcl | 15 ++ .../vivado/promgen.tcl | 13 + firmware/targets/shared_version.mk | 4 +- 16 files changed, 814 insertions(+), 17 deletions(-) create mode 100755 firmware/Makefile create mode 100755 firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile create mode 100755 firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd create mode 100755 firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl create mode 100755 firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl create mode 100755 firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile create mode 100755 firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd create mode 100755 firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl create mode 100755 firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl create mode 100755 firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile create mode 100755 firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd create mode 100755 firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl create mode 100755 firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl diff --git a/firmware/Makefile b/firmware/Makefile new file mode 100755 index 0000000..f40e015 --- /dev/null +++ b/firmware/Makefile @@ -0,0 +1,28 @@ +export TARGET_DIRS = $(PWD)/targets/RoCEv2_1GbeRudpKcu105Examples \ + $(PWD)/targets/RoCEv2_10GbeRudpKcu105Example \ + $(PWD)/targets/RoCEv2_Rj45RudpKcu105Example \ + $(PWD)/targets/Simple1GbeRudpKcu105Example \ + $(PWD)/targets/Simple10GbeRudpKcu105Example \ + $(PWD)/targets/SimpleRj45RudpKcu105Example + +.PHONY: all build clean + +# Default +all: build + +# Check variables +test: + @echo TARGET_DIRS: + @echo -e "$(foreach ARG,$(TARGET_DIRS),\t$(ARG)\n)" + +# Clean all firmware builds +clean: + for i in $(TARGET_DIRS); do \ + cd $$i; make clean; \ + done + +# Build targets +build: + for i in $(TARGET_DIRS); do \ + echo $$i; cd $$i; make clean; rm -rf images; make; \ + done diff --git a/firmware/releases.yaml b/firmware/releases.yaml index 0aefd12..8ebc2c2 100644 --- a/firmware/releases.yaml +++ b/firmware/releases.yaml @@ -1,23 +1,10 @@ GitBase: .. -TopRoguePackage: simple_10gbe_rudp_kcu105_example - -RoguePackages: - - submodules/surf/python - - python - -RogueConfig: - - ../software/config - -RogueScripts: - - ../software/scripts/devGui.py - Targets: Simple10GbeRudpKcu105Example: ImageDir: targets/Simple10GbeRudpKcu105Example/images Extensions: - - bit - mcs - ltx @@ -27,4 +14,4 @@ Releases: Targets: - Simple10GbeRudpKcu105Example Types: - - Rogue + - FW_only diff --git a/firmware/submodules/surf b/firmware/submodules/surf index 634c8d9..e9bcd67 160000 --- a/firmware/submodules/surf +++ b/firmware/submodules/surf @@ -1 +1 @@ -Subproject commit 634c8d9656e7f6c99e52c7d943fb6f72e74a95fa +Subproject commit e9bcd6763d07076468cc188aa4fb8def37bdb122 diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd new file mode 100755 index 0000000..4f8540d --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd @@ -0,0 +1,231 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 10G-BASER Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; +use surf.RocePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_10GbeRudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := true; + DCQCN_EN_G : boolean := true; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_10GbeRudpKcu105Example; + +architecture top_level of RoCEv2_10GbeRudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RoCE Engine + signal workReqMaster : RoceWorkReqMasterType; + signal workReqSlave : RoceWorkReqSlaveType; + signal workCompMaster : RoceWorkCompMasterType; + signal workCompSlave : RoceWorkCompSlaveType; + signal dmaReadRespMaster : RoceDmaReadRespMasterType; + signal dmaReadRespSlave : RoceDmaReadRespSlaveType; + signal dmaReadReqMaster : RoceDmaReadReqMasterType; + signal dmaReadReqSlave : RoceDmaReadReqSlaveType; + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + DCQCN_EN_G => DCQCN_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => SFP_10G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..540b450 --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/ruckus.tcl @@ -0,0 +1,10 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../Simple10GbeRudpKcu105Example/hdl" diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd new file mode 100755 index 0000000..d57c546 --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd @@ -0,0 +1,230 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 1000baseX/Full Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; +use surf.RocePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_1GbeRudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_1GbeRudpKcu105Example; + +architecture top_level of RoCEv2_1GbeRudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RoCE Engine + signal workReqMaster : RoceWorkReqMasterType; + signal workReqSlave : RoceWorkReqSlaveType; + signal workCompMaster : RoceWorkCompMasterType; + signal workCompSlave : RoceWorkCompSlaveType; + signal dmaReadRespMaster : RoceDmaReadRespMasterType; + signal dmaReadRespSlave : RoceDmaReadRespSlaveType; + signal dmaReadReqMaster : RoceDmaReadReqMasterType; + signal dmaReadReqSlave : RoceDmaReadReqSlaveType; + + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => SFP_1G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..6890cbf --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/ruckus.tcl @@ -0,0 +1,10 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../Simple1GbeRudpKcu105Example/hdl" diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile new file mode 100755 index 0000000..a4be7fd --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/Makefile @@ -0,0 +1,6 @@ +# Define the TOP_DIR path +export TOP_DIR = $(abspath $(PWD)/../..) + +# Use top level makefile +include ../shared_version.mk +include $(TOP_DIR)/submodules/ruckus/system_vivado.mk diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd new file mode 100755 index 0000000..111eadf --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd @@ -0,0 +1,229 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Simple 1000baseT/Full Example +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; +use surf.RocePkg.all; + +library work; +use work.CorePkg.all; + +entity RoCEv2_Rj45RudpKcu105Example is + generic ( + TPD_G : time := 1 ns; + BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + SIMULATION_G : boolean := false; + IP_ADDR_G : slv(31 downto 0) := x"0A02A8C0"; -- 192.168.2.10 + DHCP_G : boolean := false); + port ( + -- I2C Ports + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; + -- XADC Ports + vPIn : in sl; + vNIn : in sl; + -- System Ports + emcClk : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + led : out slv(7 downto 0); + -- Boot Memory Ports + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; + -- SFP ETH Ports + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; + -- RJ45 ETH Ports + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); +end RoCEv2_Rj45RudpKcu105Example; + +architecture top_level of RoCEv2_Rj45RudpKcu105Example is + + signal heartbeat : sl; + signal phyReady : sl; + signal rssiLinkUp : slv(1 downto 0); + + -- Clock and Reset + signal axilClk : sl; + signal axilRst : sl; + + -- AXI-Stream: Stream Interface + signal ibRudpMaster : AxiStreamMasterType; + signal ibRudpSlave : AxiStreamSlaveType; + signal obRudpMaster : AxiStreamMasterType; + signal obRudpSlave : AxiStreamSlaveType; + + -- AXI-Lite: Register Access + signal axilReadMaster : AxiLiteReadMasterType; + signal axilReadSlave : AxiLiteReadSlaveType; + signal axilWriteMaster : AxiLiteWriteMasterType; + signal axilWriteSlave : AxiLiteWriteSlaveType; + + -- RoCE Engine + signal workReqMaster : RoceWorkReqMasterType; + signal workReqSlave : RoceWorkReqSlaveType; + signal workCompMaster : RoceWorkCompMasterType; + signal workCompSlave : RoceWorkCompSlaveType; + signal dmaReadRespMaster : RoceDmaReadRespMasterType; + signal dmaReadRespSlave : RoceDmaReadRespSlaveType; + signal dmaReadReqMaster : RoceDmaReadReqMasterType; + signal dmaReadReqSlave : RoceDmaReadReqSlaveType; + +begin + + led(7) <= '1'; + led(6) <= '0'; + led(5) <= heartbeat; + led(4) <= axilRst; + led(3) <= not(axilRst); + led(2) <= rssiLinkUp(1); + led(1) <= rssiLinkUp(0); + led(0) <= phyReady; + + ----------------------- + -- Core Firmware Module + ----------------------- + U_Core : entity work.Core + generic map ( + TPD_G => TPD_G, + BUILD_INFO_G => BUILD_INFO_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G, + ETH_BUILD_G => RJ45_1G_C, + IP_ADDR_G => IP_ADDR_G, + DHCP_G => DHCP_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- I2C Ports + sfpTxDisL => sfpTxDisL, + i2cRstL => i2cRstL, + i2cScl => i2cScl, + i2cSda => i2cSda, + -- SYSMON Ports + vPIn => vPIn, + vNIn => vNIn, + -- System Ports + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, + emcClk => emcClk, + heartbeat => heartbeat, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, + -- Boot Memory Ports + flashCsL => flashCsL, + flashMosi => flashMosi, + flashMiso => flashMiso, + flashHoldL => flashHoldL, + flashWp => flashWp, + -- SFP ETH Ports + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, + -- RJ45 ETH Ports + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); + + ------------------------------ + -- Application Firmware Module + ------------------------------ + U_App : entity work.App + generic map ( + TPD_G => TPD_G, + ROCEV2_EN_G => ROCEV2_EN_G, + SIMULATION_G => SIMULATION_G) + port map ( + -- Clock and Reset + axilClk => axilClk, + axilRst => axilRst, + -- AXI-Stream Interface + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, + -- RoCE Work Request/Completion Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- RoCE DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + -- AXI-Lite Interface + axilReadMaster => axilReadMaster, + axilReadSlave => axilReadSlave, + axilWriteMaster => axilWriteMaster, + axilWriteSlave => axilWriteSlave); + +end top_level; diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl new file mode 100755 index 0000000..fb03bbd --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/ruckus.tcl @@ -0,0 +1,15 @@ +# Load RUCKUS environment and library +source $::env(RUCKUS_PROC_TCL) + +# Load shared and sub-module ruckus.tcl files +loadRuckusTcl $::env(TOP_DIR)/submodules/surf +loadRuckusTcl $::env(TOP_DIR)/shared + +# Load local source Code and constraints +loadSource -dir "$::DIR_PATH/hdl" +loadConstraints -dir "$::DIR_PATH/../SimpleRj45RudpKcu105Example/hdl" + +# Modified the .XDC property +set_property PROCESSING_ORDER {EARLY} [get_files {GigEthLvdsUltraScaleCore.xdc}] +set_property SCOPED_TO_REF {GigEthLvdsUltraScaleCore} [get_files {GigEthLvdsUltraScaleCore.xdc}] +set_property SCOPED_TO_CELLS {U0} [get_files {GigEthLvdsUltraScaleCore.xdc}] diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl new file mode 100755 index 0000000..938b225 --- /dev/null +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/vivado/promgen.tcl @@ -0,0 +1,13 @@ +############################################################################## +## This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +## It is subject to the license terms in the LICENSE.txt file found in the +## top-level directory of this distribution and at: +## https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +## No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +## may be copied, modified, propagated, or distributed except according to +## the terms contained in the LICENSE.txt file. +############################################################################## + +set format "mcs" +set inteface "SPIx8" +set size "512" diff --git a/firmware/targets/shared_version.mk b/firmware/targets/shared_version.mk index 6e9398f..d76c28f 100755 --- a/firmware/targets/shared_version.mk +++ b/firmware/targets/shared_version.mk @@ -1,5 +1,5 @@ -# Define Firmware Version: v2.19.0.0 -export PRJ_VERSION = 0x02190000 +# Define Firmware Version: v3.0.0.0 +export PRJ_VERSION = 0x03000000 # Define target output target: prom From b6734074c84b17d89c37f0d5c15ff2f0b7575cbe Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Sat, 6 Jun 2026 12:31:40 -0700 Subject: [PATCH 2/4] updating shared for RoCEv2 --- firmware/shared/rtl/App.vhd | 94 ++++++- firmware/shared/rtl/Core.vhd | 183 +++++++------ firmware/shared/rtl/DmaTestPatternServer.vhd | 234 ++++++++++++++++ firmware/shared/rtl/Rudp.vhd | 171 ++++++++---- firmware/shared/rtl/WorkCompChecker.vhd | 189 +++++++++++++ firmware/shared/rtl/WorkReqDispatcher.vhd | 259 ++++++++++++++++++ firmware/shared/ruckus.tcl | 3 + .../Simple10GbeRudpKcu105Example/ruckus.tcl | 3 - .../Simple1GbeRudpKcu105Example/ruckus.tcl | 3 - .../SimpleRj45RudpKcu105Example/ruckus.tcl | 3 - 10 files changed, 990 insertions(+), 152 deletions(-) mode change 100644 => 100755 firmware/shared/rtl/App.vhd create mode 100755 firmware/shared/rtl/DmaTestPatternServer.vhd create mode 100755 firmware/shared/rtl/WorkCompChecker.vhd create mode 100755 firmware/shared/rtl/WorkReqDispatcher.vhd diff --git a/firmware/shared/rtl/App.vhd b/firmware/shared/rtl/App.vhd old mode 100644 new mode 100755 index 3337b60..6881d75 --- a/firmware/shared/rtl/App.vhd +++ b/firmware/shared/rtl/App.vhd @@ -19,33 +19,47 @@ library surf; use surf.StdRtlPkg.all; use surf.AxiStreamPkg.all; use surf.AxiLitePkg.all; +use surf.RoCEv2Pkg.all; entity App is generic ( TPD_G : time := 1 ns; + ROCEV2_EN_G : boolean := false; SIMULATION_G : boolean := false); port ( -- Clock and Reset - axilClk : in sl; - axilRst : in sl; + axilClk : in sl; + axilRst : in sl; -- AXI-Stream Interface - ibRudpMaster : out AxiStreamMasterType; - ibRudpSlave : in AxiStreamSlaveType; - obRudpMaster : in AxiStreamMasterType; - obRudpSlave : out AxiStreamSlaveType; + ibRudpMaster : out AxiStreamMasterType; + ibRudpSlave : in AxiStreamSlaveType; + obRudpMaster : in AxiStreamMasterType; + obRudpSlave : out AxiStreamSlaveType; + -- RoCEv2 Work Request/Completion Interface + workReqMaster : out RoceWorkReqMasterType; + workReqSlave : in RoceWorkReqSlaveType := ROCE_WORK_REQ_SLAVE_INIT_C; + workCompMaster : in RoceWorkCompMasterType := ROCE_WORK_COMP_MASTER_INIT_C; + workCompSlave : out RoceWorkCompSlaveType; + -- RoCEv2 DMA Interface + dmaReadRespMaster : out RoceDmaReadRespMasterType; + dmaReadRespSlave : in RoceDmaReadRespSlaveType := ROCE_DMA_READ_RESP_SLAVE_INIT_C; + dmaReadReqMaster : in RoceDmaReadReqMasterType := ROCE_DMA_READ_REQ_MASTER_INIT_C; + dmaReadReqSlave : out RoceDmaReadReqSlaveType; -- AXI-Lite Interface - axilReadMaster : in AxiLiteReadMasterType; - axilReadSlave : out AxiLiteReadSlaveType; - axilWriteMaster : in AxiLiteWriteMasterType; - axilWriteSlave : out AxiLiteWriteSlaveType); + axilReadMaster : in AxiLiteReadMasterType; + axilReadSlave : out AxiLiteReadSlaveType; + axilWriteMaster : in AxiLiteWriteMasterType; + axilWriteSlave : out AxiLiteWriteSlaveType); end App; architecture mapping of App is - constant TX_INDEX_C : natural := 0; - constant MEM_INDEX_C : natural := 1; + constant TX_INDEX_C : natural := 0; + constant MEM_INDEX_C : natural := 1; + constant ROCE_DISPATCHER_INDEX_C : natural := 2; + constant ROCE_CHECKER_INDEX_C : natural := 3; - constant NUM_AXIL_MASTERS_C : positive := 2; + constant NUM_AXIL_MASTERS_C : positive := 4; constant XBAR_CONFIG_C : AxiLiteCrossbarMasterConfigArray(NUM_AXIL_MASTERS_C-1 downto 0) := genAxiLiteConfig(NUM_AXIL_MASTERS_C, x"8000_0000", 20, 16); @@ -54,6 +68,8 @@ architecture mapping of App is signal axilReadMasters : AxiLiteReadMasterArray(NUM_AXIL_MASTERS_C-1 downto 0); signal axilReadSlaves : AxiLiteReadSlaveArray(NUM_AXIL_MASTERS_C-1 downto 0) := (others => AXI_LITE_READ_SLAVE_EMPTY_SLVERR_C); + signal startingDispatch : sl; + begin ------------------------------- @@ -121,4 +137,56 @@ begin axiWriteMaster => axilWriteMasters(MEM_INDEX_C), axiWriteSlave => axilWriteSlaves(MEM_INDEX_C)); + GEN_ROCEV2_APP_LOGIC : if ROCEV2_EN_G generate + + -------------------------------- + -- DmaTestPatternServer + -------------------------------- + U_DmaTestPatternServer : entity work.DmaTestPatternServer + generic map ( + TPD_G => TPD_G) + port map ( + roceClk => axilClk, + roceRst => axilRst, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave); + + -------------------------------- + -- WorkReqDispatcher + -------------------------------- + U_WorkReqDispatcher : entity work.WorkReqDispatcher + generic map ( + TPD_G => TPD_G) + port map ( + roceClk => axilClk, + roceRst => axilRst, + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + startingDispatch => startingDispatch, + axilReadMaster => axilReadMasters(ROCE_DISPATCHER_INDEX_C), + axilReadSlave => axilReadSlaves(ROCE_DISPATCHER_INDEX_C), + axilWriteMaster => axilWriteMasters(ROCE_DISPATCHER_INDEX_C), + axilWriteSlave => axilWriteSlaves(ROCE_DISPATCHER_INDEX_C)); + + -------------------------------- + -- WorkCompChecker + -------------------------------- + U_WorkCompChecker : entity work.WorkCompChecker + generic map ( + TPD_G => TPD_G) + port map ( + roceClk => axilClk, + roceRst => axilRst, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + startingDispatch => startingDispatch, + axilReadMaster => axilReadMasters(ROCE_CHECKER_INDEX_C), + axilReadSlave => axilReadSlaves(ROCE_CHECKER_INDEX_C), + axilWriteMaster => axilWriteMasters(ROCE_CHECKER_INDEX_C), + axilWriteSlave => axilWriteSlaves(ROCE_CHECKER_INDEX_C)); + + end generate GEN_ROCEV2_APP_LOGIC; + end mapping; diff --git a/firmware/shared/rtl/Core.vhd b/firmware/shared/rtl/Core.vhd index 06fcafc..f547a70 100755 --- a/firmware/shared/rtl/Core.vhd +++ b/firmware/shared/rtl/Core.vhd @@ -22,6 +22,7 @@ use surf.AxiLitePkg.all; use surf.RssiPkg.all; use surf.I2cPkg.all; use surf.I2cMuxPkg.all; +use surf.RoCEv2Pkg.all; library work; use work.CorePkg.all; @@ -31,66 +32,77 @@ use unisim.vcomponents.all; entity Core is generic ( - TPD_G : time := 1 ns; + TPD_G : time := 1 ns; BUILD_INFO_G : BuildInfoType; + ROCEV2_EN_G : boolean := false; + DCQCN_EN_G : boolean := false; SIMULATION_G : boolean; ETH_BUILD_G : BuildEthType; IP_ADDR_G : slv(31 downto 0); DHCP_G : boolean); port ( -- Clock and Reset - axilClk : out sl; - axilRst : out sl; + axilClk : out sl; + axilRst : out sl; -- AXI-Stream Interface - ibRudpMaster : in AxiStreamMasterType; - ibRudpSlave : out AxiStreamSlaveType; - obRudpMaster : out AxiStreamMasterType; - obRudpSlave : in AxiStreamSlaveType; + ibRudpMaster : in AxiStreamMasterType; + ibRudpSlave : out AxiStreamSlaveType; + obRudpMaster : out AxiStreamMasterType; + obRudpSlave : in AxiStreamSlaveType; -- AXI-Lite Interface - axilReadMaster : out AxiLiteReadMasterType; - axilReadSlave : in AxiLiteReadSlaveType; - axilWriteMaster : out AxiLiteWriteMasterType; - axilWriteSlave : in AxiLiteWriteSlaveType; + axilReadMaster : out AxiLiteReadMasterType; + axilReadSlave : in AxiLiteReadSlaveType; + axilWriteMaster : out AxiLiteWriteMasterType; + axilWriteSlave : in AxiLiteWriteSlaveType; + -- RoCE engine Interface + workReqMaster : in RoceWorkReqMasterType := ROCE_WORK_REQ_MASTER_INIT_C; + workReqSlave : out RoceWorkReqSlaveType; + workCompMaster : out RoceWorkCompMasterType; + workCompSlave : in RoceWorkCompSlaveType := ROCE_WORK_COMP_SLAVE_INIT_C; + dmaReadRespMaster : in RoceDmaReadRespMasterType := ROCE_DMA_READ_RESP_MASTER_INIT_C; + dmaReadRespSlave : out RoceDmaReadRespSlaveType; + dmaReadReqMaster : out RoceDmaReadReqMasterType; + dmaReadReqSlave : in RoceDmaReadReqSlaveType := ROCE_DMA_READ_REQ_SLAVE_INIT_C; -- I2C Ports - sfpTxDisL : out sl; - i2cRstL : out sl; - i2cScl : inout sl; - i2cSda : inout sl; + sfpTxDisL : out sl; + i2cRstL : out sl; + i2cScl : inout sl; + i2cSda : inout sl; -- SYSMON Ports - vPIn : in sl; - vNIn : in sl; + vPIn : in sl; + vNIn : in sl; -- System Ports - extRst : in sl; - sysClk300P : in sl; - sysClk300N : in sl; - emcClk : in sl; - heartbeat : out sl; - phyReady : out sl; - rssiLinkUp : out slv(1 downto 0); + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; + emcClk : in sl; + heartbeat : out sl; + phyReady : out sl; + rssiLinkUp : out slv(1 downto 0); -- Boot Memory Ports - flashCsL : out sl; - flashMosi : out sl; - flashMiso : in sl; - flashHoldL : out sl; - flashWp : out sl; + flashCsL : out sl; + flashMosi : out sl; + flashMiso : in sl; + flashHoldL : out sl; + flashWp : out sl; -- SFP ETH Ports - ethClkP : in sl; - ethClkN : in sl; - ethRxP : in sl; - ethRxN : in sl; - ethTxP : out sl; - ethTxN : out sl; + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; -- RJ45 ETH Ports - phyClkP : in sl; - phyClkN : in sl; - phyRxP : in sl; - phyRxN : in sl; - phyTxP : out sl; - phyTxN : out sl; - phyMdc : out sl; - phyMdio : inout sl; - phyRstN : out sl; - phyIrqN : in sl); + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); end Core; architecture mapping of Core is @@ -199,51 +211,62 @@ begin ETH_BUILD_G => ETH_BUILD_G, IP_ADDR_G => IP_ADDR_G, DHCP_G => DHCP_G, + ROCEV2_EN_G => ROCEV2_EN_G, + DCQCN_EN_G => DCQCN_EN_G, AXIL_BASE_ADDR_G => XBAR_CONFIG_C(ETH_INDEX_C).baseAddr) port map ( -- System Ports - extRst => extRst, - sysClk300P => sysClk300P, - sysClk300N => sysClk300N, + extRst => extRst, + sysClk300P => sysClk300P, + sysClk300N => sysClk300N, -- Ethernet Status - phyReady => phyReady, - rssiLinkUp => rssiLinkUp, + phyReady => phyReady, + rssiLinkUp => rssiLinkUp, -- Clock and Reset - axilClk => clk, - axilRst => rst, + axilClk => clk, + axilRst => rst, -- AXI-Stream Interface - ibRudpMaster => ibRudpMaster, - ibRudpSlave => ibRudpSlave, - obRudpMaster => obRudpMaster, - obRudpSlave => obRudpSlave, + ibRudpMaster => ibRudpMaster, + ibRudpSlave => ibRudpSlave, + obRudpMaster => obRudpMaster, + obRudpSlave => obRudpSlave, -- Master AXI-Lite Interface - mAxilReadMaster => mAxilReadMaster, - mAxilReadSlave => mAxilReadSlave, - mAxilWriteMaster => mAxilWriteMaster, - mAxilWriteSlave => mAxilWriteSlave, + mAxilReadMaster => mAxilReadMaster, + mAxilReadSlave => mAxilReadSlave, + mAxilWriteMaster => mAxilWriteMaster, + mAxilWriteSlave => mAxilWriteSlave, -- Slave AXI-Lite Interfaces - sAxilReadMaster => axilReadMasters(ETH_INDEX_C), - sAxilReadSlave => axilReadSlaves(ETH_INDEX_C), - sAxilWriteMaster => axilWriteMasters(ETH_INDEX_C), - sAxilWriteSlave => axilWriteSlaves(ETH_INDEX_C), + sAxilReadMaster => axilReadMasters(ETH_INDEX_C), + sAxilReadSlave => axilReadSlaves(ETH_INDEX_C), + sAxilWriteMaster => axilWriteMasters(ETH_INDEX_C), + sAxilWriteSlave => axilWriteSlaves(ETH_INDEX_C), + -- RoCE engine Interface + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave, -- SFP ETH Ports - ethClkP => ethClkP, - ethClkN => ethClkN, - ethRxP => ethRxP, - ethRxN => ethRxN, - ethTxP => ethTxP, - ethTxN => ethTxN, + ethClkP => ethClkP, + ethClkN => ethClkN, + ethRxP => ethRxP, + ethRxN => ethRxN, + ethTxP => ethTxP, + ethTxN => ethTxN, -- RJ45 ETH Ports - phyClkP => phyClkP, - phyClkN => phyClkN, - phyRxP => phyRxP, - phyRxN => phyRxN, - phyTxP => phyTxP, - phyTxN => phyTxN, - phyMdc => phyMdc, - phyMdio => phyMdio, - phyRstN => phyRstN, - phyIrqN => phyIrqN); + phyClkP => phyClkP, + phyClkN => phyClkN, + phyRxP => phyRxP, + phyRxN => phyRxN, + phyTxP => phyTxP, + phyTxN => phyTxN, + phyMdc => phyMdc, + phyMdio => phyMdio, + phyRstN => phyRstN, + phyIrqN => phyIrqN); end generate; diff --git a/firmware/shared/rtl/DmaTestPatternServer.vhd b/firmware/shared/rtl/DmaTestPatternServer.vhd new file mode 100755 index 0000000..356ff7c --- /dev/null +++ b/firmware/shared/rtl/DmaTestPatternServer.vhd @@ -0,0 +1,234 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: Dma server to inject a test pattern +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; +use ieee.numeric_std.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.SsiPkg.all; +use surf.RoCEv2Pkg.all; + +entity DmaTestPatternServer is + generic ( + TPD_G : time := 1 ns; + RST_ASYNC_G : boolean := false); + port ( + roceClk : in sl; + roceRst : in sl; + dmaReadReqMaster : in RoceDmaReadReqMasterType; + dmaReadReqSlave : out RoceDmaReadReqSlaveType; + dmaReadRespMaster : out RoceDmaReadRespMasterType; + dmaReadRespSlave : in RoceDmaReadRespSlaveType); +end entity DmaTestPatternServer; + +architecture rtl of DmaTestPatternServer is + + type RegState is (st0_idle, st1_send_pkg); + + type RegType is record + state : RegState; + reqLatched : RoceDmaReadReqMasterType; + -- Global byte counter — persists across requests for continuous pattern + globalByteCounter : unsigned(7 downto 0); + -- Running beat offset within current request + byteAddr : unsigned(63 downto 0); + first : boolean; + txRespMaster : RoceDmaReadRespMasterType; + rxReqSlave : RoceDmaReadReqSlaveType; + end record RegType; + + constant REG_INIT_C : RegType := ( + state => st0_idle, + reqLatched => ROCE_DMA_READ_REQ_MASTER_INIT_C, + globalByteCounter => (others => '0'), + byteAddr => (others => '0'), + first => true, + txRespMaster => ROCE_DMA_READ_RESP_MASTER_INIT_C, + rxReqSlave => ROCE_DMA_READ_REQ_SLAVE_INIT_C + ); + + signal r : RegType := REG_INIT_C; + signal rin : RegType; + + constant AXI_STREAM_CONFIG_C : AxiStreamConfigType := ( + TSTRB_EN_C => false, + TDATA_BYTES_C => 38, + TDEST_BITS_C => 0, + TID_BITS_C => 0, + TKEEP_MODE_C => TKEEP_NORMAL_C, + TUSER_BITS_C => 0, + TUSER_MODE_C => TUSER_NORMAL_C + ); + + signal s_sAxisSlave : AxiStreamSlaveType; + signal s_mAxisMaster : AxiStreamMasterType; + signal s_dmaReadReqSlave : RoceDmaReadReqSlaveType; + signal s_dmaReadReqMaster : RoceDmaReadReqMasterType; + +begin + + ----------------------------------------------------------------------------- + -- FIFO of requests + ----------------------------------------------------------------------------- + AxiStreamFifoV2_1 : entity surf.AxiStreamFifoV2 + generic map ( + TPD_G => TPD_G, + GEN_SYNC_FIFO_G => true, + FIFO_ADDR_WIDTH_G => 4, + SLAVE_AXI_CONFIG_G => AXI_STREAM_CONFIG_C, + MASTER_AXI_CONFIG_G => AXI_STREAM_CONFIG_C) + port map ( + sAxisClk => RoceClk, + sAxisRst => RoceRst, + sAxisMaster => DmaReadReqToAxiStreamMaster(dmaReadReqMaster), + sAxisSlave => s_sAxisSlave, + mAxisClk => RoceClk, + mAxisRst => RoceRst, + mAxisMaster => s_mAxisMaster, + mAxisSlave => DmaReadReqToAxiStreamSlave(s_dmaReadReqSlave) + ); + + dmaReadReqSlave <= AxiStreamToDmaReadReqSlave(s_sAxisSlave); + s_dmaReadReqMaster <= AxiStreamToDmaReadReqMaster(s_mAxisMaster); + + ----------------------------------------------------------------------------- + -- Combinatorial process + ----------------------------------------------------------------------------- + comb : process (dmaReadRespSlave, r, s_dmaReadReqMaster) is + variable v : RegType; + variable isFirst : sl; + variable isLast : sl; + variable byteEn : slv(AXI_STREAM_MAX_TKEEP_WIDTH_C-1 downto 0); + variable dataStream : slv(255 downto 0); + variable len : natural; + variable beatBytes : natural; + begin + v := r; + + v.rxReqSlave.ready := '0'; + + if dmaReadRespSlave.ready = '1' then + v.txRespMaster.valid := '0'; + end if; + + case r.state is + + ----------------------------------------------------------------------- + when st0_idle => + if s_dmaReadReqMaster.valid = '1' then + v.rxReqSlave.ready := '1'; + v.reqLatched := s_dmaReadReqMaster; + -- Do NOT seed byteAddr from startAddr — use globalByteCounter + -- so the pattern is continuous across frames + v.byteAddr := (others => '0'); + v.state := st1_send_pkg; + end if; + + ----------------------------------------------------------------------- + when st1_send_pkg => + if v.txRespMaster.valid = '0' then + + -- isFirst flag + isFirst := '0'; + if r.first then + isFirst := '1'; + v.first := false; + end if; + + -- How many valid bytes in this beat? + len := to_integer(unsigned(r.reqLatched.len)); + if len < 33 then + beatBytes := len; + isLast := '1'; + else + beatBytes := 32; + isLast := '0'; + end if; + + -- Fill dataStream using globalByteCounter + beat offset. + -- byte i gets value (globalByteCounter + byteAddr + i) mod 256. + for i in 0 to 31 loop + dataStream((31-i)*8 + 7 downto (31-i)*8) := + std_logic_vector( + (r.globalByteCounter + + resize(r.byteAddr, 8) + + to_unsigned(i, 8)) + ); + end loop; + + -- byteEnable + byteEn := (others => '0'); + for i in 0 to 31 loop + if i < beatBytes then + byteEn(i) := '1'; + end if; + end loop; + + v.txRespMaster.dataStream := dataStream & byteEn(31 downto 0) & isFirst & isLast; + v.txRespMaster.initiator := r.reqLatched.initiator; + v.txRespMaster.sQpn := r.reqLatched.sQpn; + v.txRespMaster.wrId := r.reqLatched.wrId; + v.txRespMaster.isRespErr := '0'; + v.txRespMaster.valid := '1'; + + if isLast = '0' then + -- Advance beat offset and remaining length + v.byteAddr := r.byteAddr + 32; + v.reqLatched.len := std_logic_vector( + unsigned(r.reqLatched.len) - 32 + ); + v.state := st1_send_pkg; + else + -- Advance global counter by the full payload length of this request + v.globalByteCounter := r.globalByteCounter + + resize(unsigned(r.reqLatched.len) + + resize(r.byteAddr, 13), 8); + v.state := st0_idle; + v.first := true; + end if; + + end if; + + ----------------------------------------------------------------------- + when others => + v := REG_INIT_C; + + end case; + + s_dmaReadReqSlave <= v.rxReqSlave; + dmaReadRespMaster <= r.txRespMaster; + rin <= v; + + end process comb; + + ----------------------------------------------------------------------------- + -- Sequential process + ----------------------------------------------------------------------------- + seq : process (RoceClk, RoceRst) is + begin + if (RST_ASYNC_G) and (RoceRst = '1') then + r <= REG_INIT_C after TPD_G; + elsif rising_edge(RoceClk) then + if (RST_ASYNC_G = false and RoceRst = '1') then + r <= REG_INIT_C after TPD_G; + else + r <= rin after TPD_G; + end if; + end if; + end process seq; + +end architecture rtl; diff --git a/firmware/shared/rtl/Rudp.vhd b/firmware/shared/rtl/Rudp.vhd index e5e0783..f67cb6e 100755 --- a/firmware/shared/rtl/Rudp.vhd +++ b/firmware/shared/rtl/Rudp.vhd @@ -21,6 +21,7 @@ use surf.AxiStreamPkg.all; use surf.AxiLitePkg.all; use surf.EthMacPkg.all; use surf.RssiPkg.all; +use surf.RoCEv2Pkg.all; library work; use work.CorePkg.all; @@ -30,55 +31,66 @@ use unisim.vcomponents.all; entity Rudp is generic ( - TPD_G : time := 1 ns; + TPD_G : time := 1 ns; ETH_BUILD_G : BuildEthType; IP_ADDR_G : slv(31 downto 0); DHCP_G : boolean; + ROCEV2_EN_G : boolean := false; + DCQCN_EN_G : boolean := false; AXIL_BASE_ADDR_G : slv(31 downto 0)); port ( -- System Ports - extRst : in sl; - sysClk300P : in sl; - sysClk300N : in sl; + extRst : in sl; + sysClk300P : in sl; + sysClk300N : in sl; -- Ethernet Status - phyReady : out sl; - rssiLinkUp : out slv(1 downto 0); + phyReady : out sl; + rssiLinkUp : out slv(1 downto 0); -- Clock and Reset - axilClk : out sl; - axilRst : out sl; + axilClk : out sl; + axilRst : out sl; -- AXI-Stream Interface - ibRudpMaster : in AxiStreamMasterType; - ibRudpSlave : out AxiStreamSlaveType; - obRudpMaster : out AxiStreamMasterType; - obRudpSlave : in AxiStreamSlaveType; + ibRudpMaster : in AxiStreamMasterType; + ibRudpSlave : out AxiStreamSlaveType; + obRudpMaster : out AxiStreamMasterType; + obRudpSlave : in AxiStreamSlaveType; -- Master AXI-Lite Interface - mAxilReadMaster : out AxiLiteReadMasterType; - mAxilReadSlave : in AxiLiteReadSlaveType; - mAxilWriteMaster : out AxiLiteWriteMasterType; - mAxilWriteSlave : in AxiLiteWriteSlaveType; + mAxilReadMaster : out AxiLiteReadMasterType; + mAxilReadSlave : in AxiLiteReadSlaveType; + mAxilWriteMaster : out AxiLiteWriteMasterType; + mAxilWriteSlave : in AxiLiteWriteSlaveType; -- Slave AXI-Lite Interfaces - sAxilReadMaster : in AxiLiteReadMasterType; - sAxilReadSlave : out AxiLiteReadSlaveType; - sAxilWriteMaster : in AxiLiteWriteMasterType; - sAxilWriteSlave : out AxiLiteWriteSlaveType; + sAxilReadMaster : in AxiLiteReadMasterType; + sAxilReadSlave : out AxiLiteReadSlaveType; + sAxilWriteMaster : in AxiLiteWriteMasterType; + sAxilWriteSlave : out AxiLiteWriteSlaveType; + -- RoCE engine Interface + workReqMaster : in RoceWorkReqMasterType := ROCE_WORK_REQ_MASTER_INIT_C; + workReqSlave : out RoceWorkReqSlaveType; + workCompMaster : out RoceWorkCompMasterType; + workCompSlave : in RoceWorkCompSlaveType := ROCE_WORK_COMP_SLAVE_INIT_C; + dmaReadRespMaster : in RoceDmaReadRespMasterType := ROCE_DMA_READ_RESP_MASTER_INIT_C; + dmaReadRespSlave : out RoceDmaReadRespSlaveType; + dmaReadReqMaster : out RoceDmaReadReqMasterType; + dmaReadReqSlave : in RoceDmaReadReqSlaveType := ROCE_DMA_READ_REQ_SLAVE_INIT_C; -- SFP ETH Ports - ethClkP : in sl; - ethClkN : in sl; - ethRxP : in sl; - ethRxN : in sl; - ethTxP : out sl; - ethTxN : out sl; + ethClkP : in sl; + ethClkN : in sl; + ethRxP : in sl; + ethRxN : in sl; + ethTxP : out sl; + ethTxN : out sl; -- RJ45 ETH Ports - phyClkP : in sl; - phyClkN : in sl; - phyRxP : in sl; - phyRxN : in sl; - phyTxP : out sl; - phyTxN : out sl; - phyMdc : out sl; - phyMdio : inout sl; - phyRstN : out sl; - phyIrqN : in sl); + phyClkP : in sl; + phyClkN : in sl; + phyRxP : in sl; + phyRxN : in sl; + phyTxP : out sl; + phyTxN : out sl; + phyMdc : out sl; + phyMdio : inout sl; + phyRstN : out sl; + phyIrqN : in sl); end Rudp; architecture mapping of Rudp is @@ -87,8 +99,9 @@ architecture mapping of Rudp is constant UDP_INDEX_C : natural := 1; constant RSSI_INDEX_C : natural := 2; -- 2:3 constant AXIS_MON_INDEX_C : natural := 4; + constant ROCE_INDEX_C : natural := 5; - constant NUM_AXIL_MASTERS_C : positive := 5; + constant NUM_AXIL_MASTERS_C : positive := 6; constant XBAR_CONFIG_C : AxiLiteCrossbarMasterConfigArray(NUM_AXIL_MASTERS_C-1 downto 0) := genAxiLiteConfig(NUM_AXIL_MASTERS_C, AXIL_BASE_ADDR_G, 20, 16); @@ -99,7 +112,7 @@ architecture mapping of Rudp is constant CLK_FREQUENCY_C : real := ite((ETH_BUILD_G = SFP_10G_C), 156.25E+6, 125.0E+6); - -- UDP constants + -- UDP Server constants constant UDP_SRV_SRP_IDX_C : natural := 0; constant UDP_SRV_DATA_IDX_C : natural := 1; constant UDP_SRV_XVC_IDX_C : natural := 2; @@ -109,6 +122,12 @@ architecture mapping of Rudp is UDP_SRV_DATA_IDX_C => 8193, -- Streaming data UDP_SRV_XVC_IDX_C => 2542); -- Xilinx XVC + -- UDP Client constants + constant UDP_CLT_ROCE_IDX_C : natural := 0; + constant CLIENT_SIZE_C : positive := 1; + constant CLIENT_PORTS_C : PositiveArray(CLIENT_SIZE_C-1 downto 0) := ( + UDP_CLT_ROCE_IDX_C => 4791); -- RoCEv2 + -- RSSI constants constant RSSI_SIZE_C : positive := 1; -- Implementing only 1 VC per RSSI link constant AXIS_CONFIG_C : AxiStreamConfigArray(RSSI_SIZE_C-1 downto 0) := ( @@ -119,10 +138,15 @@ architecture mapping of Rudp is signal obMacMaster : AxiStreamMasterType; signal obMacSlave : AxiStreamSlaveType; - signal obServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); - signal obServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0); - signal ibServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); - signal ibServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0); + signal obServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal obServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + signal ibServerMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal ibServerSlaves : AxiStreamSlaveArray(SERVER_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + + signal obClientMasters : AxiStreamMasterArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal obClientSlaves : AxiStreamSlaveArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); + signal ibClientMasters : AxiStreamMasterArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_MASTER_INIT_C); + signal ibClientSlaves : AxiStreamSlaveArray(CLIENT_SIZE_C-1 downto 0) := (others => AXI_STREAM_SLAVE_FORCE_C); -- One RSSI per UDP port (which is why SERVER_SIZE_C used instead of SERVER_SIZE_C) signal rssiIbMasters : AxiStreamMasterArray(SERVER_SIZE_C-1 downto 0); @@ -223,6 +247,7 @@ begin TPD_G => TPD_G, NUM_LANE_G => 1, PAUSE_EN_G => true, -- Enable ETH pause + ROCEV2_EN_G => ROCEV2_EN_G, -- Enable RoCEv2 EN_AXI_REG_G => true) -- Enable diagnostic AXI-Lite interface port map ( -- Local Configurations @@ -273,6 +298,7 @@ begin TPD_G => TPD_G, NUM_LANE_G => 1, PAUSE_EN_G => true, -- Enable ETH pause + ROCEV2_EN_G => ROCEV2_EN_G, EN_AXI_REG_G => true, -- Enable diagnostic AXI-Lite interface -- QUAD PLL Configurations USE_GTREFCLK_G => false, @@ -356,6 +382,7 @@ begin generic map ( TPD_G => TPD_G, STABLE_CLK_FREQ_G => 300.0E+6, + ROCEV2_EN_G => ROCEV2_EN_G, PAUSE_EN_G => false, EN_AXIL_REG_G => true, AXIS_CONFIG_G => EMAC_AXIS_CONFIG_C) @@ -402,17 +429,22 @@ begin U_UDP : entity surf.UdpEngineWrapper generic map ( -- Simulation Generics - TPD_G => TPD_G, + TPD_G => TPD_G, -- UDP Server Generics - SERVER_EN_G => true, -- UDP Server only - SERVER_SIZE_G => SERVER_SIZE_C, - SERVER_PORTS_G => SERVER_PORTS_C, + SERVER_EN_G => true, -- UDP Server only + SERVER_SIZE_G => SERVER_SIZE_C, + SERVER_PORTS_G => SERVER_PORTS_C, -- UDP Client Generics - CLIENT_EN_G => false, -- UDP Server only + CLIENT_EN_G => ROCEV2_EN_G, -- UDP Server only if not RoCE + CLIENT_SIZE_G => CLIENT_SIZE_C, + CLIENT_PORTS_G => CLIENT_PORTS_C, + CLIENT_EXT_CONFIG_G => false, -- General IPv4/ARP/DHCP Generics - DHCP_G => DHCP_G, - CLK_FREQ_G => CLK_FREQUENCY_C, - COMM_TIMEOUT_G => 10) -- Timeout used for ARP and DHCP + DHCP_G => DHCP_G, + CLK_FREQ_G => CLK_FREQUENCY_C, + DSCP_G => 26, + ECN_G => "10", + COMM_TIMEOUT_G => 10) -- Timeout used for ARP and DHCP port map ( -- Local Configurations localMac => localMac, @@ -427,6 +459,10 @@ begin obServerSlaves => obServerSlaves, ibServerMasters => ibServerMasters, ibServerSlaves => ibServerSlaves, + obClientMasters => obClientMasters, + obClientSlaves => obClientSlaves, + ibClientMasters => ibClientMasters, + ibClientSlaves => ibClientSlaves, -- AXI-Lite Interface axilReadMaster => axilReadMasters(UDP_INDEX_C), axilReadSlave => axilReadSlaves(UDP_INDEX_C), @@ -536,6 +572,41 @@ begin obRudpMaster <= rssiObMasters(1); rssiObSlaves(1) <= obRudpSlave; + --------------------------------------------------------------- + -- RoCEv2 Engine + --------------------------------------------------------------- + GEN_ROCE_ENGINE : if ROCEV2_EN_G generate + U_RoceEngineWrapper : entity surf.RoCEv2Engine + generic map ( + TPD_G => TPD_G, + EXT_ROCE_CONFIG_G => false, + DCQCN_EN_G => DCQCN_EN_G, + AXIL_BASE_ADDR_G => XBAR_CONFIG_C(ROCE_INDEX_C).baseAddr) + port map ( + clk => ethClk, + rst => ethRst, + -- Work Requests and Comps + workReqMaster => workReqMaster, + workReqSlave => workReqSlave, + workCompMaster => workCompMaster, + workCompSlave => workCompSlave, + -- Interface to UDP Engine + obUdpMaster => obClientMasters(UDP_CLT_ROCE_IDX_C), + obUdpSlave => obClientSlaves(UDP_CLT_ROCE_IDX_C), + ibUdpMaster => ibClientMasters(UDP_CLT_ROCE_IDX_C), + ibUdpSlave => ibClientSlaves(UDP_CLT_ROCE_IDX_C), + -- Axi-Lite interface + axilReadMaster => axilReadMasters(ROCE_INDEX_C), + axilReadSlave => axilReadSlaves(ROCE_INDEX_C), + axilWriteMaster => axilWriteMasters(ROCE_INDEX_C), + axilWriteSlave => axilWriteSlaves(ROCE_INDEX_C), + -- DMA Interface + dmaReadRespMaster => dmaReadRespMaster, + dmaReadRespSlave => dmaReadRespSlave, + dmaReadReqMaster => dmaReadReqMaster, + dmaReadReqSlave => dmaReadReqSlave); + end generate GEN_ROCE_ENGINE; + ------------------------ -- AXI Stream Monitoring ------------------------ diff --git a/firmware/shared/rtl/WorkCompChecker.vhd b/firmware/shared/rtl/WorkCompChecker.vhd new file mode 100755 index 0000000..29fe082 --- /dev/null +++ b/firmware/shared/rtl/WorkCompChecker.vhd @@ -0,0 +1,189 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: RoCEv2 work completion checker +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; +use ieee.std_logic_unsigned.all; +use ieee.std_logic_arith.all; +use ieee.std_logic_misc.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiLitePkg.all; +use surf.RoCEv2Pkg.all; + +entity WorkCompChecker is + generic ( + TPD_G : time := 1 ns; + RST_ASYNC_G : boolean := false; + DISPATCH_COUNTER_BITS_G : natural range 0 to 31 := 24); + port ( + roceClk : in sl; + roceRst : in sl; + workCompMaster : in RoceWorkCompMasterType; + workCompSlave : out RoceWorkCompSlaveType; + -- Starting flag + startingDispatch : in sl; + -- Axi-Lite Registers + axilReadMaster : in AxiLiteReadMasterType := AXI_LITE_READ_MASTER_INIT_C; + axilReadSlave : out AxiLiteReadSlaveType; + axilWriteMaster : in AxiLiteWriteMasterType := AXI_LITE_WRITE_MASTER_INIT_C; + axilWriteSlave : out AxiLiteWriteSlaveType); +end entity WorkCompChecker; + +architecture rtl of WorkCompChecker is + + type AxilRegType is record + resetCounters : sl; + axilReadSlave : AxiLiteReadSlaveType; + axilWriteSlave : AxiLiteWriteSlaveType; + end record AxilRegType; + + constant AXIL_REG_INIT_C : AxilRegType := ( + resetCounters => '0', + axilReadSlave => AXI_LITE_READ_SLAVE_INIT_C, + axilWriteSlave => AXI_LITE_WRITE_SLAVE_INIT_C + ); + + signal regR : AxilRegType := AXIL_REG_INIT_C; + signal regRin : AxilRegType; + + type RecStateType is (st0_idle, st1_received); + + type RecRegType is record + state : RecStateType; + successCounter : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + unsuccessCounter : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + status : slv(4 downto 0); + rxSlave : RoceWorkCompSlaveType; + end record RecRegType; + + constant REC_REG_INIT_C : RecRegType := ( + state => st0_idle, + successCounter => (others => '0'), + unsuccessCounter => (others => '0'), + status => (others => '0'), + rxSlave => ROCE_WORK_COMP_SLAVE_INIT_C + ); + + signal recR : RecRegType := REC_REG_INIT_C; + signal recRin : RecRegType; + +begin -- architecture rtl + + ----------------------------------------------------------------------------- + -- Axi-Lite Regs + ----------------------------------------------------------------------------- + regComb : process (axilReadMaster, axilWriteMaster, recR, regR) is + variable v : AxilRegType; + variable regCon : AxiLiteEndPointType; + begin -- process regComb + + v := regR; + + -- Determine the transaction type + axiSlaveWaitTxn(regCon, axilWriteMaster, axilReadMaster, v.axilWriteSlave, v.axilReadSlave); + + -- Gen registers + axiSlaveRegisterR(regCon, x"F00", 0, recR.successCounter); + axiSlaveRegisterR(regCon, x"F04", 0, recR.unsuccessCounter); + axiSlaveRegister(regCon, x"F08", 0, v.resetCounters); + + -- Closeout the transaction + axiSlaveDefault(regCon, v.axilWriteSlave, v.axilReadSlave, AXI_RESP_DECERR_C); + + -- Outputs + axilWriteSlave <= regR.axilWriteSlave; + axilReadSlave <= regR.axilReadSlave; + + -- Register update + regRin <= v; + + end process regComb; + + regSeq : process (RoceClk, RoceRst) is + begin + if (RST_ASYNC_G) and (RoceRst = '1') then + regR <= AXIL_REG_INIT_C after TPD_G; + elsif (rising_edge(RoceClk)) then + if (RST_ASYNC_G = false) and (RoceRst = '1') then + regR <= AXIL_REG_INIT_C after TPD_G; + else + regR <= regRin after TPD_G; + end if; + end if; + end process regSeq; + + ----------------------------------------------------------------------------- + -- Checker + ----------------------------------------------------------------------------- + recComb : process (WorkCompMaster, recR, regR) is + variable v : recRegType; + begin -- process recComb + + -- Latch the current value + v := recR; + + -- init ready + v.rxSlave.ready := '0'; + + -- Reset counters + if regR.resetCounters = '1' then + v.successCounter := (others => '0'); + v.unsuccessCounter := (others => '0'); + end if; + + -- FSM + case recR.state is + ------------------------------------------------------------------------- + when st0_idle => + if WorkCompMaster.valid = '1' then + v.rxSlave.ready := '1'; + v.status := WorkCompMaster.status; + v.state := st1_received; + end if; + ----------------------------------------------------------------------- + when st1_received => + if recR.status = "00000" then + v.successCounter := recR.successCounter + 1; + v.state := st0_idle; + else + v.unsuccessCounter := recR.unsuccessCounter + 1; + v.state := st0_idle; + end if; + ----------------------------------------------------------------------- + when others => + v := REC_REG_INIT_C; + end case; + + WorkCompSlave <= v.rxSlave; + + recRin <= v; + + end process recComb; + + recSeq : process (RoceClk, RoceRst) is + begin + if (RST_ASYNC_G) and (RoceRst = '1') then + recR <= REC_REG_INIT_C after TPD_G; + elsif (rising_edge(RoceClk)) then + if (RST_ASYNC_G = false) and (RoceRst = '1') then + recR <= REC_REG_INIT_C after TPD_G; + else + recR <= recRin after TPD_G; + end if; + end if; + end process recSeq; + +end architecture rtl; diff --git a/firmware/shared/rtl/WorkReqDispatcher.vhd b/firmware/shared/rtl/WorkReqDispatcher.vhd new file mode 100755 index 0000000..e85fc6a --- /dev/null +++ b/firmware/shared/rtl/WorkReqDispatcher.vhd @@ -0,0 +1,259 @@ +------------------------------------------------------------------------------- +-- Company : SLAC National Accelerator Laboratory +------------------------------------------------------------------------------- +-- Description: RoCEv2 dispatcher of work requests to the RoCEv2 engine +------------------------------------------------------------------------------- +-- This file is part of 'Simple-10GbE-RUDP-KCU105-Example'. +-- It is subject to the license terms in the LICENSE.txt file found in the +-- top-level directory of this distribution and at: +-- https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +-- No part of 'Simple-10GbE-RUDP-KCU105-Example', including this file, +-- may be copied, modified, propagated, or distributed except according to +-- the terms contained in the LICENSE.txt file. +------------------------------------------------------------------------------- + +library ieee; +use ieee.std_logic_1164.all; +use ieee.std_logic_unsigned.all; +use ieee.std_logic_arith.all; + +library surf; +use surf.StdRtlPkg.all; +use surf.AxiStreamPkg.all; +use surf.AxiLitePkg.all; +use surf.RoCEv2Pkg.all; + +entity WorkReqDispatcher is + generic ( + TPD_G : time := 1 ns; + RST_ASYNC_G : boolean := false; + DISPATCH_COUNTER_BITS_G : natural range 0 to 31 := 24); + port ( + roceClk : in std_logic; + roceRst : in std_logic; + -- Roce Work Req + workReqMaster : out RoceWorkReqMasterType; + workReqSlave : in RoceWorkReqSlaveType; + -- Starting flag + startingDispatch : out sl; + -- Axi-Lite Registers + axilReadMaster : in AxiLiteReadMasterType := AXI_LITE_READ_MASTER_INIT_C; + axilReadSlave : out AxiLiteReadSlaveType; + axilWriteMaster : in AxiLiteWriteMasterType := AXI_LITE_WRITE_MASTER_INIT_C; + axilWriteSlave : out AxiLiteWriteSlaveType); +end entity WorkReqDispatcher; + +architecture rtl of WorkReqDispatcher is + + type AxilRegType is record + startDispatching : sl; + dispatchCounter : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + len : slv(31 downto 0); + rKey : slv(31 downto 0); + lKey : slv(31 downto 0); + sQpn : slv(23 downto 0); + dQpn : slv(24 downto 0); + rAddr : slv(63 downto 0); + addrWrapCount : slv(31 downto 0); + axilReadSlave : AxiLiteReadSlaveType; + axilWriteSlave : AxiLiteWriteSlaveType; + end record AxilRegType; + + constant AXIL_REG_INIT_C : AxilRegType := ( + startDispatching => '0', + dispatchCounter => (others => '0'), + len => (others => '0'), + rKey => (others => '0'), + lKey => (others => '0'), + sQpn => (others => '0'), + dQpn => (others => '0'), + rAddr => (others => '0'), + addrWrapCount => (others => '0'), + axilReadSlave => AXI_LITE_READ_SLAVE_INIT_C, + axilWriteSlave => AXI_LITE_WRITE_SLAVE_INIT_C + ); + + signal regR : AxilRegType := AXIL_REG_INIT_C; + signal regRin : AxilRegType; + + -- Track previous rAddr to detect when startZmq restarts with a new MR. + -- Initialised to all-ones so the very first write always triggers a reset. + signal prevRAddr : slv(63 downto 0) := (others => '1'); + + type DispStateType is (st0_idle, st1_sending); + + type DispRegType is record + state : DispStateType; + count : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + addrCount : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + txMaster : RoceWorkReqMasterType; + end record DispRegType; + + constant DISP_REG_INIT_C : DispRegType := ( + state => st0_idle, + count => (others => '0'), + addrCount => (others => '0'), + txMaster => ROCE_WORK_REQ_MASTER_INIT_C + ); + + signal dispR : DispRegType := DISP_REG_INIT_C; + signal dispRin : DispRegType; + + signal startDispatching : sl; + +begin -- architecture rtl + + ----------------------------------------------------------------------------- + -- Axi-Lite Regs + ----------------------------------------------------------------------------- + regComb : process (axilReadMaster, axilWriteMaster, regR) is + variable v : AxilRegType; + variable regCon : AxiLiteEndPointType; + begin + + v := regR; + + axiSlaveWaitTxn(regCon, axilWriteMaster, axilReadMaster, v.axilWriteSlave, v.axilReadSlave); + + axiSlaveRegister (regCon, x"F00", 0, v.startDispatching); + axiSlaveRegister (regCon, x"F00", 1, v.dispatchCounter); + axiSlaveRegister (regCon, x"F04", 0, v.len); + axiSlaveRegister (regCon, x"F08", 0, v.rKey); + axiSlaveRegister (regCon, x"F0C", 0, v.lKey); + axiSlaveRegister (regCon, x"F10", 0, v.sQpn); + axiSlaveRegister (regCon, x"F14", 0, v.dQpn); + axiSlaveRegister (regCon, x"F18", 0, v.rAddr); + axiSlaveRegister (regCon, x"F20", 0, v.addrWrapCount); + + axiSlaveDefault(regCon, v.axilWriteSlave, v.axilReadSlave, AXI_RESP_DECERR_C); + + axilWriteSlave <= regR.axilWriteSlave; + axilReadSlave <= regR.axilReadSlave; + + regRin <= v; + + end process regComb; + + regSeq : process (RoceClk, RoceRst) is + begin + if (RST_ASYNC_G) and (RoceRst = '1') then + regR <= AXIL_REG_INIT_C after TPD_G; + prevRAddr <= (others => '1') after TPD_G; + elsif (rising_edge(RoceClk)) then + if (RST_ASYNC_G = false) and (RoceRst = '1') then + regR <= AXIL_REG_INIT_C after TPD_G; + prevRAddr <= (others => '1') after TPD_G; + else + regR <= regRin after TPD_G; + -- Track previous value of rAddr to detect changes in dispComb + prevRAddr <= regR.rAddr after TPD_G; + end if; + end if; + end process regSeq; + + SynchronizerEdge_1 : entity surf.SynchronizerEdge + generic map ( + TPD_G => TPD_G, + BYPASS_SYNC_G => true + ) + port map ( + clk => RoceClk, + dataIn => regR.startDispatching, + risingEdge => startDispatching + ); + + startingDispatch <= startDispatching; + + ----------------------------------------------------------------------------- + -- Dispatcher + ----------------------------------------------------------------------------- + dispComb : process (dispR, prevRAddr, regR, startDispatching, workReqSlave) is + variable v : dispRegType; + variable idPadding : slv(63 downto DISPATCH_COUNTER_BITS_G) := (others => '0'); + variable nextAddr : slv(DISPATCH_COUNTER_BITS_G-1 downto 0); + begin + + v := dispR; + + if workReqSlave.ready = '1' then + v.txMaster.valid := '0'; + end if; + + case dispR.state is + ------------------------------------------------------------------------- + when st0_idle => + -- Reset addrCount when rAddr changes (new MR after startZmq restart). + -- addrCount persists across multiple dispatch bursts within a session. + if regR.rAddr /= prevRAddr then + v.addrCount := (others => '0'); + end if; + if startDispatching = '1' then + v.state := st1_sending; + end if; + ----------------------------------------------------------------------- + when st1_sending => + if v.txMaster.valid = '0' then + v.txMaster.id := idPadding & dispR.count + 1; + v.txMaster.opCode := x"1"; + v.txMaster.flags := "00010"; -- RDMA Write with Immediate + v.txMaster.rAddr := regR.rAddr + (dispR.addrCount * regR.len); + v.txMaster.rKey := regR.rKey; + v.txMaster.len := regR.len; + v.txMaster.lAddr := (others => '0'); + v.txMaster.lKey := regR.lKey; + v.txMaster.sQpn := regR.sQpn; + v.txMaster.solicited := '0'; + v.txMaster.comp := (others => '0'); + v.txMaster.swap := (others => '0'); + -- Set immediate data channel ID to 1 so the rogue StreamWriter routes + -- incoming RDMA frames to channel 1 (dataWriter.getChannel(1)). + -- Bits [7:0] of immDt are decoded by Server.cpp as the rogue stream + -- channel; channel 0 is reserved for the UDP/RSSI stream. + v.txMaster.immDt := conv_std_logic_vector(1, v.txMaster.immDt'length); + v.txMaster.rKeyToInv := (others => '0'); + v.txMaster.srqn := (others => '0'); + v.txMaster.dQpn := (others => '0'); + v.txMaster.qKey := (others => '0'); + v.txMaster.valid := '1'; + + -- Advance addrCount, wrapping at addrWrapCount + nextAddr := dispR.addrCount + 1; + if nextAddr >= regR.addrWrapCount then + v.addrCount := (others => '0'); + else + v.addrCount := nextAddr; + end if; + + -- Advance dispatch counter + if dispR.count < regR.dispatchCounter - 1 then + v.count := v.count + 1; + v.state := st1_sending; + else + v.count := (others => '0'); + v.state := st0_idle; + end if; + end if; + ----------------------------------------------------------------------- + when others => + v := DISP_REG_INIT_C; + end case; + + workReqMaster <= dispR.txMaster; + dispRin <= v; + + end process dispComb; + + dispSeq : process (RoceClk, RoceRst) is + begin + if (RST_ASYNC_G) and (RoceRst = '1') then + dispR <= DISP_REG_INIT_C after TPD_G; + elsif (rising_edge(RoceClk)) then + if (RST_ASYNC_G = false) and (RoceRst = '1') then + dispR <= DISP_REG_INIT_C after TPD_G; + else + dispR <= dispRin after TPD_G; + end if; + end if; + end process dispSeq; + +end architecture rtl; diff --git a/firmware/shared/ruckus.tcl b/firmware/shared/ruckus.tcl index 2a32225..cb33773 100644 --- a/firmware/shared/ruckus.tcl +++ b/firmware/shared/ruckus.tcl @@ -1,6 +1,9 @@ # Load RUCKUS library source $::env(RUCKUS_PROC_TCL) +# Check for version 2023.1 of Vivado (or later) +if { [VersionCheck 2023.1] < 0 } {exit -1} + # Load Source Code loadSource -dir "$::DIR_PATH/rtl" loadIpCore -path "$::DIR_PATH/ip/SystemManagementCore.xci" diff --git a/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl index cb943a4..bde61f5 100644 --- a/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl +++ b/firmware/targets/Simple10GbeRudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared diff --git a/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl b/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl index 73dd252..39c6bb1 100755 --- a/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl +++ b/firmware/targets/Simple1GbeRudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared diff --git a/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl b/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl index 37da0de..80fc595 100755 --- a/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl +++ b/firmware/targets/SimpleRj45RudpKcu105Example/ruckus.tcl @@ -1,9 +1,6 @@ # Load RUCKUS environment and library source $::env(RUCKUS_PROC_TCL) -# Check for version 2023.1 of Vivado (or later) -if { [VersionCheck 2023.1] < 0 } {exit -1} - # Load shared and sub-module ruckus.tcl files loadRuckusTcl $::env(TOP_DIR)/submodules/surf loadRuckusTcl $::env(TOP_DIR)/shared From c0cd991c351e5075b1dfddc0333865c94b15a6f0 Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Sat, 6 Jun 2026 13:42:50 -0700 Subject: [PATCH 3/4] adding the RoCEv2 from https://github.com/FilMarini/Simple-10GbE-RUDP-KCU105-Example/tree/rocev2 --- .../rocev2_10gbe_rudp_kcu105_example/_App.py | 28 ++ .../_RoceChecker.py | 38 +++ .../_RoceDispatcher.py | 83 ++++++ .../rocev2_10gbe_rudp_kcu105_example/_Root.py | 255 ++++++++++++++++++ .../__init__.py | 4 + .../simple_10gbe_rudp_kcu105_example/_Core.py | 14 +- .../hdl/RoCEv2_10GbeRudpKcu105Example.vhd | 2 +- .../hdl/RoCEv2_1GbeRudpKcu105Example.vhd | 2 +- .../hdl/RoCEv2_Rj45RudpKcu105Example.vhd | 2 +- software/scripts/fileReader.py | 205 ++++++++++++-- software/scripts/runDispatch.py | 151 +++++++++++ software/scripts/startZmq.py | 210 +++++++++++++++ 12 files changed, 973 insertions(+), 21 deletions(-) create mode 100644 firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py create mode 100644 firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceChecker.py create mode 100644 firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceDispatcher.py create mode 100644 firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py create mode 100644 firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py create mode 100644 software/scripts/runDispatch.py create mode 100644 software/scripts/startZmq.py diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py new file mode 100644 index 0000000..5c4406c --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_App.py @@ -0,0 +1,28 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import simple_10gbe_rudp_kcu105_example as baseBoard +import rocev2_10gbe_rudp_kcu105_example as roceBoard + +class App(baseBoard.App): + def __init__( self, dispatchBits=24, **kwargs): + super().__init__(**kwargs) + + self.add(roceBoard.RoceDispatcher( + offset = 0x0002_0000, + dispatchBits = dispatchBits, + expand = False, + )) + + self.add(roceBoard.RoceChecker( + offset = 0x0003_0000, + dispatchBits = dispatchBits, + expand = False, + )) diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceChecker.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceChecker.py new file mode 100644 index 0000000..5563ef6 --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceChecker.py @@ -0,0 +1,38 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import pyrogue as pr + +class RoceChecker(pr.Device): + def __init__( self, + dispatchBits=24, + **kwargs): + super().__init__(**kwargs) + + self.add(pr.RemoteVariable( + name = 'SuccessCounter', + offset = 0xF00, + bitSize = dispatchBits, + mode = 'RO', + )) + + self.add(pr.RemoteVariable( + name = 'UnsuccessCounter', + offset = 0xF04, + bitSize = dispatchBits, + mode = 'RO', + )) + + self.add(pr.RemoteVariable( + name = 'ResetCounters', + offset = 0xF08, + bitSize = 1, + mode = 'WO', + )) diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceDispatcher.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceDispatcher.py new file mode 100644 index 0000000..b0398cc --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_RoceDispatcher.py @@ -0,0 +1,83 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import pyrogue as pr + +class RoceDispatcher(pr.Device): + def __init__( self, + dispatchBits=24, + **kwargs): + super().__init__(**kwargs) + + self.add(pr.RemoteVariable( + name = 'StartDispatching', + offset = 0xF00, + bitSize = 1, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'DispatchCounter', + offset = 0xF00, + bitSize = dispatchBits, + bitOffset = 1, + disp = '{:d}', + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'Len', + offset = 0xF04, + bitSize = 32, + disp = '{:d}', + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'RKey', + offset = 0xF08, + bitSize = 32, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'LKey', + offset = 0xF0C, + bitSize = 32, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'SQpn', + offset = 0xF10, + bitSize = 24, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'DQpn', + offset = 0xF14, + bitSize = 24, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'RemAddr', + offset = 0xF18, + bitSize = 64, + mode = 'RW', + )) + + self.add(pr.RemoteVariable( + name = 'AddrWrapCount', + offset = 0xF20, + bitSize = 32, + mode = 'RW', + )) diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py new file mode 100644 index 0000000..ef543ea --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py @@ -0,0 +1,255 @@ +#----------------------------------------------------------------------------- +# This file is part of the 'Simple-10GbE-RUDP-KCU105-Example'. It is subject to +# the license terms in the LICENSE.txt file found in the top-level directory +# of this distribution and at: +# https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html. +# No part of the 'Simple-10GbE-RUDP-KCU105-Example', including this file, may be +# copied, modified, propagated, or distributed except according to the terms +# contained in the LICENSE.txt file. +#----------------------------------------------------------------------------- + +import time + +import pyrogue as pr +import pyrogue.protocols +import pyrogue.utilities.fileio +import pyrogue.interfaces.simulation + +import rogue +import rogue.hardware.axi +import rogue.interfaces.stream +import rogue.utilities.fileio + +import simple_10gbe_rudp_kcu105_example as baseBoard +import rocev2_10gbe_rudp_kcu105_example as roceBoard + +# TODO: Uncomment this after the rogue v6.14.0 tag release +#rogue.Version.minVersion('6.14.0') + +# IBV_MTU enum values — mirrors libibverbs ibv_mtu +IBV_MTU_256 = 1 +IBV_MTU_512 = 2 +IBV_MTU_1024 = 3 +IBV_MTU_2048 = 4 +IBV_MTU_4096 = 5 + +_MTU_BYTES = {1: 256, 2: 512, 3: 1024, 4: 2048, 5: 4096} + + +class Root(pr.Root): + def __init__(self, + ip = '192.168.2.10', + promProg = False, # Flag to disable all devices not related to PROM programming + enSwRx = True, # Flag to enable the software stream receiver + xvcSrvEn = True, # Flag to include the XVC server + zmqSrvPort = 9099, # Set to zero if dynamic (instead of static) + # ---------------------------------------------------------------- + # RoCEv2 options (meta mode only) + # ---------------------------------------------------------------- + useRoce = False, # Add RoCEv2 RDMA receive channel alongside UDP/RSSI + useDcqcn = True, + roceDevice = 'rxe0', # ibverbs device name (rxe0=softRoCE, mlx5_0=HW NIC) + roceIbPort = 1, # ibverbs port number + roceGidIndex = -1, # GID index (-1 = auto-detect from ip) + roceMaxPay = None, # Max payload bytes per RDMA WRITE (None = 9000) + roceQDepth = None, # RX queue depth (None = 256) + rocePmtu = IBV_MTU_4096, # Path MTU: IBV_MTU_256/512/1024/2048/4096 + roceOffset = 0x0000_0000, # AXI-lite byte offset of RoCEv2 engine registers + roceMinRnrTimer = 1, # IB min_rnr_timer (1=0.01ms, 31=491ms) + roceRnrRetry = 7, # FPGA RNR retry count (7=infinite) + roceRetryCount = 3, # FPGA retry count for non-RNR errors + **kwargs): + super().__init__(timeout=(5.0 if (ip != 'sim') else 100.0), **kwargs) + + ################################################################# + + self.zmqServer = pyrogue.interfaces.ZmqServer(root=self, addr='127.0.0.1', port=zmqSrvPort) + self.addInterface(self.zmqServer) + + ################################################################# + + self.enSwRx = not promProg and enSwRx + self.promProg = promProg + self.sim = (ip == 'sim') + self.useRoce = useRoce and not promProg and not self.sim + self.useDcqcn = useDcqcn and not promProg and not self.sim + + # Resolve RoCEv2 defaults + if self.useRoce: + import rogue.protocols.rocev2 as _rv2 + _maxPay = roceMaxPay if roceMaxPay is not None else _rv2.DefaultMaxPayload + _qDepth = roceQDepth if roceQDepth is not None else _rv2.DefaultRxQueueDepth + _gidIndex = roceGidIndex if roceGidIndex >= 0 else self._autoGidIndex(roceDevice, ip) + _mtu_b = _MTU_BYTES.get(rocePmtu, '?') + self._log.info( + f"RoCEv2 streaming enabled: device={roceDevice} " + f"gidIndex={_gidIndex} pmtu={_mtu_b} bytes " + f"maxPayload={_maxPay} queueDepth={_qDepth}" + ) + + ################################################################# + # Transport / memory path (unchanged from upstream) + ################################################################# + + if ip == 'emu': + self.srp = pr.interfaces.simulation.MemEmulate() + self.stream = rogue.interfaces.stream.Master() + + elif ip == 'sim': + self._pollEn = False + self._initRead = False + self.srp = rogue.interfaces.memory.TcpClient('localhost', 10000) + self.stream = rogue.interfaces.stream.TcpClient('localhost', 10002) + + else: + # UDP/RSSI clients — both always present + self.rudp = [None for i in range(2)] + for i in range(2): + self.rudp[i] = pr.protocols.UdpRssiPack( + name = f'SwRudpClient[{i}]', + host = ip, + port = 8192 + i, + packVer = 2, + jumbo = (i > 0), + expand = False, + ) + self.add(self.rudp[i]) + + # SRPv3 for register access + self.srp = rogue.protocols.srp.SrpV3() + self.srp == self.rudp[0].application(0) + + # Streaming path — RUDP[1] always connected as upstream + self.stream = self.rudp[1].application(0) + + # ---- RoCEv2 receive channel (additive, alongside RUDP) ---- + if self.useRoce: + # Core must be added first so we can reference Core.RoceEngine. + # We add it here early; the block below skips re-adding it. + self.add(baseBoard.Core( + offset = 0x0000_0000, + memBase = self.srp, + sim = self.sim, + promProg = promProg, + rocev2 = self.useRoce, + dcqcn = self.useDcqcn, + expand = True, + )) + self._coreAlreadyAdded = True + + _rdmaRx = pr.protocols.RoCEv2Server( + name = 'rdmaRx', + ip = ip, + deviceName = roceDevice, + ibPort = roceIbPort, + gidIndex = _gidIndex, + maxPayload = _maxPay, + rxQueueDepth = _qDepth, + pmtu = rocePmtu, + minRnrTimer = roceMinRnrTimer, + rnrRetry = roceRnrRetry, + retryCount = roceRetryCount, + roceEngineOffset = roceOffset, + roceMemBase = self.srp, + roceEngine = self.Core.RoceEngine, + expand = False, + ) + self.add(_rdmaRx) + # rdmaRx.stream is the RDMA receive endpoint; + # self.stream remains the RUDP streaming endpoint + self.rdmaStream = self.rdmaRx.stream + + # XVC server (unchanged from upstream) + if not self.promProg and xvcSrvEn: + self.udpClient = rogue.protocols.udp.Client(ip, 2542, False) + self.xvc = rogue.protocols.xilinx.Xvc(2542) + self.addProtocol(self.xvc) + self.udpClient == self.xvc + + ################################################################# + # Software receiver and file writer (unchanged from upstream) + ################################################################# + + if self.enSwRx: + self.dataWriter = pr.utilities.fileio.StreamWriter() + self.add(self.dataWriter) + + self.swRx = baseBoard.SwRx(expand=True) + self.add(self.swRx) + + self.stream >> self.swRx + self.stream >> self.dataWriter.getChannel(0) + + # If RoCEv2 is enabled, also write RDMA frames to a separate channel + if self.useRoce: + self.rdmaStream >> self.dataWriter.getChannel(1) + + ################################################################# + # Devices (Core added early above if useRoce, otherwise add here) + ################################################################# + + if not getattr(self, '_coreAlreadyAdded', False): + self.add(baseBoard.Core( + offset = 0x0000_0000, + memBase = self.srp, + sim = self.sim, + promProg = promProg, + expand = True, + )) + + if not promProg: + self.add(roceBoard.App( + offset = 0x8000_0000, + memBase = self.srp, + sim = self.sim, + rocev2 = self.useRoce, + expand = True, + )) + + def start(self, **kwargs): + super().start(**kwargs) + if not self.sim: + appTx = self.find(typ=baseBoard.AppTx) + for devPtr in appTx: + devPtr.ContinuousMode.set(False) + self.CountReset() + + def _start(self) -> None: + """ + Wait for SRP/RSSI link before starting RoCEv2 connection sequence. + """ + if self.useRoce and hasattr(self, 'rudp'): + self._log.info("Waiting for SRP/RSSI link before RoCEv2 setup...") + deadline = time.monotonic() + 30.0 + while not self.rudp[0]._rssi.getOpen(): + if time.monotonic() > deadline: + raise RuntimeError( + "Timeout waiting for SRP/RSSI link to come up") + time.sleep(0.1) + self._log.info("SRP/RSSI link is up — proceeding with RoCEv2 setup") + time.sleep(0.5) + super()._start() + + def stop(self) -> None: + """Tear down FPGA QP before transport is stopped.""" + if self.useRoce and hasattr(self, 'rdmaRx') and hasattr(self.rdmaRx, 'teardownFpgaQp'): + self.rdmaRx.teardownFpgaQp() + super().stop() + + @staticmethod + def _autoGidIndex(device: str, ip: str) -> int: + """Find the GID index matching ip on the given ibverbs device.""" + import subprocess + try: + out = subprocess.check_output( + ['ibv_devinfo', '-v', '-d', device], + stderr=subprocess.DEVNULL, + text=True, + ) + for line in out.splitlines(): + line = line.strip() + if 'GID[' in line and ip in line: + return int(line.split('[')[1].split(']')[0]) + except Exception: + pass + return 1 # safe default for softRoCE diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py new file mode 100644 index 0000000..b39667f --- /dev/null +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/__init__.py @@ -0,0 +1,4 @@ +from rocev2_10gbe_rudp_kcu105_example._RoceChecker import * +from rocev2_10gbe_rudp_kcu105_example._RoceDispatcher import * +from rocev2_10gbe_rudp_kcu105_example._App import * +from rocev2_10gbe_rudp_kcu105_example._Root import * diff --git a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py index 8aa3f87..a1fc662 100644 --- a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py +++ b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py @@ -17,11 +17,14 @@ import surf.ethernet.ten_gig as mac import surf.protocols.rssi as rssi import surf.xilinx as xil +import surf.ethernet.roce as roce class Core(pr.Device): def __init__( self, sim = False, promProg = False, + rocev2 = False, + dcqcn = True, **kwargs): super().__init__(**kwargs) @@ -52,9 +55,18 @@ def __init__( self, self.add(udp.UdpEngine( offset = 0x0011_0000, numSrv = 2, + numClt = 1 if rocev2 else 0, enabled = not sim, )) + if rocev2: + self.add(roce.RoceEngine( + offset = 0x0015_0000, + dcqcn = dcqcn, + expand = False, + enabled = not sim, + )) + for i in range(2): self.add(rssi.RssiCore( name = f'FwRudpServer[{i}]', @@ -79,4 +91,4 @@ def __init__( self, self.add(xceiver.Sfp( offset = 0x0020_2000, enabled = not sim, - )) \ No newline at end of file + )) diff --git a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd index 4f8540d..1d7f168 100755 --- a/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd +++ b/firmware/targets/RoCEv2_10GbeRudpKcu105Example/hdl/RoCEv2_10GbeRudpKcu105Example.vhd @@ -19,7 +19,7 @@ library surf; use surf.StdRtlPkg.all; use surf.AxiStreamPkg.all; use surf.AxiLitePkg.all; -use surf.RocePkg.all; +use surf.RoCEv2Pkg.all; library work; use work.CorePkg.all; diff --git a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd index d57c546..8285442 100755 --- a/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd +++ b/firmware/targets/RoCEv2_1GbeRudpKcu105Example/hdl/RoCEv2_1GbeRudpKcu105Example.vhd @@ -19,7 +19,7 @@ library surf; use surf.StdRtlPkg.all; use surf.AxiStreamPkg.all; use surf.AxiLitePkg.all; -use surf.RocePkg.all; +use surf.RoCEv2Pkg.all; library work; use work.CorePkg.all; diff --git a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd index 111eadf..7827be2 100755 --- a/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd +++ b/firmware/targets/RoCEv2_Rj45RudpKcu105Example/hdl/RoCEv2_Rj45RudpKcu105Example.vhd @@ -19,7 +19,7 @@ library surf; use surf.StdRtlPkg.all; use surf.AxiStreamPkg.all; use surf.AxiLitePkg.all; -use surf.RocePkg.all; +use surf.RoCEv2Pkg.all; library work; use work.CorePkg.all; diff --git a/software/scripts/fileReader.py b/software/scripts/fileReader.py index 4ce38e2..d6d9399 100644 --- a/software/scripts/fileReader.py +++ b/software/scripts/fileReader.py @@ -8,54 +8,225 @@ # copied, modified, propagated, or distributed except according to the terms # contained in the LICENSE.txt file. #----------------------------------------------------------------------------- +""" +fileReader.py — Read rogue .dat files written by StreamWriter. + +Extends the upstream fileReader with: + - --channel : filter by rogue stream channel + (channel 0 = UDP/RSSI stream, channel 1 = RoCEv2 RDMA stream) + - --check-cont : verify the payload bytes are contiguous across frames + (i.e. each frame starts where the previous one ended, + using an incrementing 0x00..0xff byte pattern) + - --hex : hex dump each frame payload + - --debug : enable SwRx DebugPrint (prints header word of each frame) + +Usage: + # Read all frames, print summary + python3 fileReader.py --dataFile data.dat + + # Check RoCEv2 frames (channel 1) for contiguous byte pattern + python3 fileReader.py --dataFile data.dat --channel 1 --check-cont + + # Hex dump UDP frames (channel 0) + python3 fileReader.py --dataFile data.dat --channel 0 --hex + + # Enable upstream SwRx debug print + python3 fileReader.py --dataFile data.dat --debug +""" import setupLibPaths import pyrogue as pr - import rogue.utilities.fileio - +import rogue.interfaces.stream as ris import argparse +import numpy as np import simple_10gbe_rudp_kcu105_example as devBoard -if __name__ == "__main__": - ################################################################# +# --------------------------------------------------------------------------- +# Contiguity-checking stream slave +# --------------------------------------------------------------------------- +class ContiguityChecker(ris.Slave): + """ + Stream slave that checks received frames for a contiguous incrementing + byte pattern (0x00 0x01 ... 0xff 0x00 ...) across frame boundaries. + + Connect downstream of a channel filter so it only sees the frames + you care about. + """ + def __init__(self, channel=None): + super().__init__() + self._channel = channel + self._last_byte = None + self._frame_num = 0 + self._gap_count = 0 + self._byte_count = 0 + + def _acceptFrame(self, frame): + # Read payload + size = frame.getPayload() + buf = bytearray(size) + frame.read(buf, 0) + + self._frame_num += 1 + self._byte_count += size + + ch_str = f'ch{self._channel} ' if self._channel is not None else '' + print( + f'Frame {self._frame_num:>6d} | ' + f'{ch_str}' + f'size={size:>8d} bytes' + ) + + if size == 0: + return + + first_byte = buf[0] + + if self._last_byte is not None: + expected = (self._last_byte + 1) % 256 + if first_byte != expected: + self._gap_count += 1 + print( + f' *** GAP at frame {self._frame_num}: ' + f'expected 0x{expected:02x} got 0x{first_byte:02x} ' + f'(delta={(first_byte - expected) % 256})' + ) + + self._last_byte = buf[-1] + + def summary(self): + print() + print( + f'Total: {self._frame_num} frame(s), ' + f'{self._byte_count} bytes ' + f'({self._byte_count / 1024:.1f} KB)' + ) + if self._gap_count == 0: + print('Contiguity: OK — no gaps detected') + else: + print(f'Contiguity: FAIL — {self._gap_count} gap(s) detected') + + +# --------------------------------------------------------------------------- +# Hex-dumping stream slave +# --------------------------------------------------------------------------- +class HexDumper(ris.Slave): + """Stream slave that hex-dumps each received frame.""" + + def __init__(self, bytes_per_row=16): + super().__init__() + self._bytes_per_row = bytes_per_row + self._frame_num = 0 + + def _acceptFrame(self, frame): + size = frame.getPayload() + buf = bytearray(size) + frame.read(buf, 0) + self._frame_num += 1 + print(f'--- Frame {self._frame_num} ({size} bytes) ---') + bpr = self._bytes_per_row + for i in range(0, size, bpr): + chunk = buf[i:i + bpr] + hex_ = ' '.join(f'{b:02x}' for b in chunk) + ascii_ = ''.join(chr(b) if 32 <= b < 127 else '.' for b in chunk) + print(f' {i:08x}: {hex_:<{bpr * 3}} {ascii_}') + print() + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +if __name__ == "__main__": - # Set the argument parser - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description='Read rogue .dat files with optional contiguity check') - # Add arguments parser.add_argument( "--dataFile", type = str, required = True, - help = "path to data file", + help = "Path to .dat file written by StreamWriter", + ) + parser.add_argument( + "--channel", + type = int, + default = None, + help = "Filter by rogue stream channel. " + "0 = UDP/RSSI stream, 1 = RoCEv2 RDMA stream. " + "Default: show all channels", + ) + parser.add_argument( + "--check-cont", + action = 'store_true', + help = "Check that frame payloads form a contiguous incrementing " + "byte pattern across frame boundaries", + ) + parser.add_argument( + "--hex", + action = 'store_true', + help = "Hex dump each frame payload", + ) + parser.add_argument( + "--debug", + action = 'store_true', + help = "Enable SwRx DebugPrint (prints header word of each frame)", ) - # Get the arguments args = parser.parse_args() ################################################################# - # Create the File reader streaming interface + # File reader dataReader = rogue.utilities.fileio.StreamReader() - # Create the Event reader streaming interface + # pyrogue root with upstream SwRx root = pr.Root() root.add(devBoard.SwRx()) root.start() - root.SwRx.DebugPrint.setDisp(True) + if args.debug: + root.SwRx.DebugPrint.setDisp(True) - # Connect the file reader ---> root.SwRx + # Wire: file reader → SwRx (upstream behaviour, always present) dataReader >> root.SwRx - # Open the file - dataReader.open(args.dataFile) + # Optional channel filter + contiguity checker / hex dumper + checkers = [] + + if args.check_cont or args.hex: + if args.channel is not None: + # Filter to the requested channel then attach checker/dumper + filt = ris.Filter(False, args.channel) + dataReader >> filt + + if args.check_cont: + checker = ContiguityChecker(channel=args.channel) + filt >> checker + checkers.append(checker) + + if args.hex: + dumper = HexDumper() + filt >> dumper - # Close file once everything processed + else: + # No channel filter — attach directly + if args.check_cont: + checker = ContiguityChecker() + dataReader >> checker + checkers.append(checker) + + if args.hex: + dumper = HexDumper() + dataReader >> dumper + + # Open and process the file + dataReader.open(args.dataFile) dataReader.closeWait() + root.stop() - ################################################################# + # Print contiguity summary + for c in checkers: + c.summary() diff --git a/software/scripts/runDispatch.py b/software/scripts/runDispatch.py new file mode 100644 index 0000000..e34281a --- /dev/null +++ b/software/scripts/runDispatch.py @@ -0,0 +1,151 @@ +import setupLibPaths +import sys +import logging +import time +import argparse +import pyrogue as pr +import pyrogue.interfaces + + +if __name__ == "__main__": + + # Convert str to bool + argBool = lambda s: s.lower() in ['true', 't', 'yes', '1'] + + # Set the argument parser + parser = argparse.ArgumentParser(description='Dispatch RDMA WRITE-with-Immediate packets') + + parser.add_argument( + "--addr", + type = str, + required= False, + default = 'localhost', + help = "ZMQ server address", + ) + parser.add_argument( + "--port", + type = int, + required= False, + default = 9099, + help = "ZMQ server port", + ) + parser.add_argument( + "--sim", + action = 'store_true', + help = "Simulation mode", + ) + parser.add_argument( + "--cases", + required= False, + default = 1, + type = int, + help = "Number of packets to send", + ) + # Get the arguments + args = parser.parse_args() + + # Set up logging + logging.basicConfig( + format = '%(levelname)s:%(name)s:%(message)s', + stream = sys.stdout, + ) + log = logging.getLogger('dispatch') + log.setLevel(logging.DEBUG) + + ################################################################# + with pr.interfaces.VirtualClient( + addr = args.addr, + port = args.port, + ) as client: + + rx = client.root.rdmaRx + root = client.root + + # ---------------------------------------------------------------- + # Validate connection state + # ---------------------------------------------------------------- + state = rx.ConnectionState.get() + if state != 'Connected': + log.error(f'RoCEv2 not connected (state={state}) — aborting') + sys.exit(1) + + # ---------------------------------------------------------------- + # Retrieve MR parameters from rogue local variables + # ---------------------------------------------------------------- + rx_queue_depth = rx.RxQueueDepth.get() + max_payload = rx.MaxPayload.get() + mr_len = rx_queue_depth * max_payload + remQpn = rx.HostQpn.get() + mrRKey = rx.MrRkey.get() + mrAddr = rx.MrAddr.get() + locKey = rx.FpgaLkey.get() + + # Payload = MaxPayload so MR slots are always fully used + # and the FPGA writes contiguously with no gaps + payload = max_payload + addr_wrap = rx_queue_depth # = mr_len // payload since payload == max_payload + + # ---------------------------------------------------------------- + # Print connection info + # ---------------------------------------------------------------- + log.info('--- RoCEv2 connection parameters ---') + log.info(f' ConnectionState : {rx.ConnectionState.get()}') + log.info(f' Host QPN : {hex(remQpn)}') + log.info(f' Host GID : {rx.HostGid.get()}') + log.info(f' Host RQ PSN : {hex(rx.HostRqPsn.get())}') + log.info(f' Host SQ PSN : {hex(rx.HostSqPsn.get())}') + log.info(f' MR addr : {hex(mrAddr)}') + log.info(f' MR rkey : {hex(mrRKey)}') + log.info(f' FPGA lkey : {hex(locKey)}') + log.info(f' FPGA QPN : {hex(rx.FpgaQpn.get())}') + log.info(f' FPGA GID : {rx.FpgaGid.get()}') + log.info(f' MaxPayload : {max_payload}') + log.info(f' RxQueueDepth : {rx_queue_depth}') + log.info(f' MrLen : {mr_len}') + log.info(f' Payload (= MaxPayload) : {payload}') + log.info(f' AddrWrapCount : {addr_wrap}') + log.info('------------------------------------') + + # ---------------------------------------------------------------- + # Set UDP engine destination + # ---------------------------------------------------------------- + root.Core.UdpEngine.ClientRemotePort[0].set(4791) + root.Core.UdpEngine.ClientRemoteIp[0].set("192.168.2.100") + + # ---------------------------------------------------------------- + # Reset checker counters + # ---------------------------------------------------------------- + log.info('Resetting counters...') + root.App.RoceChecker.ResetCounters.set(0) + root.App.RoceChecker.ResetCounters.set(1) + root.App.RoceChecker.ResetCounters.set(0) + + # ---------------------------------------------------------------- + # Configure dispatcher + # ---------------------------------------------------------------- + root.App.RoceDispatcher.Len.set(payload) + root.App.RoceDispatcher.DispatchCounter.set(args.cases) + root.App.RoceDispatcher.RKey.set(mrRKey) + root.App.RoceDispatcher.RemAddr.set(mrAddr) + root.App.RoceDispatcher.AddrWrapCount.set(addr_wrap) + + root.App.RoceDispatcher.SQpn.set(remQpn) + root.App.RoceDispatcher.LKey.set(locKey) + + # ---------------------------------------------------------------- + # Trigger dispatch + # ---------------------------------------------------------------- + log.info(f'Dispatching {args.cases} packet(s) of {payload} bytes...') + root.App.RoceDispatcher.StartDispatching.set(0) + root.App.RoceDispatcher.StartDispatching.set(1) + root.App.RoceDispatcher.StartDispatching.set(0) + + # ---------------------------------------------------------------- + # Wait and check results + # ---------------------------------------------------------------- + time.sleep(2) + success = root.App.RoceChecker.SuccessCounter.get() + log.info(f'Correctly received {success} / {args.cases} packet(s)') + if success != args.cases: + log.warning(f'{args.cases - success} packet(s) missing or errored') + diff --git a/software/scripts/startZmq.py b/software/scripts/startZmq.py new file mode 100644 index 0000000..7e38d12 --- /dev/null +++ b/software/scripts/startZmq.py @@ -0,0 +1,210 @@ +import setupLibPaths +import argparse +import pyrogue as pr +import pyrogue.pydm +import logging +import simple_10gbe_rudp_kcu105_example as devBoard + +# IBV_MTU constants for the --rocePmtu argument +IBV_MTU_256 = 1 +IBV_MTU_512 = 2 +IBV_MTU_1024 = 3 +IBV_MTU_2048 = 4 +IBV_MTU_4096 = 5 + +if __name__ == "__main__": + + # Convert str to bool + argBool = lambda s: s.lower() in ['true', 't', 'yes', '1'] + + # Set the argument parser + parser = argparse.ArgumentParser() + + # ------------------------------------------------------------------------- + # Original arguments (unchanged) + # ------------------------------------------------------------------------- + parser.add_argument( + "--ip", + type = str, + required = False, + default = '192.168.2.10', + help = "IP address", + ) + parser.add_argument( + "--pollEn", + type = argBool, + required = False, + default = True, + help = "Enable auto-polling", + ) + parser.add_argument( + "--initRead", + type = argBool, + required = False, + default = True, + help = "Enable read all variables at start", + ) + parser.add_argument( + "--LmkRegFile", + type = str, + required = False, + default = "../configurations/LmkForCtaJMode7.txt", + help = "Path to the LMK04828 register .txt file from TICS-PRO", + ) + parser.add_argument( + "--numStreams", + type = int, + required = False, + default = 1, + help = "Number of AXI-Streams in RUDP core", + ) + parser.add_argument( + "--sysrefCtrl", + type = argBool, + required = False, + default = False, + help = "Enable Sysref Ctrl for internal programmable delay", + ) + + # ------------------------------------------------------------------------- + # RoCEv2 arguments + # ------------------------------------------------------------------------- + parser.add_argument( + "--useRoce", + action = 'store_true', + default = False, + help = "Use RoCEv2 RDMA for streaming instead of UDP/RSSI", + ) + parser.add_argument( + "--useDcqcn", + action = 'store_true', + default = False, + help = "Use RoCEv2 DCQCN for congestion control", + ) + parser.add_argument( + "--roceDevice", + type = str, + required = False, + default = 'rxe0', + help = "ibverbs device name. " + "Use 'rxe0' for softRoCE or e.g. 'mlx5_0' for a hardware RoCEv2 NIC. " + "Run 'ibv_devinfo' to list available devices. " + "Default: rxe0", + ) + parser.add_argument( + "--roceGidIndex", + type = int, + required = False, + default = -1, + help = "GID table index for the host NIC's RoCEv2 IPv4 address. " + "Run 'ibv_devinfo -v -d | grep GID' to find the right index. " + "-1 = auto-detect from --ip. " + "Default: -1 (auto)", + ) + parser.add_argument( + "--rocePmtu", + type = int, + required = False, + default = IBV_MTU_4096, + choices = [IBV_MTU_256, IBV_MTU_512, IBV_MTU_1024, IBV_MTU_2048, IBV_MTU_4096], + metavar = '{1=256, 2=512, 3=1024, 4=2048, 5=4096}', + help = "Path MTU for the RC QP. Must match the FPGA firmware setting. " + "1=256 B 2=512 B 3=1024 B 4=2048 B 5=4096 B. " + "Default: 5 (4096 bytes)", + ) + parser.add_argument( + "--roceMaxPay", + type = int, + required = False, + default = None, + help = "Max payload bytes per RDMA WRITE. " + "Must be >= the largest single transfer your FPGA will send. " + "Default: 9000", + ) + parser.add_argument( + "--roceQDepth", + type = int, + required = False, + default = None, + help = "Number of receive slots (zero-copy queue depth). " + "Increase if frames are dropped at high data rates. " + "Default: 256", + ) + parser.add_argument( + "--roceOffset", + type = lambda x: int(x, 0), # accepts hex (0x...) or decimal + required = False, + default = 0x0000_0000, + help = "[meta mode] AXI-lite byte offset of the RoCEv2 engine register block. " + "Default: 0x0", + ) + + # ------------------------------------------------------------------------- + # RoCEv2 QP tuning arguments + # ------------------------------------------------------------------------- + parser.add_argument( + "--roceMinRnrTimer", + type = int, + required = False, + default = 1, + metavar = '{0..31}', + help = "Minimum RNR timer code embedded in RNR NAK packets sent to the FPGA. " + "The FPGA must wait at least this long before retrying after an RNR NAK. " + "IB spec: 0=655ms 1=0.01ms 14=1ms 18=4ms 22=16ms 31=491ms. " + "Default: 1 (0.01ms)", + ) + parser.add_argument( + "--roceRnrRetry", + type = int, + required = False, + default = 7, + metavar = '{0..7}', + help = "Number of times the FPGA retries after receiving an RNR NAK. " + "7 = infinite retries (recommended). " + "Default: 7", + ) + parser.add_argument( + "--roceRetryCount", + type = int, + required = False, + default = 3, + metavar = '{0..7}', + help = "Number of times the FPGA retries after a non-RNR error " + "(e.g. timeout, sequence error). " + "Default: 3", + ) + + # ------------------------------------------------------------------------- + # Parse and launch + # ------------------------------------------------------------------------- + args = parser.parse_args() + + if args.useRoce: + logging.getLogger('pyrogue.Device.RoCEv2Server').setLevel(logging.DEBUG) + logging.getLogger('pyrogue.Root').setLevel(logging.INFO) + + root = devBoard.Root( + ip = args.ip, + xvcSrvEn = False, + # RoCEv2 + useRoce = args.useRoce, + useDcqcn = args.useDcqcn, + roceDevice = args.roceDevice, + roceGidIndex = args.roceGidIndex, + rocePmtu = args.rocePmtu, + roceMaxPay = args.roceMaxPay, + roceQDepth = args.roceQDepth, + roceOffset = args.roceOffset, + # RoCEv2 QP tuning + roceMinRnrTimer = args.roceMinRnrTimer, + roceRnrRetry = args.roceRnrRetry, + roceRetryCount = args.roceRetryCount, + ) + + with root as hw: + pr.waitCntrlC() + # Tear down FPGA QP before the root context manager calls stop() + # and tears down the UDP transport + if args.useRoce and hasattr(root, 'rdmaRx') and hasattr(root.rdmaRx, 'teardownFpgaQp'): + root.rdmaRx.teardownFpgaQp() + From 1f420bb57540d55abb77bcd574f61b0df911af1a Mon Sep 17 00:00:00 2001 From: Larry Ruckman Date: Sat, 6 Jun 2026 15:02:30 -0700 Subject: [PATCH 4/4] bug fixes --- firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py | 5 ++--- firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py | 2 +- software/scripts/runDispatch.py | 6 +++++- software/scripts/startZmq.py | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py index ef543ea..b8acbbd 100644 --- a/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py +++ b/firmware/python/rocev2_10gbe_rudp_kcu105_example/_Root.py @@ -124,7 +124,7 @@ def __init__(self, # ---- RoCEv2 receive channel (additive, alongside RUDP) ---- if self.useRoce: - # Core must be added first so we can reference Core.RoceEngine. + # Core must be added first so we can reference Core.RoCEv2Engine. # We add it here early; the block below skips re-adding it. self.add(baseBoard.Core( offset = 0x0000_0000, @@ -151,7 +151,7 @@ def __init__(self, retryCount = roceRetryCount, roceEngineOffset = roceOffset, roceMemBase = self.srp, - roceEngine = self.Core.RoceEngine, + roceEngine = self.Core.RoCEv2Engine, expand = False, ) self.add(_rdmaRx) @@ -202,7 +202,6 @@ def __init__(self, offset = 0x8000_0000, memBase = self.srp, sim = self.sim, - rocev2 = self.useRoce, expand = True, )) diff --git a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py index a1fc662..a28107c 100644 --- a/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py +++ b/firmware/python/simple_10gbe_rudp_kcu105_example/_Core.py @@ -60,7 +60,7 @@ def __init__( self, )) if rocev2: - self.add(roce.RoceEngine( + self.add(roce.RoCEv2Engine( offset = 0x0015_0000, dcqcn = dcqcn, expand = False, diff --git a/software/scripts/runDispatch.py b/software/scripts/runDispatch.py index e34281a..0eab568 100644 --- a/software/scripts/runDispatch.py +++ b/software/scripts/runDispatch.py @@ -108,9 +108,13 @@ # ---------------------------------------------------------------- # Set UDP engine destination + # Derive the host IP from the IPv4-mapped HostGid (last 4 bytes) # ---------------------------------------------------------------- + gidWords = rx.HostGid.get().split(':') + hostIp = '.'.join(str(b) for b in bytes.fromhex(gidWords[-2] + gidWords[-1])) + log.info(f'Setting UDP engine destination to {hostIp}:4791') root.Core.UdpEngine.ClientRemotePort[0].set(4791) - root.Core.UdpEngine.ClientRemoteIp[0].set("192.168.2.100") + root.Core.UdpEngine.ClientRemoteIp[0].set(hostIp) # ---------------------------------------------------------------- # Reset checker counters diff --git a/software/scripts/startZmq.py b/software/scripts/startZmq.py index 7e38d12..3aae97b 100644 --- a/software/scripts/startZmq.py +++ b/software/scripts/startZmq.py @@ -3,7 +3,7 @@ import pyrogue as pr import pyrogue.pydm import logging -import simple_10gbe_rudp_kcu105_example as devBoard +import rocev2_10gbe_rudp_kcu105_example as devBoard # IBV_MTU constants for the --rocePmtu argument IBV_MTU_256 = 1