diff --git a/Bender.lock b/Bender.lock index ef9b323..6544a26 100644 --- a/Bender.lock +++ b/Bender.lock @@ -23,10 +23,10 @@ packages: Git: https://github.com/pulp-platform/common_verification.git dependencies: [] fpnew: - revision: bf2100ff68cf736c50b095169d87231a1fa3cc66 + revision: 0f6caf1dc76eb4f96c388364e35e8bf03ca19625 version: null source: - Git: https://github.com/Jayanth2209/cvfpu.git + Git: https://github.com/arpansur/cvfpu.git dependencies: - common_cells - fpu_div_sqrt_mvp diff --git a/Bender.yml b/Bender.yml index d0f172d..a2f25a8 100644 --- a/Bender.yml +++ b/Bender.yml @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 package: - name: spatz_core + name: spatz_vpu authors: - "Matheus Cavalcante " - "Mattia Sinigaglia " @@ -12,7 +12,7 @@ package: dependencies: riscv-dbg: { git: "https://github.com/pulp-platform/riscv-dbg.git", version: 0.7.0 } common_cells: { git: "https://github.com/pulp-platform/common_cells.git", version: 1.28.0 } - FPnew: { git: "https://github.com/Jayanth2209/cvfpu.git", rev: bf2100ff68cf736c50b095169d87231a1fa3cc66 } + FPnew: { git: "https://github.com/arpansur/cvfpu.git", rev: 0f6caf1dc76eb4f96c388364e35e8bf03ca19625 } # branch: pace-pipe-rebase tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.13 } snitch: { path: "hw/ip/snitch" } reqrsp_interface: { path: "hw/ip/reqrsp_interface" } @@ -34,6 +34,7 @@ sources: # Level 2 - hw/src/spatz_decoder.sv - hw/src/spatz_simd_lane.sv + - hw/src/pace_mem.sv - target: fpga files: - hw/src/vregfile_fpga.sv @@ -45,6 +46,7 @@ sources: - hw/src/spatz_ipu.sv - hw/src/spatz_vfu.sv - hw/src/spatz_vlsu.sv + - hw/src/spatz_doublebw_vlsu.sv - hw/src/spatz_vrf.sv - hw/src/spatz_vsldu.sv # Level 4 diff --git a/hw/ip/snitch/src/riscv_instr.sv b/hw/ip/snitch/src/riscv_instr.sv index 394a5f9..2199038 100644 --- a/hw/ip/snitch/src/riscv_instr.sv +++ b/hw/ip/snitch/src/riscv_instr.sv @@ -1975,6 +1975,7 @@ package riscv_instr; localparam logic [11:0] CSR_MHPMCOUNTER29H = 12'hb9d; localparam logic [11:0] CSR_MHPMCOUNTER30H = 12'hb9e; localparam logic [11:0] CSR_MHPMCOUNTER31H = 12'hb9f; + localparam logic [11:0] CSR_PACE = 12'hba0; // MXCore custom instructions (opcode custom-3: 1110111) localparam logic [31:0] MXDOTP_B0 = 32'b?????????????????000?????1110111; diff --git a/hw/src/generated/spatz_pkg.sv b/hw/src/generated/spatz_pkg.sv index a598e9d..c97f7b6 100644 --- a/hw/src/generated/spatz_pkg.sv +++ b/hw/src/generated/spatz_pkg.sv @@ -18,7 +18,7 @@ package spatz_pkg; // Number of IPUs in each VFU (between 1 and 8) localparam int unsigned N_IPU = 1; // Number of FPUs in each VFU (between 1 and 8) - localparam int unsigned N_FPU = 8; + localparam int unsigned N_FPU = 4; // Number of FUs in each VFU localparam int unsigned N_FU = N_IPU > N_FPU ? N_IPU : N_FPU; // FPU support @@ -62,7 +62,7 @@ package spatz_pkg; localparam int unsigned NrWordsPerBank = NrVRFWords / NrVRFBanks; // Number of VLSU interfaces - localparam int unsigned NumVLSUInterfaces = 1; + localparam int unsigned NumVLSUInterfaces = 2; // Width of scalar register file adresses // Depends on whether we have a FP regfile or not @@ -76,6 +76,11 @@ package spatz_pkg; // Encodes both the scalar RD and the VD address in the VRF localparam int VFURespAddrWidth = GPRWidth > $clog2(NrVRFWords) ? GPRWidth : $clog2(NrVRFWords); + localparam int PaceDegree = 2; + localparam int PaceParts = 16; + localparam int PaceEps = 1; + localparam int PaceDataWidth = 32; + localparam int PaceParamWidth = ((PaceDegree + 1)*PaceParts + PaceParts - 1 + 2*PaceEps) * PaceDataWidth; ////////////////////// // Type Definitions // @@ -311,30 +316,34 @@ package spatz_pkg; // VRF/SB Ports // ////////////////// - typedef enum logic [idx_width(4 + 2 * 1):0] { + typedef enum logic [idx_width(4 + 2 * 2):0] { VFU_VS2_RD, VFU_VS1_RD, VFU_VD_RD, - VLSU_VS2_RD, - VLSU_VD_RD, + VLSU_VD_RD0, + VLSU_VS2_RD0, + VLSU_VD_RD1, + VLSU_VS2_RD1, VSLDU_VS2_RD } vreg_port_rd_e; - typedef enum logic [idx_width(2 + 1):0] { + typedef enum logic [idx_width(2 + 2):0] { VFU_VD_WD, - VLSU_VD_WD, + VLSU_VD_WD[2], VSLDU_VD_WD } vreg_port_wd_e; - typedef enum logic [idx_width(6 + 3 * 1):0] { + typedef enum logic [idx_width(6 + 3 * 2):0] { SB_VFU_VS2_RD, SB_VFU_VS1_RD, SB_VFU_VD_RD, - SB_VLSU_VS2_RD, - SB_VLSU_VD_RD, + SB_VLSU_VD_RD0, + SB_VLSU_VS2_RD0, + SB_VLSU_VD_RD1, + SB_VLSU_VS2_RD1, SB_VSLDU_VS2_RD, SB_VFU_VD_WD, - SB_VLSU_VD_WD, + SB_VLSU_VD_WD[2], SB_VSLDU_VD_WD } sb_port_e; @@ -358,7 +367,8 @@ package spatz_pkg; // INT8 INT16 INT32 INT64 IntFmtMask : {1'b1, 1'b1, 1'b1, 1'b1}, MxFpFmtMask : 9'b0, - MxIntFmtMask : 4'b0 + MxIntFmtMask : 4'b0, + PaceFeatures : '{PaceDegree: PaceDegree, PaceParts: PaceParts, PaceEps: 1'b1, PaceDataWidth: PaceDataWidth, PaceParamWidth: PaceParamWidth, PaceBstPipeRegs: 4'b0100, FmtConfig: 9'b101010000} } : // Single Precision FPU '{ @@ -370,7 +380,8 @@ package spatz_pkg; // INT8 INT16 INT32 INT64 IntFmtMask : {1'b1, 1'b1, 1'b1, 1'b0}, MxFpFmtMask : 9'b0, - MxIntFmtMask : 4'b0 + MxIntFmtMask : 4'b0, + PaceFeatures : '{PaceDegree: PaceDegree, PaceParts: PaceParts, PaceEps: 1'b1, PaceDataWidth: PaceDataWidth, PaceParamWidth: PaceParamWidth, PaceBstPipeRegs: 4'b0100, FmtConfig: 9'b101010000} }; // FP format conversion diff --git a/hw/src/pace_mem.sv b/hw/src/pace_mem.sv new file mode 100644 index 0000000..cb9d691 --- /dev/null +++ b/hw/src/pace_mem.sv @@ -0,0 +1,63 @@ +// Copyright 2026 ETH Zurich and University of Bologna. +// +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// SPDX-License-Identifier: SHL-0.51 + +// Author: Arpan Suravi Prasad +module pace_mem #( + parameter int unsigned BufDepth = 2, + parameter int unsigned BufWidth = 256, + parameter int unsigned ParamWidth = 256, + localparam int unsigned AddrWidth = $clog2(BufDepth) +)( + input logic clk_i, + input logic rst_ni, + input logic we_i, + output logic done_o, + input logic init_i, + input logic [BufWidth-1:0] data_i, + output logic [ParamWidth-1:0] data_o +); + logic [BufDepth-1:0][BufWidth-1:0] mem_d, mem_q; + logic [AddrWidth-1:0] cnt_d, cnt_q; + + always_comb begin + mem_d = mem_q; + cnt_d = cnt_q; + mem_d[cnt_q] = we_i ? data_i : mem_q[cnt_q]; + cnt_d = (init_i == 1'b0) ? '0 : we_i ? cnt_q + 1 : cnt_q; + end + + assign done_o = (cnt_q == BufDepth); + + always_ff @(posedge clk_i, negedge rst_ni) begin + if(~rst_ni) begin + cnt_q <= '0; + mem_q <= '0; + end else begin + cnt_q <= cnt_d; + mem_q <= mem_d; + end + end + + localparam NumParamWords = ParamWidth / BufWidth; + localparam RemParamWords = ParamWidth % BufWidth; + + if(NumParamWords > 0) begin + for(genvar ii=0; ii 0) begin + localparam int unsigned RemWidth = ParamWidth - NumParamWords*BufWidth; + assign data_o[NumParamWords*BufWidth+:RemWidth] = mem_q[NumParamWords][0+:RemWidth]; + end +endmodule diff --git a/hw/src/spatz.sv b/hw/src/spatz.sv index 341a36b..9e0d8bd 100644 --- a/hw/src/spatz.sv +++ b/hw/src/spatz.sv @@ -90,6 +90,7 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( // FPU side channel input roundmode_e fpu_rnd_mode_i, input fmt_mode_t fpu_fmt_mode_i, + input pace_mode_t fpu_pace_mode_i, output status_t fpu_status_o ); @@ -403,14 +404,53 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( vrf_addr_t [NrWritePorts-1:0] vrf_waddr, vrf_waddr_buf; vrf_data_t [NrWritePorts-1:0] vrf_wdata, vrf_wdata_buf; logic [NrWritePorts-1:0] vrf_we; + logic [NrWritePorts-1:0] vrf_we_mask; vrf_be_t [NrWritePorts-1:0] vrf_wbe, vrf_wbe_buf; logic [NrWritePorts-1:0] vrf_wvalid; + logic [NrWritePorts-1:0] vrf_wvalid_mask; // Read ports vrf_addr_t [NrReadPorts-1:0] vrf_raddr; logic [NrReadPorts-1:0] vrf_re; vrf_data_t [NrReadPorts-1:0] vrf_rdata; logic [NrReadPorts-1:0] vrf_rvalid; + // PACE parameter memory + // With DOUBLE_BW both VLSU write ports carry consecutive 256-bit chunks each cycle, + // so we must capture both to reconstruct the contiguous parameter stream. +`ifdef DOUBLE_BW + // Use WD0-only for pace_mem: each beat captures 256 bits from WD0 (lower addresses). + // This avoids requiring simultaneous WD0+WD1 grants, which is unreliable in DOUBLE_BW. + localparam int unsigned PaceBufWidth = N_FU * ELEN; +`else + localparam int unsigned PaceBufWidth = N_FU * ELEN; +`endif +`ifdef DOUBLE_BW + localparam int unsigned PaceLdIdx = VLSU_VD_WD0; +`else + localparam int unsigned PaceLdIdx = VLSU_VD_WD; +`endif + localparam int unsigned PaceBufDepth = (PaceParamWidth + PaceBufWidth - 1) / PaceBufWidth; + logic pace_mem_we; + logic [PaceParamWidth-1:0] pace_params; + logic pace_mem_init_done; + +`ifdef DOUBLE_BW + assign pace_mem_we = fpu_pace_mode_i.enable & vrf_we[VLSU_VD_WD0] & (~pace_mem_init_done); +`else + assign pace_mem_we = fpu_pace_mode_i.enable & vrf_we[PaceLdIdx] & (~pace_mem_init_done); +`endif + + always_comb begin + vrf_we_mask = vrf_we; + vrf_wvalid = vrf_wvalid_mask; + vrf_we_mask[PaceLdIdx] = fpu_pace_mode_i.enable & (~pace_mem_init_done) ? 1'b0 : vrf_we[PaceLdIdx]; + vrf_wvalid[PaceLdIdx] = fpu_pace_mode_i.enable & (~pace_mem_init_done) ? vrf_we[PaceLdIdx] : vrf_wvalid_mask[PaceLdIdx]; +`ifdef DOUBLE_BW + vrf_we_mask[VLSU_VD_WD1] = fpu_pace_mode_i.enable & (~pace_mem_init_done) ? 1'b0 : vrf_we[VLSU_VD_WD1]; + vrf_wvalid[VLSU_VD_WD1] = fpu_pace_mode_i.enable & (~pace_mem_init_done) ? vrf_we[VLSU_VD_WD1] : vrf_wvalid_mask[VLSU_VD_WD1]; +`endif + end + spatz_vrf #( .NrReadPorts (NrReadPorts ), .NrWritePorts(NrWritePorts), @@ -422,9 +462,9 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( // Write Ports .waddr_i (vrf_waddr_buf ), .wdata_i (vrf_wdata_buf ), - .we_i (vrf_we ), - .wbe_i (vrf_wbe_buf ), - .wvalid_o (vrf_wvalid ), + .we_i (vrf_we_mask ), + .wbe_i (vrf_wbe_buf ), + .wvalid_o (vrf_wvalid_mask), `ifdef BUF_FPU .fpu_buf_usage_i (vfu_buf_usage ), `endif @@ -435,6 +475,24 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( .rvalid_o (vrf_rvalid ) ); + pace_mem #( + .BufDepth (PaceBufDepth ), + .BufWidth (PaceBufWidth ), + .ParamWidth(PaceParamWidth) + ) i_pace_mem ( + .clk_i (clk_i ), + .rst_ni (rst_ni ), + .we_i (pace_mem_we ), + .done_o (pace_mem_init_done ), + .init_i (fpu_pace_mode_i.enable ), +`ifdef DOUBLE_BW + .data_i (vrf_wdata_buf[VLSU_VD_WD0]), +`else + .data_i (vrf_wdata_buf[PaceLdIdx]), +`endif + .data_o (pace_params ) + ); + //////////////// // Controller // //////////////// @@ -654,6 +712,8 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( .vrf_rvalid_i (vrf_rvalid[VFU_VD_RD:VFU_VS2_RD] ), .vrf_id_o ({sb_id[SB_VFU_VD_WD], sb_id[SB_VFU_VD_RD:SB_VFU_VS2_RD]}), // FPU side-channel + .pace_mode_i (fpu_pace_mode_i ), + .pace_param_i (pace_params ), .fpu_status_o (fpu_status_o ) ); @@ -676,6 +736,7 @@ module spatz import spatz_pkg::*; import rvv_pkg::*; import fpnew_pkg::*; #( // Response .vlsu_rsp_valid_o (vlsu_rsp_valid ), .vlsu_rsp_o (vlsu_rsp ), + .vlsu_buf_full_i (vlsu_buf_full ), .vlsu_buf_empty_i (vlsu_buf_empty ), // VRF .vrf_wvalid_i ({vrf_vlsu_wvalid, vrf_wvalid[VLSU_VD_WD0]} ), diff --git a/hw/src/spatz_controller.sv b/hw/src/spatz_controller.sv index cc1fe20..42f4f01 100644 --- a/hw/src/spatz_controller.sv +++ b/hw/src/spatz_controller.sv @@ -247,12 +247,18 @@ module spatz_controller // Counter to track the vlen completed for each instruction vlen_t [NrParallelInstructions-1:0] vl_cnt_d, vl_cnt_q, vl_max_d, vl_max_q; + // VLSU interfaces write independently and in parallel (unlike VFU/VSLDU, + // which share a single sequential vl_cnt progress counter), so each + // interface's own completion progress must be tracked separately. + vlen_t [NumVLSUInterfaces-1:0] [NrParallelInstructions-1:0] vlsu_vl_cnt_d, vlsu_vl_cnt_q; + // Is this instruction a narrowing instruction? logic [NrParallelInstructions-1:0] narrow_q, narrow_d; `FF(done_result_q, done_result_d, '0) `FF(vl_cnt_q, vl_cnt_d, '0) `FF(vl_max_q, vl_max_d, '0) + `FF(vlsu_vl_cnt_q, vlsu_vl_cnt_d, '0) `FF(narrow_q, narrow_d, '0) `else logic [NrParallelInstructions-1:0] wrote_result_q, wrote_result_d; @@ -285,6 +291,7 @@ module spatz_controller narrow_d = narrow_q; vl_cnt_d = vl_cnt_q; vl_max_d = vl_max_q; + vlsu_vl_cnt_d = vlsu_vl_cnt_q; `endif for (int unsigned port = 0; port < NrVregfilePorts; port++) begin @@ -311,60 +318,54 @@ module spatz_controller `ifdef DOUBLE_BW // Store the decisions - if (sb_enable_o[SB_VFU_VD_WD]) begin - // Calculate the load-store interface id to use here for chaining - automatic logic intID = (vl_cnt_q[sb_id_i[SB_VFU_VD_WD]] < vl_max_d[sb_id_i[SB_VFU_VD_WD]]) ? 0 : 1; - automatic int VRFWriteSize = narrow_q[sb_id_i[SB_VFU_VD_WD]] ? VRFWordBWidth >> 1 : VRFWordBWidth; - - // Update vl_cnt if actually written into the VRF - if (sb_wrote_result_i[SB_VFU_VD_WD - SB_VFU_VD_WD]) - vl_cnt_d[sb_id_i[SB_VFU_VD_WD]] += VRFWriteSize; - - if (vl_cnt_q[sb_id_i[SB_VFU_VD_WD]] >= (vl_max_d[sb_id_i[SB_VFU_VD_WD]] * (intID + 1) - VRFWriteSize)) begin - done_result_d[intID][sb_id_i[SB_VFU_VD_WD]] = 1'b1; - end - - wrote_result_narrowing_d[sb_id_i[SB_VFU_VD_WD]] = sb_wrote_result_i[SB_VFU_VD_WD - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VFU_VD_WD]]; - wrote_result_d[intID][sb_id_i[SB_VFU_VD_WD]] = sb_wrote_result_i[SB_VFU_VD_WD - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VFU_VD_WD]] || wrote_result_narrowing_q[sb_id_i[SB_VFU_VD_WD]]); - end - if (sb_enable_o[SB_VLSU_VD_WD0]) begin - // Calculate the load-store interface id to use here for chaining - wrote_result_narrowing_d[sb_id_i[SB_VLSU_VD_WD0]] = sb_wrote_result_i[SB_VLSU_VD_WD0 - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VLSU_VD_WD0]]; - wrote_result_d[0][sb_id_i[SB_VLSU_VD_WD0]] = sb_wrote_result_i[SB_VLSU_VD_WD0 - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VLSU_VD_WD0]] || wrote_result_narrowing_q[sb_id_i[SB_VLSU_VD_WD0]]); - end - if (sb_enable_o[SB_VLSU_VD_WD1]) begin - wrote_result_narrowing_d[sb_id_i[SB_VLSU_VD_WD1]] = sb_wrote_result_i[SB_VLSU_VD_WD1 - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VLSU_VD_WD1]]; - wrote_result_d[1][sb_id_i[SB_VLSU_VD_WD1]] = sb_wrote_result_i[SB_VLSU_VD_WD1 - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VLSU_VD_WD1]] || wrote_result_narrowing_q[sb_id_i[SB_VLSU_VD_WD1]]); - end - if (sb_enable_o[SB_VSLDU_VD_WD]) begin - // Calculate the load-store interface id to use here for chaining - automatic logic intID = (vl_cnt_q[sb_id_i[SB_VSLDU_VD_WD]] < vl_max_d[sb_id_i[SB_VSLDU_VD_WD]]) ? 0 : 1; - automatic int VRFWriteSize = narrow_q[sb_id_i[SB_VSLDU_VD_WD]] ? VRFWordBWidth >> 1 : VRFWordBWidth; - - // Update vl_cnt if actually written into the VRF - if (sb_wrote_result_i[SB_VSLDU_VD_WD - SB_VFU_VD_WD]) - vl_cnt_d[sb_id_i[SB_VSLDU_VD_WD]] += VRFWriteSize; - - if (vl_cnt_q[sb_id_i[SB_VSLDU_VD_WD]] >= (vl_max_d[sb_id_i[SB_VSLDU_VD_WD]] * (intID + 1) - VRFWriteSize)) begin - done_result_d[intID][sb_id_i[SB_VSLDU_VD_WD]] = 1'b1; + for (int unsigned port = 0; port < NrVregfilePorts; port++) begin + automatic int unsigned port_idx = port - SB_VFU_VD_WD; + if (sb_enable_o[port]) begin + // VFU and VSLDU: intID derived from vl_cnt progress, vl_cnt updated every successful write + if (port inside {SB_VFU_VD_WD, SB_VSLDU_VD_WD}) begin + automatic logic intID = (vl_cnt_q[sb_id_i[port]] < vl_max_d[sb_id_i[port]]) ? 0 : 1; + automatic int VRFWriteSize = narrow_q[sb_id_i[port]] ? VRFWordBWidth >> 1 : VRFWordBWidth; + + // Update vl_cnt if actually written into the VRF + if (sb_wrote_result_i[port_idx]) + vl_cnt_d[sb_id_i[port]] += VRFWriteSize; + + wrote_result_narrowing_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] ^ narrow_wide_q[sb_id_i[port]]; + wrote_result_d[intID][sb_id_i[port]] = sb_wrote_result_i[port_idx] && (!narrow_wide_q[sb_id_i[port]] || wrote_result_narrowing_q[sb_id_i[port]]); + if (vl_cnt_q[sb_id_i[port]] >= (vl_max_d[sb_id_i[port]] * (intID + 1) - VRFWriteSize)) + done_result_d[intID][sb_id_i[port]] = wrote_result_d[intID][sb_id_i[port]]; + end + // VLSU: intID is fixed per interface (0 for WD0, 1 for WD1) + if (port inside {SB_VLSU_VD_WD0, SB_VLSU_VD_WD1}) begin + automatic int unsigned intID = port - SB_VLSU_VD_WD0; + wrote_result_narrowing_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] ^ narrow_wide_q[sb_id_i[port]]; + wrote_result_d[intID][sb_id_i[port]] = sb_wrote_result_i[port_idx] && (!narrow_wide_q[sb_id_i[port]] || wrote_result_narrowing_q[sb_id_i[port]]); + + // Track this interface's own write progress (VLSU interfaces + // write independently/in parallel, not sequentially like + // VFU/VSLDU's shared vl_cnt) and latch a sticky "done" once this + // interface has written its full share. Without this, a chained + // consumer's read gate could only ever pass by coinciding with + // the single-cycle wrote_result_q pulse, which is not guaranteed + // to align with when the consumer's read is actually evaluated + if (sb_wrote_result_i[port_idx]) + vlsu_vl_cnt_d[intID][sb_id_i[port]] += VRFWordBWidth; + // Written as cnt+size >= max rather than cnt >= max-size: vl_max_d + // can be smaller than a single VRFWordBWidth write (e.g. a tiny + // vl=4,e32 request), and max-size would underflow in that case. + if ((vlsu_vl_cnt_q[intID][sb_id_i[port]] + VRFWordBWidth) >= vl_max_d[sb_id_i[port]]) + done_result_d[intID][sb_id_i[port]] = wrote_result_d[intID][sb_id_i[port]]; + end end - - wrote_result_narrowing_d[sb_id_i[SB_VSLDU_VD_WD]] = sb_wrote_result_i[SB_VSLDU_VD_WD - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VSLDU_VD_WD]]; - wrote_result_d[intID][sb_id_i[SB_VSLDU_VD_WD]] = sb_wrote_result_i[SB_VSLDU_VD_WD - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VSLDU_VD_WD]] || wrote_result_narrowing_q[sb_id_i[SB_VSLDU_VD_WD]]); end `else - // Store the decisions - if (sb_enable_o[SB_VFU_VD_WD]) begin - wrote_result_narrowing_d[sb_id_i[SB_VFU_VD_WD]] = sb_wrote_result_i[SB_VFU_VD_WD - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VFU_VD_WD]]; - wrote_result_d[sb_id_i[SB_VFU_VD_WD]] = sb_wrote_result_i[SB_VFU_VD_WD - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VFU_VD_WD]] || wrote_result_narrowing_q[sb_id_i[SB_VFU_VD_WD]]); - end - if (sb_enable_o[SB_VLSU_VD_WD]) begin - wrote_result_narrowing_d[sb_id_i[SB_VLSU_VD_WD]] = sb_wrote_result_i[SB_VLSU_VD_WD - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VLSU_VD_WD]]; - wrote_result_d[sb_id_i[SB_VLSU_VD_WD]] = sb_wrote_result_i[SB_VLSU_VD_WD - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VLSU_VD_WD]] || wrote_result_narrowing_q[sb_id_i[SB_VLSU_VD_WD]]); - end - if (sb_enable_o[SB_VSLDU_VD_WD]) begin - wrote_result_narrowing_d[sb_id_i[SB_VSLDU_VD_WD]] = sb_wrote_result_i[SB_VSLDU_VD_WD - SB_VFU_VD_WD] ^ narrow_wide_q[sb_id_i[SB_VSLDU_VD_WD]]; - wrote_result_d[sb_id_i[SB_VSLDU_VD_WD]] = sb_wrote_result_i[SB_VSLDU_VD_WD - SB_VFU_VD_WD] && (!narrow_wide_q[sb_id_i[SB_VSLDU_VD_WD]] || wrote_result_narrowing_q[sb_id_i[SB_VSLDU_VD_WD]]); + // Store the decisions for all write-destination ports: VFU, VLSU, VSLDU + for (int unsigned port = 0; port < NrVregfilePorts; port++) begin + if (sb_enable_o[port] && (port inside {SB_VFU_VD_WD, SB_VLSU_VD_WD, SB_VSLDU_VD_WD})) begin + automatic int unsigned port_idx = port - SB_VFU_VD_WD; + wrote_result_narrowing_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] ^ narrow_wide_q[sb_id_i[port]]; + wrote_result_d[sb_id_i[port]] = sb_wrote_result_i[port_idx] && (!narrow_wide_q[sb_id_i[port]] || wrote_result_narrowing_q[sb_id_i[port]]); + end end `endif @@ -410,6 +411,8 @@ module spatz_controller done_result_d[0][vlsu_rsp_i.id] = 1'b0; done_result_d[1][vlsu_rsp_i.id] = 1'b0; vl_cnt_d[vlsu_rsp_i.id] = '0; + vlsu_vl_cnt_d[0][vlsu_rsp_i.id] = '0; + vlsu_vl_cnt_d[1][vlsu_rsp_i.id] = '0; `endif for (int unsigned insn = 0; insn < NrParallelInstructions; insn++) scoreboard_d[insn].deps[vlsu_rsp_i.id] = 1'b0; @@ -473,8 +476,10 @@ module spatz_controller // Track request vl for vector chaining, used only for DOUBLE_BW // Default spatz uses 1-bit credit counter using wrote_result_q for chaining - vl_max_d[spatz_req.id] = (spatz_req.vl >> 1) << spatz_req.vtype.vsew; - vl_cnt_d[spatz_req.id] = '0; + vl_max_d[spatz_req.id] = (spatz_req.vl >> 1) << spatz_req.vtype.vsew; + vl_cnt_d[spatz_req.id] = '0; + vlsu_vl_cnt_d[0][spatz_req.id] = '0; + vlsu_vl_cnt_d[1][spatz_req.id] = '0; `endif end @@ -605,8 +610,8 @@ module spatz_controller issue_rsp_o.writeback = spatz_req.use_rd; end // CON VFU: begin - // vtype is illegal -> illegal instruction - if (vtype_q.vill) begin + // Scalar FP ops use the VFU path but do not depend on the vector vtype CSR. + if (vtype_q.vill && !decoder_rsp.spatz_req.op_arith.is_scalar) begin issue_rsp_o.accept = 1'b0; end end // VFU diff --git a/hw/src/spatz_doublebw_vlsu.sv b/hw/src/spatz_doublebw_vlsu.sv index c54ac40..8a83e19 100644 --- a/hw/src/spatz_doublebw_vlsu.sv +++ b/hw/src/spatz_doublebw_vlsu.sv @@ -34,6 +34,7 @@ module spatz_doublebw_vlsu output logic vlsu_rsp_valid_o, output vlsu_rsp_t vlsu_rsp_o, input logic vlsu_buf_empty_i, + input logic vlsu_buf_full_i, // Interface with the VRF output vrf_addr_t [NrInterfaces-1:0] vrf_waddr_o, output vrf_data_t [NrInterfaces-1:0] vrf_wdata_o, @@ -86,6 +87,8 @@ module spatz_doublebw_vlsu logic mem_spatz_req_valid; logic mem_spatz_req_ready; + logic spatz_req_ready; + spill_register #( .T(spatz_req_t) ) i_operation_queue ( @@ -93,7 +96,7 @@ module spatz_doublebw_vlsu .rst_ni (rst_ni ), .data_i (spatz_req_d ), .valid_i(spatz_req_valid_i && spatz_req_i.ex_unit == LSU), - .ready_o(spatz_req_ready_o ), + .ready_o(spatz_req_ready ), .data_o (mem_spatz_req ), .valid_o(mem_spatz_req_valid ), .ready_i(mem_spatz_req_ready ) @@ -125,11 +128,11 @@ module spatz_doublebw_vlsu // Do we have a strided memory access logic mem_is_strided; - assign mem_is_strided = (mem_spatz_req.op == VLSE) || (mem_spatz_req.op == VSSE); + assign mem_is_strided = mem_spatz_req_valid && ((mem_spatz_req.op == VLSE) || (mem_spatz_req.op == VSSE)); // Do we have an indexed memory access logic mem_is_indexed; - assign mem_is_indexed = (mem_spatz_req.op == VLXE) || (mem_spatz_req.op == VSXE); + assign mem_is_indexed = mem_spatz_req_valid && ((mem_spatz_req.op == VLXE) || (mem_spatz_req.op == VSXE)); ///////////// // State // @@ -306,6 +309,9 @@ module spatz_doublebw_vlsu logic [NrParallelInstructions-1:0] mem_insn_pending_q, mem_insn_pending_d; `FF(mem_insn_pending_q, mem_insn_pending_d, '0) + // Is there are pending write request to be sent to the memory + logic write_pending; + /////////////////// // VRF request // /////////////////// @@ -329,7 +335,7 @@ module spatz_doublebw_vlsu logic commit_insn_push; commit_metadata_t commit_insn_q; logic commit_insn_pop; - logic commit_insn_empty; + logic commit_insn_empty, commit_insn_full; logic commit_insn_valid; fifo_v3 #( @@ -343,13 +349,15 @@ module spatz_doublebw_vlsu .testmode_i(1'b0 ), .data_i (commit_insn_d ), .push_i (commit_insn_push ), - .full_o (/* Unused */ ), + .full_o (commit_insn_full ), .data_o (commit_insn_q ), .empty_o (commit_insn_empty), .pop_i (commit_insn_pop ), .usage_o (/* Unused */ ) ); + assign spatz_req_ready_o = spatz_req_ready & !commit_insn_full; + assign commit_insn_valid = !commit_insn_empty; assign commit_insn_d = '{ id : mem_spatz_req.id, @@ -375,13 +383,13 @@ module spatz_doublebw_vlsu commit_insn_push = 1'b0; // Did we start a new instruction? - if (mem_spatz_req_valid && !mem_insn_pending_q[mem_spatz_req.id]) begin + if (mem_spatz_req_valid && !mem_insn_pending_q[mem_spatz_req.id] && !commit_insn_full) begin mem_insn_pending_d[mem_spatz_req.id] = 1'b1; commit_insn_push = 1'b1; end // Did an instruction finished its requests? - if (&(mem_port_finished_q | (mem_port_finished_d & mem_counter_en))) begin + if (&(mem_port_finished_q | (mem_port_finished_d & mem_counter_en)) & !write_pending) begin mem_insn_finished_d[mem_spatz_req.id] = 1'b1; mem_spatz_req_ready = 1'b1; end @@ -486,22 +494,29 @@ module spatz_doublebw_vlsu default: offset = $signed(vrf_rdata_i[intf][1][8 * word_index +: 32]); endcase end else begin - offset = ({mem_counter_q[intf][fu][$bits(vlen_t)-1:MAXEW] << $clog2(N_FU), mem_counter_q[intf][fu][int'(MAXEW)-1:0]} + (fu << MAXEW)) * stride; + offset = ({mem_counter_q[intf][fu][$bits(vlen_t)-1:MAXEW] << $clog2(N_FU), mem_counter_q[intf][fu][int'(MAXEW)-1:0]} + (fu << MAXEW)); end // The second interface starts from half of the vector to straighten the write-back VRF access pattern // To ensure that the 2 interfaces do not also conflict at the TCDM, there is HW scrambling of addresses to TCDM // such that they access different superbanks. - if (!mem_is_indexed && !mem_is_strided && intf == 1) begin + if (!mem_is_indexed && intf == 1) begin // Align the vector length with SpatzMemBytes bytes offset += ((mem_spatz_req.vl + (SpatzMemBytes / 2)) >> $clog2(SpatzMemBytes) << $clog2(SpatzMemBytes)) / 2; end + offset *= stride; - addr = mem_spatz_req.rs1 + offset; + addr = mem_spatz_req.rs1 + offset; mem_req_addr[intf][fu] = (addr >> MAXEW) << MAXEW; mem_req_addr_offset[intf][fu] = addr[int'(MAXEW)-1:0]; - fetch_next_idx[intf][fu] = (mem_idx_counter_q[intf][fu][$clog2(NrWordsPerVector*ELENB)-1:0] == (num_idx_maxew_bytes - (1'b1 << mem_spatz_req.op_mem.ew))) && mem_counter_en[intf][fu]; + // If the index byte counter moved to another VRF word, fetch that word + // from vs2 before issuing the memory request. + if (NrWordsPerVector == 1) begin + fetch_next_idx[intf][fu] = (mem_idx_counter_q[intf][fu] >> MAXEW) != vs2_elem_id_q[intf]; + end else begin + fetch_next_idx[intf][fu] = (mem_idx_counter_q[intf][fu][$clog2(NrWordsPerVector*ELENB)-1:0] >> MAXEW) != vs2_vreg_addr[intf][$clog2(NrWordsPerVector)-1:0]; + end end end: gen_mem_req_addr_intf_fu end: gen_mem_req_addr_intf @@ -647,16 +662,16 @@ module spatz_doublebw_vlsu vlsu_rsp_t rsp; logic rsp_valid; + vlen_t commit_vl; } vrf_req_t; vrf_req_t [NrInterfaces-1:0] vrf_req_d, vrf_req_q; logic [NrInterfaces-1:0] vrf_req_valid_d, vrf_req_ready_d; logic [NrInterfaces-1:0] vrf_req_valid_q, vrf_req_ready_q; - logic [NrInterfaces-1:0] vrf_commit_pending_d, vrf_commit_pending_q, vrf_valid_rsp; + logic [NrInterfaces-1:0] vrf_commit_waiting_d, vrf_commit_waiting_q, vrf_valid_rsp; logic [NrInterfaces-1:0] vrf_commit_intf_valid, vrf_commit_intf_valid_q; - logic vrf_commit_bypass_d, vrf_commit_bypass_q; - `FF(vrf_commit_bypass_q, vrf_commit_bypass_d, 1'b0) + logic vrf_commit_bypass; for (genvar intf = 0; intf < NrInterfaces; intf++) begin : gen_vrf_req_register_intf spill_register #( @@ -675,15 +690,22 @@ module spatz_doublebw_vlsu assign vrf_waddr_o[intf] = vrf_req_q[intf].waddr; assign vrf_wdata_o[intf] = vrf_req_q[intf].wdata; assign vrf_wbe_o[intf] = vrf_req_q[intf].wbe; - // If VLSU0 is the last interface to finish, make sure there is nothing in the VLSU1 buffer to VRF to avoid - // clearing the instruction from controller before all commits to VRF are done for the instruction - // If VLSU1 is last to finish, since it goes through the buffer, we know it will never go ahead of VLSU0 - assign vrf_we_o[intf] = vrf_req_valid_q[intf] & ~vrf_commit_pending_q[intf] & ((intf == 0) && vrf_valid_rsp[0] && !vrf_valid_rsp[1] ? vlsu_buf_empty_i : 1'b1); + // Ensure simpler synchronization for commits from both interfaces + // Writeback: + // For interface 1, check if interface 0 commit can go through + // For interface 0, check if interface1 buffer is not full + // Retiring: + // If Interface 1, is resp interface (usually the default) + // If Interface 0, is resp interface (if interface 0 has more vector elements), then ensure interface 1 has nothing in buffer + // to avoid retiring before interface 1 commits to the VRF + assign vrf_we_o[intf] = ((&vrf_req_valid_q) | ((intf==0) ? vrf_req_valid_q[0] & (vrf_commit_bypass | vrf_commit_waiting_q[1]) & vlsu_buf_empty_i : 1'b0)) & + ((intf==1) ? (vrf_wvalid_i[0] & (vrf_req_q[1].rsp.id == vrf_req_q[0].rsp.id)) : 1'b1) & + !vlsu_buf_full_i; assign vrf_id_o[intf] = {vrf_req_q[intf].rsp.id, mem_spatz_req.id, commit_insn_q.id}; assign vrf_req_ready_q[intf] = vrf_wvalid_i[intf]; `FF(vrf_commit_intf_valid_q[intf], vrf_commit_intf_valid[intf], 1'b0) - `FF(vrf_commit_pending_q[intf], vrf_commit_pending_d[intf], 1'b0) + `FF(vrf_commit_waiting_q[intf], vrf_commit_waiting_d[intf], 1'b0) end ////////////////////////////////////// @@ -693,22 +715,21 @@ module spatz_doublebw_vlsu always_comb begin vrf_valid_rsp = '0; vrf_commit_intf_valid = vrf_commit_intf_valid_q; - vrf_commit_pending_d = vrf_commit_pending_q; - vrf_commit_bypass_d = vrf_commit_bypass_q; + vrf_commit_waiting_d = vrf_commit_waiting_q; // To track if the second interface is committing or not for small vector lengths - vrf_commit_bypass_d = commit_insn_valid ? ((commit_insn_q.vl <= ( SpatzMemBytes / 2)) ? 1'b1 : 1'b0) : vrf_commit_bypass_q; + vrf_commit_bypass = vrf_req_valid_q[0] ? ((vrf_req_q[0].commit_vl <= ( SpatzMemBytes / 2)) ? 1'b1 : 1'b0) : 1'b0; for (int intf = 0; intf < NrInterfaces; intf++) begin // We have a final resp to write to VRF vrf_valid_rsp[intf] = (vrf_req_valid_q[intf] & vrf_req_q[intf].rsp_valid); // Track if the final resp has already been written to the VRF - vrf_commit_intf_valid[intf] = ((vrf_valid_rsp[intf] & vrf_wvalid_i[intf]) | vrf_commit_pending_q[intf]) | (intf == 1 ? vrf_commit_bypass_q : 1'b0); + vrf_commit_intf_valid[intf] = ((vrf_valid_rsp[intf] & vrf_wvalid_i[intf]) | vrf_commit_waiting_q[intf]) | (intf == 1 ? vrf_commit_bypass : 1'b0); // To track if the last packet has already been committed for the interface, but a response cannot be sent to the controller // since waiting for the other interface to finish - vrf_commit_pending_d[intf] = vrf_commit_intf_valid[intf] ? (vlsu_rsp_valid_o ? 1'b0 : 1'b1) : 1'b0; + vrf_commit_waiting_d[intf] = vrf_commit_intf_valid[intf] ? (vlsu_rsp_valid_o ? 1'b0 : 1'b1) : 1'b0; end end @@ -721,7 +742,7 @@ module spatz_doublebw_vlsu // Check if interface 1 is the interface trying to commit, if so take resp information from interface 1 // If both interfaces in sync, interface 1 is given priority - assign resp_intf = vrf_commit_intf_valid_q [1] == 1'b0 ? 1'b1 : 1'b0; + assign resp_intf = ((vrf_commit_intf_valid [1] == 1'b1) & !vrf_commit_bypass) ? 1'b1 : 1'b0; // Check is both interfaces has reached to a completion and if the last write to the VRF is also done // Assign the instruction id from the interface that completes the last @@ -833,6 +854,14 @@ module spatz_doublebw_vlsu always_comb begin: p_state // Maintain state state_d = state_q; + write_pending = 1'b0; + + for (int intf = 0; intf < NrInterfaces; intf++) begin + for (int fu = 0; fu < N_FU; fu++) begin + automatic int unsigned port = intf * N_FU + fu; + write_pending |= (store_count_d[intf][fu] != '0); + end + end unique case (state_q) VLSU_RunningLoad: begin @@ -844,7 +873,8 @@ module spatz_doublebw_vlsu VLSU_RunningStore: begin if (commit_insn_valid && commit_insn_q.is_load) if (&rob_empty) - state_d = VLSU_RunningLoad; + if (!write_pending) + state_d = VLSU_RunningLoad; end default:; @@ -911,9 +941,14 @@ module spatz_doublebw_vlsu vrf_req_d[intf].rsp.id = commit_insn_q.id; vrf_req_d[intf].rsp.intf_id = intf; vrf_req_d[intf].rsp_valid = commit_insn_valid && &commit_finished_d[intf] && mem_insn_finished_d[commit_insn_q.id]; + vrf_req_d[intf].commit_vl = commit_insn_q.vl; - // Request indexes - vrf_re_o[intf][1] = mem_is_indexed; + // Request indexes. Once this interface has no more elements left to + // process, stop requesting: otherwise this interface's continuous + // index-VRF read request keeps winning the shared read port against + // the other interface (which may still have genuine outstanding + // work), starving it of vrf_rvalid_i forever. + vrf_re_o[intf][1] = mem_is_indexed && |mem_operation_valid[intf]; // Count which vs2 element we should load (indexed loads) vs2_elem_id_d = vs2_elem_id_q; @@ -1014,7 +1049,7 @@ module spatz_doublebw_vlsu `endif if (!rob_full[intf][fu] && !offset_queue_full[intf][fu] && mem_operation_valid[intf][fu]) begin rob_req_id[intf][fu] = spatz_mem_req_ready[intf][fu] & spatz_mem_req_valid[intf][fu]; - mem_req_lvalid[intf][fu] = (!mem_is_indexed || vrf_rvalid_i[intf][1]) && mem_spatz_req.op_mem.is_load; + mem_req_lvalid[intf][fu] = (!mem_is_indexed || (vrf_rvalid_i[intf][1] & !fetch_next_idx[intf][fu])) && mem_spatz_req.op_mem.is_load; mem_req_id[intf][fu] = rob_id[intf][fu]; mem_req_last[intf][fu] = mem_operation_last[intf][fu]; end @@ -1081,7 +1116,7 @@ module spatz_doublebw_vlsu default: mem_req_data[intf][fu] = data; endcase - mem_req_svalid[intf][fu] = rob_rvalid[intf][fu] && (!mem_is_indexed || vrf_rvalid_i[intf][1]) && !mem_spatz_req.op_mem.is_load; + mem_req_svalid[intf][fu] = rob_rvalid[intf][fu] && (!mem_is_indexed || (vrf_rvalid_i[intf][1] & !fetch_next_idx[intf][fu])) && !mem_spatz_req.op_mem.is_load; mem_req_id[intf][fu] = rob_rid[intf][fu]; mem_req_last[intf][fu] = mem_operation_last[intf][fu]; rob_pop[intf][fu] = spatz_mem_req_valid[intf][fu] && spatz_mem_req_ready[intf][fu]; @@ -1143,7 +1178,7 @@ module spatz_doublebw_vlsu `else assign spatz_mem_req[intf][fu].addr = mem_req_addr[intf][fu]; assign spatz_mem_req[intf][fu].write = !mem_is_load; - assign spatz_mem_req[intf][fu].amo = snitch_pkg::AMONone; + assign spatz_mem_req[intf][fu].amo = reqrsp_pkg::AMONone; assign spatz_mem_req[intf][fu].data = mem_req_data[intf][fu]; assign spatz_mem_req[intf][fu].strb = mem_req_strb[intf][fu]; assign spatz_mem_req[intf][fu].user = '0; diff --git a/hw/src/spatz_fpu_sequencer.sv b/hw/src/spatz_fpu_sequencer.sv index e21eaad..543267e 100644 --- a/hw/src/spatz_fpu_sequencer.sv +++ b/hw/src/spatz_fpu_sequencer.sv @@ -98,7 +98,7 @@ module spatz_fpu_sequencer .AddrWidth ( FPRRegWidth ), .NrReadPorts ( 3 ), .NrWritePorts ( NrWritePorts ), - .ZeroRegZero ( 1 ) + .ZeroRegZero ( 0 ) ) i_fpr ( .clk_i (clk_i ), .rst_ni (rst_ni ), @@ -560,6 +560,7 @@ module spatz_fpu_sequencer .lsu_qamo_i (fp_lsu_qamo ), .lsu_qrepd_i (1'b0 ), .lsu_quser_i ('0 ), + .lsu_qmcast_i ('0 ), .lsu_qvalid_i (fp_lsu_qvalid ), .lsu_qready_o (fp_lsu_qready ), // Response interface diff --git a/hw/src/spatz_pkg.sv.tpl b/hw/src/spatz_pkg.sv.tpl index 63a0aa4..2853fa9 100644 --- a/hw/src/spatz_pkg.sv.tpl +++ b/hw/src/spatz_pkg.sv.tpl @@ -101,6 +101,11 @@ package spatz_pkg; // Encodes both the scalar RD and the VD address in the VRF localparam int VFURespAddrWidth = GPRWidth > $clog2(NrVRFWords) ? GPRWidth : $clog2(NrVRFWords); + localparam int PaceDegree = 2; + localparam int PaceParts = 16; + localparam int PaceEps = 1; + localparam int PaceDataWidth = 32; + localparam int PaceParamWidth = ((PaceDegree + 1)*PaceParts + PaceParts - 1 + 2*PaceEps) * PaceDataWidth; ////////////////////// // Type Definitions // @@ -425,7 +430,8 @@ package spatz_pkg; // INT8 INT16 INT32 INT64 IntFmtMask : {1'b1, 1'b1, 1'b1, 1'b1}, MxFpFmtMask : 9'b0, - MxIntFmtMask : 4'b0 + MxIntFmtMask : 4'b0, + PaceFeatures : '{PaceDegree: PaceDegree, PaceParts: PaceParts, PaceEps: 1'b1, PaceDataWidth: PaceDataWidth, PaceParamWidth: PaceParamWidth, PaceBstPipeRegs: 4'b0100, FmtConfig: 9'b101010000} } : // Single Precision FPU '{ @@ -437,7 +443,8 @@ package spatz_pkg; // INT8 INT16 INT32 INT64 IntFmtMask : {1'b1, 1'b1, 1'b1, 1'b0}, MxFpFmtMask : 9'b0, - MxIntFmtMask : 4'b0 + MxIntFmtMask : 4'b0, + PaceFeatures : '{PaceDegree: PaceDegree, PaceParts: PaceParts, PaceEps: 1'b1, PaceDataWidth: PaceDataWidth, PaceParamWidth: PaceParamWidth, PaceBstPipeRegs: 4'b0100, FmtConfig: 9'b101010000} }; % if cfg['mempool']: diff --git a/hw/src/spatz_vfu.sv b/hw/src/spatz_vfu.sv index 4c26d09..e2ecbf8 100644 --- a/hw/src/spatz_vfu.sv +++ b/hw/src/spatz_vfu.sv @@ -14,7 +14,8 @@ module spatz_vfu import cf_math_pkg::idx_width; import fpnew_pkg::*; #( /// FPU configuration. - parameter fpu_implementation_t FPUImplementation = fpu_implementation_t'(0) + parameter fpu_implementation_t FPUImplementation = fpu_implementation_t'(0), + localparam type pace_param_t = logic [PaceParamWidth-1:0] ) ( input logic clk_i, input logic rst_ni, @@ -39,6 +40,8 @@ module spatz_vfu input vrf_data_t [2:0] vrf_rdata_i, input logic [2:0] vrf_rvalid_i, // FPU side channel + input pace_param_t pace_param_i, + input pace_mode_t pace_mode_i, output status_t fpu_status_o ); @@ -1060,7 +1063,9 @@ module spatz_vfu fpu_op = fpnew_pkg::ADD; fpu_op_mode = 1'b1; end - VFMUL : fpu_op = fpnew_pkg::MUL; + // Use PWPA (piecewise polynomial approximation) when PACE is enabled; + // otherwise regular FP multiply. + VFMUL : fpu_op = pace_mode_i.enable ? fpnew_pkg::PWPA : fpnew_pkg::MUL; VFMADD : fpu_op = fpnew_pkg::FMADD; VFMSUB : begin fpu_op = fpnew_pkg::FMADD; @@ -1205,6 +1210,8 @@ module spatz_vfu .flush_i (1'b0 ), .busy_o (fpu_busy_d[fpu] ), .operands_i ({fpu_operand3_q, fpu_operand2_q, fpu_operand1_q} ), + .pace_param_i (pace_param_i ), + .pace_mode_i (pace_mode_i ), // Only the FPU0 executes scalar instructions .in_valid_i (fpu_in_valid_q ), .in_ready_o (fpu_in_ready_d ), diff --git a/hw/src/spatz_vlsu.sv b/hw/src/spatz_vlsu.sv index 5a6b7b6..7506c90 100644 --- a/hw/src/spatz_vlsu.sv +++ b/hw/src/spatz_vlsu.sv @@ -79,6 +79,8 @@ module spatz_vlsu logic mem_spatz_req_valid; logic mem_spatz_req_ready; + logic spatz_req_ready; + spill_register #( .T(spatz_req_t) ) i_operation_queue ( @@ -86,7 +88,7 @@ module spatz_vlsu .rst_ni (rst_ni ), .data_i (spatz_req_d ), .valid_i(spatz_req_valid_i && spatz_req_i.ex_unit == LSU), - .ready_o(spatz_req_ready_o ), + .ready_o(spatz_req_ready ), .data_o (mem_spatz_req ), .valid_o(mem_spatz_req_valid ), .ready_i(mem_spatz_req_ready ) @@ -116,13 +118,17 @@ module spatz_vlsu endcase end: proc_spatz_req + // Only do the judgement when we have a valid instruction + // This is used to protect false triggeering the counter in some corner cases // Do we have a strided memory access logic mem_is_strided; - assign mem_is_strided = (mem_spatz_req.op == VLSE) || (mem_spatz_req.op == VSSE); + assign mem_is_strided = mem_spatz_req_valid && + ((mem_spatz_req.op == VLSE) || (mem_spatz_req.op == VSSE)); // Do we have an indexed memory access logic mem_is_indexed; - assign mem_is_indexed = (mem_spatz_req.op == VLXE) || (mem_spatz_req.op == VSXE); + assign mem_is_indexed = mem_spatz_req_valid && + ((mem_spatz_req.op == VLXE) || (mem_spatz_req.op == VSXE)); ///////////// // State // @@ -228,7 +234,7 @@ module spatz_vlsu ////////////////////// // Is the memory operation valid and are we at the last one? - logic [NrMemPorts-1:0] mem_operation_valid; + logic [NrMemPorts-1:0] mem_operation_valid, mem_idx_vrf_fetch_pending; logic [NrMemPorts-1:0] mem_operation_last; // For each memory port we count how many elements we have already loaded/stored. @@ -287,6 +293,9 @@ module spatz_vlsu logic [NrParallelInstructions-1:0] mem_insn_pending_q, mem_insn_pending_d; `FF(mem_insn_pending_q, mem_insn_pending_d, '0) + // Is there are pending write request to be sent to the memory + logic write_pending; + /////////////////// // VRF request // /////////////////// @@ -310,7 +319,7 @@ module spatz_vlsu logic commit_insn_push; commit_metadata_t commit_insn_q; logic commit_insn_pop; - logic commit_insn_empty; + logic commit_insn_empty, commit_insn_full; logic commit_insn_valid; fifo_v3 #( @@ -324,7 +333,7 @@ module spatz_vlsu .testmode_i(1'b0 ), .data_i (commit_insn_d ), .push_i (commit_insn_push ), - .full_o (/* Unused */ ), + .full_o (commit_insn_full ), .data_o (commit_insn_q ), .empty_o (commit_insn_empty), .pop_i (commit_insn_pop ), @@ -344,6 +353,8 @@ module spatz_vlsu is_indexed: mem_is_indexed }; + assign spatz_req_ready_o = spatz_req_ready & !commit_insn_full; + always_comb begin: queue_control // Maintain state mem_insn_finished_d = mem_insn_finished_q; @@ -356,13 +367,13 @@ module spatz_vlsu commit_insn_push = 1'b0; // Did we start a new instruction? - if (mem_spatz_req_valid && !mem_insn_pending_q[mem_spatz_req.id]) begin + if (mem_spatz_req_valid && !mem_insn_pending_q[mem_spatz_req.id] && !commit_insn_full) begin mem_insn_pending_d[mem_spatz_req.id] = 1'b1; commit_insn_push = 1'b1; end // Did an instruction finished its requests? - if (&mem_port_finished_q) begin + if (&mem_port_finished_q & !write_pending) begin mem_insn_finished_d[mem_spatz_req.id] = 1'b1; mem_spatz_req_ready = 1'b1; end @@ -469,9 +480,13 @@ module spatz_vlsu mem_req_addr[port] = (addr >> MAXEW) << MAXEW; mem_req_addr_offset[port] = addr[int'(MAXEW)-1:0]; - // If the request has been pushed into the memory request fifo and this is the last offset - // update address for index to fetch the next address in the following cycle - fetch_next_idx[port] = (mem_idx_counter_q[port][$clog2(NrWordsPerVector*ELENB)-1:0] == (num_idx_maxew_bytes - (1'b1 << mem_spatz_req.op_mem.ew))) && mem_counter_en[port]; + // If the index byte counter moved to another VRF word, fetch that word + // from vs2 before issuing the memory request. + if (NrWordsPerVector == 1) begin + fetch_next_idx[port] = (mem_idx_counter_q[port] >> MAXEW) != vs2_elem_id_q; + end else begin + fetch_next_idx[port] = (mem_idx_counter_q[port][$clog2(NrWordsPerVector*ELENB)-1:0] >> MAXEW) != (vs2_vreg_addr & ((1'b1 << $clog2(NrWordsPerVector))-1)); + end end end: gen_mem_req_addr @@ -553,6 +568,10 @@ module spatz_vlsu logic [3:0] mem_idx_single_element_size; assign mem_idx_single_element_size = 1'b1 << mem_spatz_req.op_mem.ew; + // Number of bytes in the index vector for an indexed memory operation + vlen_t mem_idx_vl; + assign mem_idx_vl = (mem_spatz_req.vl >> mem_spatz_req.vtype.vsew) << mem_spatz_req.op_mem.ew; + // Is the memory address unaligned logic commit_is_addr_unaligned; assign commit_is_addr_unaligned = commit_insn_q.rs1[int'(MAXEW)-1:0] != '0; @@ -632,6 +651,67 @@ module spatz_vlsu assign vlsu_rsp_o = vrf_req_q.rsp_valid && vrf_req_valid_q ? vrf_req_q.rsp : '{id: commit_insn_q.id, default: '0}; assign vlsu_rsp_valid_o = vrf_req_q.rsp_valid && vrf_req_valid_q ? vrf_req_ready_q : vlsu_finished_req && !commit_insn_q.is_load; +`ifndef TARGET_SYNTHESIS +`ifdef TRACE + // pragma translate_off + int trace_vrf_wb_fd; + int trace_mem_fd; + string trace_vrf_wb_file; + string trace_mem_file; + + initial begin + trace_vrf_wb_file = "spatz_vlsu_vrf_wb.log"; + trace_vrf_wb_fd = $fopen(trace_vrf_wb_file, "w"); + + trace_mem_file = "spatz_vlsu_mem_trace.log"; + trace_mem_fd = $fopen(trace_mem_file, "w"); + end + + always_ff @(posedge clk_i) begin + if (rst_ni && vrf_req_valid_q && vrf_req_ready_q) begin + if (trace_vrf_wb_fd != 0) begin + $fdisplay(trace_vrf_wb_fd, + "[spatz_vlsu] vrf_wb id=%0d waddr=0x%0h wbe=0x%0h wdata=0x%0h", + vrf_req_q.rsp.id, vrf_req_q.waddr, vrf_req_q.wbe, vrf_req_q.wdata); + end + end + end + + always_ff @(posedge clk_i) begin + if (rst_ni && trace_mem_fd != 0) begin + for (int unsigned port = 0; port < NrMemPorts; port++) begin + if (spatz_mem_req_valid_o[port] && spatz_mem_req_ready_i[port]) begin + if (spatz_mem_req_o[port].write) begin + $fdisplay(trace_mem_fd, + "[spatz_vlsu] mem_req port=%0d write=1 addr=0x%0h data=0x%0h", + port, spatz_mem_req_o[port].addr, spatz_mem_req_o[port].data); + end else begin + $fdisplay(trace_mem_fd, + "[spatz_vlsu] mem_req port=%0d write=0 addr=0x%0h", + port, spatz_mem_req_o[port].addr); + end + end + +`ifdef MEMPOOL_SPATZ + if (spatz_mem_rsp_valid_i[port]) begin + $fdisplay(trace_mem_fd, + "[spatz_vlsu] mem_rsp port=%0d write=%0d data=0x%0h", + port, spatz_mem_rsp_i[port].write, spatz_mem_rsp_i[port].data); + end +`else + if (spatz_mem_rsp_valid_i[port]) begin + $fdisplay(trace_mem_fd, + "[spatz_vlsu] mem_rsp port=%0d data=0x%0h", + port, spatz_mem_rsp_i[port].data); + end +`endif + end + end + end + // pragma translate_on +`endif +`endif + ////////////// // Counters // ////////////// @@ -676,7 +756,7 @@ module spatz_vlsu for (genvar port = 0; port < NrMemPorts; port++) begin: gen_mem_counter_proc // The total amount of elements we have to work through - vlen_t max_elements; + vlen_t max_elements, max_idx_elements; always_comb begin // Default value @@ -706,6 +786,29 @@ module spatz_vlsu mem_counter_max[port] = max_elements; // Index counter + + // The following logic is intended to decide if indices need to be fetched from the VRF + // The logic is as follows: + // We calculate how many indices are expected to be fetched from the VRF for this port + // This signal gives more flexibility in index fetching especially if SEW and index width are different + + // E.g. vl=2 SEW=16 index_width=32 and 32b config + // port 0 is the only port sending memory requests + // But index needs to be fetched corresponding to 2 ports - port 0 and port 1 + // This signal is used to decide if we need to fetch next index or not + + max_idx_elements = (mem_idx_vl >> $clog2(NrMemPorts*MemDataWidthB)) << $clog2(MemDataWidthB); + + if (NrMemPorts == 1) + max_idx_elements = mem_idx_vl; + else + if (mem_idx_vl[$clog2(MemDataWidthB) +: $clog2(NrMemPorts)] > port) + max_idx_elements += MemDataWidthB; + else if (mem_idx_vl[$clog2(MemDataWidthB) +: $clog2(NrMemPorts)] == port) + max_idx_elements += mem_idx_vl[$clog2(MemDataWidthB)-1:0]; + + mem_idx_vrf_fetch_pending[port] = mem_spatz_req_valid && (max_idx_elements != mem_idx_counter_q[port]); + mem_idx_counter_d[port] = mem_counter_d[port]; mem_idx_counter_delta[port] = !mem_operation_valid[port] ? 'd0 : mem_idx_single_element_size; end @@ -718,6 +821,11 @@ module spatz_vlsu always_comb begin: p_state // Maintain state state_d = state_q; + write_pending = 1'b0; + + for (int port = 0; port < NrMemPorts; port++) begin + write_pending |= (spatz_mem_req_o[port].write & spatz_mem_req_valid_o[port]); + end unique case (state_q) VLSU_RunningLoad: begin @@ -729,7 +837,8 @@ module spatz_vlsu VLSU_RunningStore: begin if (commit_insn_valid && commit_insn_q.is_load) if (&rob_empty) - state_d = VLSU_RunningLoad; + if (!write_pending) + state_d = VLSU_RunningLoad; end default:; @@ -798,8 +907,12 @@ module spatz_vlsu // Count which vs2 element we should load (indexed loads) vs2_elem_id_d = vs2_elem_id_q; - if (&(fetch_next_idx ^ ~mem_operation_valid) && mem_is_indexed) + + // Signal to fetch the next index from the VRF if we are not at the last element and we have an indexed memory operation + // We check the fetch_next_idx signal only if there is a pending fetch from the VRF + if (&(~mem_idx_vrf_fetch_pending | fetch_next_idx) && mem_is_indexed) begin vs2_elem_id_d = vs2_elem_id_q + 1; + end if (mem_spatz_req_ready) vs2_elem_id_d = '0; @@ -888,7 +1001,11 @@ module spatz_vlsu `endif if (!rob_full[port] && !offset_queue_full[port] && mem_operation_valid[port]) begin rob_req_id[port] = spatz_mem_req_ready[port] & spatz_mem_req_valid[port]; - mem_req_lvalid[port] = (!mem_is_indexed || vrf_rvalid_i[1]) && mem_spatz_req.op_mem.is_load; + + // If we have an index oepration, we ensure we are not fetching the next index + // This is because fetch_next_idx uses mem_idx_counter_q signal and requires a 1 cycle bubble before issuing the next index operation + // This could be optimized but index operations are anyways slow and not performance critical + mem_req_lvalid[port] = (!mem_is_indexed || (vrf_rvalid_i[1] && (!(fetch_next_idx[port] && mem_idx_vrf_fetch_pending[port])))) && mem_spatz_req.op_mem.is_load; mem_req_id[port] = rob_id[port]; mem_req_last[port] = mem_operation_last[port]; end @@ -953,7 +1070,7 @@ module spatz_vlsu default: mem_req_data[port] = data; endcase - mem_req_svalid[port] = rob_rvalid[port] && (!mem_is_indexed || vrf_rvalid_i[1]) && !mem_spatz_req.op_mem.is_load; + mem_req_svalid[port] = rob_rvalid[port] && (!mem_is_indexed || (vrf_rvalid_i[1] && (!(fetch_next_idx[port] && mem_idx_vrf_fetch_pending[port])))) && !mem_spatz_req.op_mem.is_load; mem_req_id[port] = rob_rid[port]; mem_req_last[port] = mem_operation_last[port]; rob_pop[port] = spatz_mem_req_valid[port] && spatz_mem_req_ready[port]; @@ -1011,7 +1128,7 @@ module spatz_vlsu `else assign spatz_mem_req[port] = '{addr: mem_req_addr[port], write: !mem_is_load, - amo: 4'h0, // snitch_pkg::AMONone + amo: reqrsp_pkg::AMONone, data: mem_req_data[port], strb: mem_req_strb[port], default: '0}; diff --git a/hw/src/spatz_vrf.sv b/hw/src/spatz_vrf.sv index a4efffb..8eefd0e 100644 --- a/hw/src/spatz_vrf.sv +++ b/hw/src/spatz_vrf.sv @@ -108,7 +108,7 @@ module spatz_vrf for (int unsigned bank = 0; bank < NrVRFBanks; bank++) begin `ifdef BUF_FPU `ifdef DOUBLE_BW - automatic logic write_request_vlsu = write_request[bank][VLSU_VD_WD0] | write_request[bank][VLSU_VD_WD1]; + automatic logic write_request_vlsu = write_request[bank][VLSU_VD_WD0]; `else automatic logic write_request_vlsu = write_request[bank][VLSU_VD_WD]; `endif