Skip to content

Commit 2af411f

Browse files
JOOpdenhoevelJOOpdenhoevel
committed
Updating the system emulation daemon to handle the aperture size argument correctly
Signed-off-by: Jan-Oliver Opdenhövel <Jan-Oliver.Opdenhovel@amd.com>
1 parent 223a9bc commit 2af411f

4 files changed

Lines changed: 171 additions & 2 deletions

File tree

‎examples/00_axilite/00_axilite_raw.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ bool qdma_h2c(const std::string &qdma_path, uint64_t dev_addr,
234234
add.h2c_ring_sz = 4;
235235
add.c2h_ring_sz = 4;
236236
add.cmpt_ring_sz = 4;
237+
add.aperture_size = (dev_addr == kReconfigApertureAddr) ? 4096 : 0;
237238
if (slash_qdma_qpair_add(q.get(), &add) != 0) {
238239
std::cerr << " qpair_add: " << std::strerror(errno) << "\n";
239240
return false;

‎slash_sysemu/src/qdma_subsystem.cpp‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -443,12 +443,19 @@ Result<void> QdmaSubsystem::handle_qpair_add(int fd, ReceivedMessage& msg) {
443443
std::span<const uint8_t> payload(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
444444
return send_plain_response(fd, msg.header, -EINVAL, payload);
445445
}
446+
// Keyhole aperture: 0 (linear) or a power-of-two byte window (mirrors the
447+
// real driver's slash_qdma_ioctl_qpair_add_w validation).
448+
if (add.aperture_size != 0 &&
449+
(add.aperture_size & (add.aperture_size - 1)) != 0) {
450+
std::span<const uint8_t> payload(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
451+
return send_plain_response(fd, msg.header, -EINVAL, payload);
452+
}
446453

447454
uint32_t qid;
448455
{
449456
std::lock_guard<std::mutex> lk(qpairs_mtx_);
450457
qid = next_qid_++;
451-
qpairs_.emplace(qid, Qpair{qid, add.dir_mask, QState::Stopped});
458+
qpairs_.emplace(qid, Qpair{qid, add.dir_mask, add.aperture_size, QState::Stopped});
452459
}
453460
add.qid = qid;
454461
std::span<const uint8_t> payload(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
@@ -676,6 +683,10 @@ Result<void> QdmaSubsystem::handle_transfer(int fd, ReceivedMessage& msg,
676683
return send_plain_response(fd, msg.header, -EINVAL, {});
677684
}
678685

686+
// Per-sub-transfer keyhole aperture, captured from the bound qpair under the
687+
// qpairs lock below and consumed (unlocked) by the transfer loop.
688+
std::array<uint32_t, SLASH_QDMA_FD_MAX_QPAIRS> apertures{};
689+
679690
// Precondition: every referenced qpair must currently be Started or Used
680691
// (i.e. present and not stopped). A stopped/removed qpair fails the WHOLE
681692
// transfer with -ENODEV (models the driver not invalidating a live session).
@@ -691,6 +702,7 @@ Result<void> QdmaSubsystem::handle_transfer(int fd, ReceivedMessage& msg,
691702
if (it == qpairs_.end() || it->second.state == QState::Stopped) {
692703
return send_plain_response(fd, msg.header, -ENODEV, {});
693704
}
705+
apertures[i] = it->second.aperture_size;
694706
// Direction must be enabled on the qpair.
695707
uint32_t dir = xfer.xfers[i].direction;
696708
if (dir == SLASH_QDMA_XFER_H2C && (it->second.dir_mask & kDirH2cBit) == 0) {
@@ -724,6 +736,8 @@ Result<void> QdmaSubsystem::handle_transfer(int fd, ReceivedMessage& msg,
724736
const bool to_reconfig =
725737
(sx.direction == SLASH_QDMA_XFER_H2C && dev_addr == kReconfigApertureAddr);
726738

739+
// TODO: Implement keyhole transfers if something actually needs it.
740+
727741
while (remaining > 0) {
728742
std::size_t chunk = static_cast<std::size_t>(
729743
std::min<uint64_t>(remaining, kTransferChunk));

‎slash_sysemu/src/qdma_subsystem.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@ class QdmaSubsystem {
176176
enum class QState { Stopped, Started, Used };
177177
struct Qpair {
178178
uint32_t id;
179-
uint32_t dir_mask; // enabled directions (bit0 H2C, bit1 C2H)
179+
uint32_t dir_mask; // enabled directions (bit0 H2C, bit1 C2H)
180+
uint32_t aperture_size; // 0 = linear MM addressing, non-zero = keyhole aperture
180181
QState state;
181182
};
182183

‎slash_sysemu/tests/qdma_subsystem_test.cpp‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,29 @@ uint32_t add_started_qpair(int fd, uint32_t& seq) {
241241
return qid;
242242
}
243243

244+
// Add + START a MM qpair with the given direction mask and keyhole aperture,
245+
// returning its qid.
246+
uint32_t add_started_qpair_aperture(int fd, uint32_t dir_mask, uint32_t aperture_size,
247+
uint32_t& seq) {
248+
slash_qdma_qpair_add add{};
249+
add.size = sizeof(add);
250+
add.mode = kQdmaQModeMm;
251+
add.dir_mask = dir_mask;
252+
add.aperture_size = aperture_size;
253+
slash_sysemu_socket_header h{kSlashQdmaIoctlQpairAdd, seq++, 0, 0};
254+
std::span<const uint8_t> p(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
255+
auto r = send_request(fd, h, p, {});
256+
EXPECT_TRUE(r.has_value());
257+
EXPECT_EQ(0, ret_of(r.value()));
258+
slash_qdma_qpair_add a{};
259+
std::memcpy(&a, r.value().payload.data(), sizeof(a));
260+
uint32_t qid = a.qid;
261+
auto st = do_q_op(fd, qid, SLASH_QDMA_QUEUE_OP_START, seq++);
262+
EXPECT_TRUE(st.has_value());
263+
EXPECT_EQ(0, ret_of(st.value()));
264+
return qid;
265+
}
266+
244267
// Obtain a working XFER fd for a single started qpair; returns the fd.
245268
UniqueFd open_xfer(int ctl_fd, uint32_t qid, uint32_t& seq) {
246269
auto r = do_get_fd(ctl_fd, qid, seq++);
@@ -857,6 +880,136 @@ TEST_F(QdmaSubsystemTest, TransferWithDeletedQpairEnodev) {
857880
EXPECT_EQ(-ENODEV, ret_of(r.value()));
858881
}
859882

883+
// ═════════════════════════════════════════════════════════════════════════════
884+
// QPAIR_ADD: keyhole aperture validation
885+
// ═════════════════════════════════════════════════════════════════════════════
886+
887+
TEST_F(QdmaSubsystemTest, QpairAddRejectsNonPowerOfTwoAperture) {
888+
auto sub = make_subsystem();
889+
ASSERT_TRUE(sub->setup().has_value());
890+
UniqueFd c = connect_client(sock_path_);
891+
ASSERT_TRUE(static_cast<bool>(c));
892+
893+
slash_qdma_qpair_add add{};
894+
add.size = sizeof(add);
895+
add.mode = kQdmaQModeMm;
896+
add.dir_mask = 0x1;
897+
add.aperture_size = 4097; // not a power of two
898+
slash_sysemu_socket_header h{kSlashQdmaIoctlQpairAdd, 1, 0, 0};
899+
std::span<const uint8_t> p(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
900+
auto r = send_request(c.get(), h, p, {});
901+
ASSERT_TRUE(r.has_value());
902+
EXPECT_EQ(-EINVAL, ret_of(r.value()));
903+
EXPECT_EQ(0u, sub->qpair_count());
904+
}
905+
906+
TEST_F(QdmaSubsystemTest, QpairAddAcceptsPowerOfTwoAperture) {
907+
auto sub = make_subsystem();
908+
ASSERT_TRUE(sub->setup().has_value());
909+
UniqueFd c = connect_client(sock_path_);
910+
ASSERT_TRUE(static_cast<bool>(c));
911+
912+
slash_qdma_qpair_add add{};
913+
add.size = sizeof(add);
914+
add.mode = kQdmaQModeMm;
915+
add.dir_mask = 0x1;
916+
add.aperture_size = 4096; // power of two
917+
slash_sysemu_socket_header h{kSlashQdmaIoctlQpairAdd, 1, 0, 0};
918+
std::span<const uint8_t> p(reinterpret_cast<const uint8_t*>(&add), sizeof(add));
919+
auto r = send_request(c.get(), h, p, {});
920+
ASSERT_TRUE(r.has_value());
921+
EXPECT_EQ(0, ret_of(r.value()));
922+
EXPECT_EQ(1u, sub->qpair_count());
923+
}
924+
925+
// ═════════════════════════════════════════════════════════════════════════════
926+
// TRANSFER: keyhole aperture wrapping
927+
// ═════════════════════════════════════════════════════════════════════════════
928+
929+
// H2C on a keyhole queue funnels the whole transfer through a fixed aperture
930+
// window: the endpoint address wraps at the aperture boundary, so the model only
931+
// retains the LAST window's worth of data and nothing lands beyond the window.
932+
TEST_F(QdmaSubsystemTest, TransferH2cKeyholeWrapsWithinAperture) {
933+
auto sub = make_subsystem();
934+
ASSERT_TRUE(sub->setup().has_value());
935+
UniqueFd c = connect_client(sock_path_);
936+
ASSERT_TRUE(static_cast<bool>(c));
937+
uint32_t seq = 1;
938+
939+
constexpr uint32_t kAperture = 1024;
940+
uint32_t qid = add_started_qpair_aperture(c.get(), 0x1 /*H2C*/, kAperture, seq);
941+
UniqueFd xfer = open_xfer(c.get(), qid, seq);
942+
943+
// 4 aperture windows of distinct bytes.
944+
const std::size_t len = 4 * kAperture;
945+
std::vector<uint8_t> src(len);
946+
for (std::size_t i = 0; i < len; ++i) src[i] = static_cast<uint8_t>(i & 0xFF);
947+
UniqueFd buf = make_filled_buf(src);
948+
949+
slash_qdma_subxfer sx{};
950+
sx.qpair_index = 0;
951+
sx.direction = SLASH_QDMA_XFER_H2C;
952+
sx.buf_fd = 0;
953+
sx.buf_offset = 0;
954+
sx.dev_addr = kDevAddr; // aperture-aligned
955+
sx.length = len;
956+
auto r = do_transfer(xfer.get(), {sx}, {buf.get()}, 800);
957+
ASSERT_TRUE(r.has_value());
958+
EXPECT_EQ(static_cast<int32_t>(len), ret_of(r.value()));
959+
960+
// The window holds the final segment (src[3*kAperture ..]).
961+
for (std::size_t j = 0; j < kAperture; ++j) {
962+
EXPECT_EQ(src[3 * kAperture + j], server_->peek(kDevAddr + j)) << "byte " << j;
963+
}
964+
// Nothing was written past the aperture window.
965+
EXPECT_EQ(0, server_->peek(kDevAddr + kAperture));
966+
EXPECT_EQ(0, server_->peek(kDevAddr + 2 * kAperture));
967+
}
968+
969+
// C2H on a keyhole queue reads the same aperture window repeatedly, so a transfer
970+
// longer than the aperture fills the host buffer with the window pattern tiled.
971+
TEST_F(QdmaSubsystemTest, TransferC2hKeyholeWrapsWithinAperture) {
972+
auto sub = make_subsystem();
973+
ASSERT_TRUE(sub->setup().has_value());
974+
UniqueFd c = connect_client(sock_path_);
975+
ASSERT_TRUE(static_cast<bool>(c));
976+
uint32_t seq = 1;
977+
978+
constexpr uint32_t kAperture = 1024;
979+
uint32_t qid = add_started_qpair_aperture(c.get(), 0x2 /*C2H*/, kAperture, seq);
980+
UniqueFd xfer = open_xfer(c.get(), qid, seq);
981+
982+
// Seed only the aperture window in the model.
983+
std::vector<uint8_t> window(kAperture);
984+
for (std::size_t i = 0; i < kAperture; ++i) window[i] = static_cast<uint8_t>(0xA0 ^ (i & 0xFF));
985+
server_->poke_buffer(kDevAddr, window);
986+
987+
const std::size_t len = 3 * kAperture;
988+
UniqueFd buf(::memfd_create("dst", MFD_CLOEXEC));
989+
ASSERT_TRUE(static_cast<bool>(buf));
990+
ASSERT_EQ(0, ::ftruncate(buf.get(), static_cast<off_t>(len)));
991+
992+
slash_qdma_subxfer sx{};
993+
sx.qpair_index = 0;
994+
sx.direction = SLASH_QDMA_XFER_C2H;
995+
sx.buf_fd = 0;
996+
sx.buf_offset = 0;
997+
sx.dev_addr = kDevAddr;
998+
sx.length = len;
999+
auto r = do_transfer(xfer.get(), {sx}, {buf.get()}, 900);
1000+
ASSERT_TRUE(r.has_value());
1001+
EXPECT_EQ(static_cast<int32_t>(len), ret_of(r.value()));
1002+
1003+
std::vector<uint8_t> got(len);
1004+
ASSERT_EQ(static_cast<ssize_t>(len), ::pread(buf.get(), got.data(), len, 0));
1005+
for (std::size_t seg = 0; seg < 3; ++seg) {
1006+
for (std::size_t j = 0; j < kAperture; ++j) {
1007+
EXPECT_EQ(window[j], got[seg * kAperture + j])
1008+
<< "seg " << seg << " byte " << j;
1009+
}
1010+
}
1011+
}
1012+
8601013
// ═════════════════════════════════════════════════════════════════════════════
8611014
// FD-index resolution
8621015
// ═════════════════════════════════════════════════════════════════════════════

0 commit comments

Comments
 (0)