ionic: Add Reorder Completion Queue (RCQ) support

kernel-headers/rdma/ionic-abi.h

@@ -46,8 +46,10 @@ struct ionic_ctx_resp {
 	__u8 udma_count;
 	__u8 expdb_mask;
 	__u8 expdb_qtypes;
+	__u8 rcq_sign_bit;
+	__u8 default_qp_transport_mode;
 
-	__u8 rsvd2[3];
+	__u8 rsvd2;
 };
 
 struct ionic_qdesc {
@@ -83,7 +85,10 @@ struct ionic_qp_req {
 	__u8 sq_cmb;
 	__u8 rq_cmb;
 	__u8 udma_mask;
-	__u8 rsvd[3];
+	__u8 num_rcq_paths;
+	__u8 transport_mode;
+	__u8 rsvd;
+	__u32 ionic_flags;
 };
 
 struct ionic_qp_resp {

providers/ionic/CMakeLists.txt

@@ -1,5 +1,5 @@
 rdma_shared_provider(ionic libionic.map
-	1 1.0.${PACKAGE_VERSION}
+	1 1.1.${PACKAGE_VERSION}
 	ionic.c
 	ionic_verbs.c
 	ionic_memory.c

providers/ionic/ionic.c

@@ -77,6 +77,12 @@ static struct verbs_context *ionic_alloc_context(struct ibv_device *ibdev,
 	ctx->sq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_SQ);
 	ctx->rq_expdb = !!(resp.expdb_qtypes & IONIC_EXPDB_RQ);
 
+	ctx->rcq_sign_bit = resp.rcq_sign_bit;
+	ctx->default_qp_transport_mode =
+		(enum ionic_dv_qp_transport_mode)resp.default_qp_transport_mode;
+	if (!ctx->default_qp_transport_mode)
+		ctx->default_qp_transport_mode = IONIC_DV_QPT_TRANSPORT_ROCE_V2;
+
 	mask = (1u << ctx->pg_shift) - 1;
 	ctx->dbpage_page = ionic_map_device(1u << ctx->pg_shift, cmd_fd,
 					    resp.dbell_offset & ~mask);

providers/ionic/ionic.h

@@ -22,6 +22,7 @@
 #include "ionic_memory.h"
 #include "ionic_queue.h"
 #include "ionic_table.h"
+#include "ionic_dv.h"
 
 #include <stdio.h>
 #include <inttypes.h>
@@ -45,6 +46,7 @@
 #define IONIC_PD_TAG_CQ	(IONIC_PD_TAG | 1)
 #define IONIC_PD_TAG_SQ	(IONIC_PD_TAG | 2)
 #define IONIC_PD_TAG_RQ	(IONIC_PD_TAG | 3)
+#define IONIC_PD_TAG_RCQ (IONIC_PD_TAG | 4)
 
 enum {
 	IONIC_CQ_SUPPORTED_WC_FLAGS =
@@ -57,6 +59,19 @@ enum {
 	    IBV_WC_EX_WITH_DLID_PATH_BITS
 };
 
+enum {
+	IONIC_DV_QP_SUPPORTED_COMP_MASK =
+		IONIC_DV_QP_INIT_ATTR_MASK_FLAGS |
+		IONIC_DV_QP_INIT_ATTR_MASK_TRANSPORT_MODE |
+		IONIC_DV_QP_INIT_ATTR_MASK_NUM_RCQ_PATHS,
+
+	IONIC_DV_QP_SUPPORTED_EXT_FLAGS =
+		IONIC_DV_CREATE_QP_TYPE_RCCL		|
+		IONIC_DV_CREATE_QP_RCCL_DATA		|
+		IONIC_DV_CREATE_QP_RCCL_RDFENCE		|
+		IONIC_DV_CREATE_QP_RCCL_RX_OFFLOAD,
+};
+
 struct ionic_ctx {
 	struct verbs_context	vctx;
 
@@ -77,6 +92,9 @@ struct ionic_ctx {
 	bool			sq_expdb;
 	bool			rq_expdb;
 
+	enum ionic_dv_qp_transport_mode	default_qp_transport_mode;
+	uint8_t			rcq_sign_bit;
+
 	void			*dbpage_page;
 	uint64_t		*dbpage;
 
@@ -147,6 +165,16 @@ struct ionic_sq_meta {
 struct ionic_rq_meta {
 	struct ionic_rq_meta	*next;
 	uint64_t		wrid;
+	struct {
+		uint64_t	timestamp;
+		uint32_t	seq;
+		uint32_t	imm_rkey;
+		uint32_t	sts_len;
+		uint8_t		valid:1;
+		uint8_t		ready:1;
+		uint8_t		error:1;
+		uint8_t		op;
+	} rcqe;
 };
 
 struct ionic_rq {
@@ -193,6 +221,8 @@ struct ionic_qp {
 	bool			has_sq;
 	bool			has_rq;
 	bool			lockfree;
+	enum ionic_dv_qp_transport_mode transport_mode;
+	uint8_t			num_rcq_paths;
 
 	struct list_node	cq_poll_sq;
 	struct list_node	cq_poll_rq;
@@ -307,4 +337,8 @@ static inline void ionic_dbg_xdump(struct ionic_ctx *ctx, const char *str,
 /* ionic_verbs.h */
 void ionic_verbs_set_ops(struct ionic_ctx *ctx);
 
+struct ibv_qp *ionic_create_qp_ex_common(struct ibv_context *ibctx,
+					 struct ibv_qp_init_attr_ex *ex,
+					 struct ionic_dv_qp_init_attr_ex *ionic_ex);
+
 #endif /* IONIC_H */

providers/ionic/ionic_dv.c

@@ -105,3 +105,15 @@ int ionic_dv_pd_set_rqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool req
 
 	return 0;
 }
+
+struct ibv_qp *ionic_dv_create_qp_ex(struct ibv_context *ibctx,
+				     struct ibv_qp_init_attr_ex *ex,
+				     struct ionic_dv_qp_init_attr_ex *ionic_ex)
+{
+	if (!is_ionic_ctx(ibctx)) {
+		errno = EINVAL;
+		return NULL;
+	}
+
+	return ionic_create_qp_ex_common(ibctx, ex, ionic_ex);
+}

providers/ionic/ionic_dv.h

@@ -68,6 +68,42 @@ int ionic_dv_pd_set_sqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool req
  */
 int ionic_dv_pd_set_rqcmb(struct ibv_pd *ibpd, bool enable, bool expdb, bool require);
 
+enum ionic_dv_qp_init_attr_mask {
+	IONIC_DV_QP_INIT_ATTR_MASK_FLAGS		= (1 << 0),
+	IONIC_DV_QP_INIT_ATTR_MASK_TRANSPORT_MODE	= (1 << 1),
+	IONIC_DV_QP_INIT_ATTR_MASK_NUM_RCQ_PATHS	= (1 << 2),
+};
+
+enum ionic_dv_qp_init_attr_flags {
+	IONIC_DV_CREATE_QP_TYPE_RCCL		= 1 << 16,
+	IONIC_DV_CREATE_QP_RCCL_DATA		= 1 << 17,
+	IONIC_DV_CREATE_QP_RCCL_RDFENCE		= 1 << 18,
+	IONIC_DV_CREATE_QP_RCCL_RX_OFFLOAD	= 1 << 19,
+};
+
+enum ionic_dv_qp_transport_mode {
+	IONIC_DV_QPT_TRANSPORT_ROCE_V2 = 1 << 0,
+	IONIC_DV_QPT_TRANSPORT_MRC = 1 << 1,
+};
+
+struct ionic_dv_qp_init_attr_ex {
+	uint64_t comp_mask;
+	uint32_t ionic_flags;
+	enum ionic_dv_qp_transport_mode transport_mode;
+	uint8_t num_rcq_paths;
+};
+
+/**
+ * ionic_dv_create_qp_ex - Create a queue pair with ionic-specific attributes.
+ *
+ * @ibctx - Device context.
+ * @ex - Standard QP init attributes.
+ * @ionic_ex - Ionic-specific QP init attributes (transport mode, rcq paths).
+ */
+struct ibv_qp *ionic_dv_create_qp_ex(struct ibv_context *ibctx,
+				     struct ibv_qp_init_attr_ex *ex,
+				     struct ionic_dv_qp_init_attr_ex *ionic_ex);
+
 #ifdef __cplusplus
 }
 #endif

providers/ionic/ionic_fw.h

@@ -217,4 +217,48 @@ static inline int ionic_v1_use_spec_sge(int min_sge, int spec)
 	return spec;
 }
 
+static inline uint32_t ionic_v1_rcqe_seq(uint32_t seq_opf)
+{
+	return seq_opf & IONIC_V1_CQE_RCQE_SEQ_MASK;
+}
+
+static inline uint8_t ionic_v1_rcqe_op(uint32_t seq_opf)
+{
+	return seq_opf >> IONIC_V1_CQE_RCQE_OP_SHIFT;
+}
+
+static inline bool ionic_v1_rcqe_valid(uint32_t seq_opf)
+{
+	return seq_opf & IONIC_V1_CQE_RCQE_FLAG_V;
+}
+
+static inline bool ionic_v1_rcqe_ready(uint32_t seq_opf)
+{
+	return seq_opf & IONIC_V1_CQE_RCQE_FLAG_I;
+}
+
+#define IONIC_RCQ_SIZE 4096
+
+struct ionic_rcq_hdr {
+	__be32 seq;
+	__be32 ack;
+};
+
+struct ionic_rcq {
+	union {
+		uint8_t bytes[IONIC_RCQ_SIZE];
+		struct ionic_rcq_hdr hdr;
+	};
+};
+
+static inline uint32_t ionic_rcq_seq(struct ionic_rcq *rcq)
+{
+	return be32toh(rcq->hdr.seq) & IONIC_V1_CQE_RCQE_SEQ_MASK;
+}
+
+static inline void ionic_rcq_ack(struct ionic_rcq *rcq, uint32_t ack)
+{
+	rcq->hdr.ack = htobe32(ack);
+}
+
 #endif /* IONIC_FW_H */

providers/ionic/ionic_fw_types.h

@@ -70,22 +70,34 @@ union ionic_v1_pld {
 	__u8			data[32];
 };
 
+struct ionic_v1_cqe_send {
+	__u8			rsvd[4];
+	__be32			msg_msn;
+	__u8			rsvd2[8];
+	__le64			npg_wqe_idx;
+};
+
+struct ionic_v1_cqe_recv {
+	__le64			wqe_idx;
+	__be32			src_qpn_op;
+	__u8			src_mac[6];
+	__be16			vlan_tag;
+	__be32			imm_data_rkey;
+};
+
+struct ionic_v1_cqe_rcqe {
+	__be64			wqe_idx;
+	__u8			rsvd[8];
+	__be32			seq_op_flags;
+	__be32			imm_data_rkey;
+};
+
 /* completion queue v1 cqe */
 struct ionic_v1_cqe {
 	union {
-		struct {
-			__le64		wqe_idx;
-			__be32		src_qpn_op;
-			__u8		src_mac[6];
-			__be16		vlan_tag;
-			__be32		imm_data_rkey;
-		} recv;
-		struct {
-			__u8		rsvd[4];
-			__be32		msg_msn;
-			__u8		rsvd2[8];
-			__le64		npg_wqe_idx;
-		} send;
+		struct ionic_v1_cqe_send send;
+		struct ionic_v1_cqe_recv recv;
+		struct ionic_v1_cqe_rcqe rcqe;
 	};
 	__be32				status_length;
 	__be32				qid_type_flags;
@@ -96,6 +108,14 @@ enum ionic_v1_cqe_wqe_idx_bits {
 	IONIC_V1_CQE_WQE_IDX_MASK	= 0xffff,
 };
 
+/* bits for rcqe seq_op_flags */
+enum ionic_v1_cqe_rcqe_op_flag_bits {
+	IONIC_V1_CQE_RCQE_SEQ_MASK	= 0xffffff,
+	IONIC_V1_CQE_RCQE_FLAG_V	= 1u << 24,
+	IONIC_V1_CQE_RCQE_FLAG_I	= 1u << 25,
+	IONIC_V1_CQE_RCQE_OP_SHIFT	= 28,
+};
+
 /* bits for cqe recv */
 enum ionic_v1_cqe_src_qpn_bits {
 	IONIC_V1_CQE_RECV_QPN_MASK	= 0xffffff,
@@ -125,7 +145,7 @@ enum ionic_v1_cqe_qtf_bits {
 	IONIC_V1_CQE_TYPE_RECV		= 1,
 	IONIC_V1_CQE_TYPE_SEND_MSN	= 2,
 	IONIC_V1_CQE_TYPE_SEND_NPG	= 3,
-	IONIC_V1_CQE_TYPE_RECV_INDIR	= 4,
+	IONIC_V1_CQE_TYPE_RECV_RCQE	= 4,
 };
 
 /* v1 base wqe header */

providers/ionic/ionic_queue.c

@@ -9,6 +9,7 @@
 #include "ionic.h"
 #include "ionic_queue.h"
 #include "ionic_memory.h"
+#include "ionic_fw.h"
 
 static void ionic_queue_map(struct ionic_queue *q, struct ionic_pd *pd, uint64_t pd_tag, int stride)
 {
@@ -65,6 +66,9 @@ int ionic_queue_init(struct ionic_queue *q, struct ionic_pd *pd,
 	q->size = BIT_ULL(q->depth_log2 + q->stride_log2);
 	q->mask = BIT(q->depth_log2) - 1;
 
+	if (pd_tag == IONIC_PD_TAG_RCQ)
+		q->size += IONIC_RCQ_SIZE;
+
 	ionic_queue_map(q, pd, pd_tag, stride);
 	if (!q->ptr)
 		return errno;