Sys/linux: Add rdma.txt

Initial description of the kernel's RDMA subsystem.
This patch covers most of the older write() interface as well as the
some ioctl functions.
Also disable rdma_cm's ib_qp_type flags as it conflicts with rdma's
definition, and rdma builds first.

Signed-off-by: Noa Osherovich <noaos@mellanox.com>

1 files changed, 1468 insertions, 0 deletions

diff --git a/sys/linux/dev_infiniband_rdma.txt b/sys/linux/dev_infiniband_rdma.txt
new file mode 100644
index 000000000..dceb32fdd
--- /dev/null
+++ b/sys/linux/dev_infiniband_rdma.txt
@@ -0,0 +1,1468 @@
+# TODO: This needs some fixups according to PR review:
+# https://github.com/google/syzkaller/pull/1053
+# In the meantime, add it as is.
+
+# Copyright 2015 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found
+# in the LICENSE file.
+
+# Description of syscall arguments. See README.md for details.
+
+include <uapi/linux/fcntl.h>
+include <rdma/ib_verbs.h>
+include <rdma/rdma_user_ioctl.h>
+include <rdma/ib_user_ioctl_verbs.h>
+include <rdma/mlx5-abi.h>
+include <uapi/rdma/ib_user_ioctl_cmds.h>
+include <uapi/rdma/mlx5_user_ioctl_cmds.h>
+
+# resources
+resource fd_rdma[fd]
+resource pd_handle[int32]
+resource ah_handle[int32]
+resource mr_handle[int32]
+resource mr_rkey[int32]
+resource mr_lkey[int32]
+resource cq_handle[int32]
+resource qp_handle[int32]
+resource qp_number[int32]
+resource mw_handle[int32]
+resource srq_handle[int32]
+resource xrcd_handle[int32]
+resource wq_handle[int32]
+resource ind_tbl_handle[int32]
+resource flow_handle[int32]
+
+# defines
+define IB_USER_VERBS_EX_CMD_QUERY_DEVICE	0x80000001
+define IB_USER_VERBS_EX_CMD_CREATE_FLOW	0x80000032
+define IB_USER_VERBS_EX_CMD_DESTROY_FLOW	0x80000033
+define MLX5_CREATE_DCT	0x8
+define MLX5_CREATE_DCI	0x16
+define IB_USER_VERBS_CMD_FLAG_EXTENDED	0x80000000
+define IB_USER_VERBS_CMD_FLAGS_SHIFT	24
+define IB_USER_VERBS_CMD_THRESHOLD	50
+define IB_USER_VERBS_CMD_CREATE_CQ	0x12
+define IB_USER_VERBS_CMD_CREATE_QP	0x18
+define EX_CREATE_CQ_CMD	IB_USER_VERBS_CMD_FLAG_EXTENDED | IB_USER_VERBS_CMD_CREATE_CQ
+define EX_CREATE_QP_CMD	IB_USER_VERBS_CMD_FLAG_EXTENDED | IB_USER_VERBS_CMD_CREATE_QP
+
+# flags
+rdma_dev_open_flags = O_RDWR, O_CLOEXEC
+ib_access_flags = IB_ACCESS_LOCAL_WRITE, IB_ACCESS_REMOTE_WRITE, IB_ACCESS_REMOTE_READ, IB_ACCESS_REMOTE_ATOMIC, IB_ACCESS_MW_BIND, IB_ZERO_BASED, IB_ACCESS_ON_DEMAND
+ib_mr_rereg_flags = IB_MR_REREG_TRANS, IB_MR_REREG_PD, IB_MR_REREG_ACCESS, IB_MR_REREG_SUPPORTED
+ib_qp_type = IB_QPT_SMI, IB_QPT_GSI, IB_QPT_RC, IB_QPT_UC, IB_QPT_UD, IB_QPT_RAW_IPV6, IB_QPT_RAW_ETHERTYPE, IB_QPT_RAW_PACKET, IB_QPT_XRC_INI, IB_QPT_XRC_TGT
+ib_qp_create_flags = IB_QP_CREATE_IPOIB_UD_LSO, IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK, IB_QP_CREATE_CROSS_CHANNEL, IB_QP_CREATE_MANAGED_SEND, IB_QP_CREATE_MANAGED_RECV, IB_QP_CREATE_NETIF_QP, IB_QP_CREATE_SIGNATURE_EN, IB_QP_CREATE_SCATTER_FCS
+ib_send_flags = IB_SEND_FENCE, IB_SEND_SIGNALED, IB_SEND_SOLICITED, IB_SEND_INLINE, IB_SEND_IP_CSUM
+ib_flow_flags = IB_FLOW_ATTR_FLAGS_DONT_TRAP
+ib_ipv4_flags = IB_IPV4_DONT_FRAG, IB_IPV4_MORE_FRAG
+mlx5_create_qp_flags = MLX5_QP_FLAG_SIGNATURE, MLX5_QP_FLAG_SCATTER_CQE, MLX5_QP_FLAG_TUNNEL_OFFLOADS, MLX5_QP_FLAG_BFREG_INDEX, MLX5_CREATE_DCT, MLX5_CREATE_DCI
+mlx5_create_cq_flags = 0
+create_cq_ex_flags = IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION, IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN
+create_cq_ex_mask = 0x0, 0x1
+mlx5_comp_cqe_res_format = MLX5_IB_CQE_RES_FORMAT_HASH, MLX5_IB_CQE_RES_FORMAT_CSUM
+wq_type = IB_WQT_RQ
+create_wq_flags = IB_WQ_FLAGS_CVLAN_STRIPPING, IB_WQ_FLAGS_SCATTER_FCS, IB_WQ_FLAGS_DELAY_DROP, IB_WQ_FLAGS_PCI_WRITE_END_PADDING
+modify_wq_attr_flags = IB_WQ_STATE, IB_WQ_CUR_STATE, IB_WQ_FLAGS
+modify_wq_flags = IB_WQ_FLAGS_CVLAN_STRIPPING, IB_WQ_FLAGS_SCATTER_FCS, IB_WQ_FLAGS_DELAY_DROP, IB_WQ_FLAGS_PCI_WRITE_END_PADDING
+srq_type = IB_SRQT_BASIC, IB_SRQT_XRC, IB_SRQT_TM
+create_srq_ex_flags = MLX5_SRQ_FLAG_SIGNATURE
+
+# structs
+# =======
+
+# context
+# ===========
+
+mlx5_get_context_cmd_resp {
+	async_fd			int32
+	num_comp_vectors		int32
+	qp_tab_size			int32
+	bf_reg_size			int32
+	tot_uuars			int32
+	cache_line_size			int32
+	max_sq_desc_sz			int16
+	max_rq_desc_sz			int16
+	max_send_wqebb			int32
+	max_recv_wr			int32
+	max_srq_recv_wr			int32
+	num_ports			int16
+	reserved1			const[0x0, int16]
+	comp_mask			int32[0:1]
+	response_length			int32
+	cqe_version_cmds_supp_uhw	int16
+	reserved2			const[0x0, int16]
+	hca_core_clock_offset		int64
+	log_uar_size			int32
+	num_uars_per_page		int32
+	num_dyn_bfregs			int32
+	reserved3			const[0x0, int32]
+}
+
+mlx5_get_context_cmd {
+	command			const[0x0, int32]
+	in_words		const[0xc, int16]
+	out_words		const[0x12, int16]
+	response		ptr[out, mlx5_get_context_cmd_resp]
+	total_num_uuars		int32
+	num_low_latency_uuars	int32
+	flags			int32
+	comp_mask		int32
+	cqe_version		int16
+	reserved1		const[0x0, int16]
+	reserved2		const[0x0, int32]
+	lib_caps		int64
+}
+
+# query_device
+# ================
+
+query_device_resp_ex {
+	fw_ver				int64
+	node_guid			int64
+	sys_image_guid			int64
+	max_mr_size			int64
+	page_size_cap			int64
+	vendor_id			int32
+	vendor_part_id			int32
+	hw_ver				int32
+	max_qp				int32
+	max_qp_wr			int32
+	device_cap_flags		int32
+	max_sge				int32
+	max_sge_rd			int32
+	max_cq				int32
+	max_cqe				int32
+	max_mr				int32
+	max_qp_rd_atom			int32
+	max_ee_rd_atom			int32
+	max_res_rd_atom			int32
+	max_qp_init_rd_atom		int32
+	max_ee_init_rd_atom		int32
+	atomic_cap			int32
+	max_ee				int32
+	max_rdd				int32
+	max_mw				int32
+	max_raw_ipv6_qp			int32
+	max_raw_ethy_qp			int32
+	max_mcast_grp			int32
+	max_mcast_qp_attach		int32
+	max_total_mcast_qp_attach	int32
+	max_ah				int32
+	max_fmr				int32
+	max_map_per_fmr			int32
+	max_srq				int32
+	max_srq_wr			int32
+	max_srq_sge			int32
+	max_pkeys			int16
+	local_ca_ack_delay		int8
+	phys_port_cnt			int8
+	reserved			const[0x0, int32]
+	comp_mask			const[0x0, int32]
+	response_length			int32
+	general_caps			int64
+	rc_odp_caps			int32
+	uc_odp_caps			int32
+	ud_odp_caps			int32
+	reserved_odp			const[0x0, int32]
+	timestamp_mask			int64
+	hca_core_clock			int64
+	device_cap_flags_ex		int64
+	supported_qpts			int32
+	max_rwq_indirection_tables	int32
+	max_rwq_indirection_table_size	int32
+	reserved_rss			const[0x0, int32]
+	max_wq_type_rq			int32
+	raw_packet_caps			int32
+	max_rndv_hdr_size		int32
+	max_num_tags			int32
+	flags				int32
+	max_ops				int32
+	max_sge_tm			int32
+	reserved_tm			int32
+	max_cq_moderation_count		int16
+	max_cq_moderation_period	int16
+	reserved_cq_mod			int32
+	max_dm_size			int64
+}
+
+query_device_cmd_ex {
+	command			const[IB_USER_VERBS_EX_CMD_QUERY_DEVICE, int32]
+	in_words		const[0x1, int16]
+	out_words		bytesize4[response, int16]
+	response		ptr[out, query_device_resp_ex]
+	provider_in_words	const[0x0, int16]
+	provider_out_words	const[0x8, int16]
+	reserved		const[0x0, int32]
+	comp_mask		const[0x0, int32]
+	reserved1		const[0x0, int32]
+}
+
+# query_port
+# ==============
+
+query_port_cmd {
+	command		const[0x2, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0xa, int16]
+	response	int64
+	port_num	const[0x1, int8]
+	reserved	array[int8, 7]
+	driver_data	array[int64]
+}
+
+# pd
+# ======
+
+alloc_pd_cmd {
+	command		const[0x3, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	bytesize4[response, int16]
+	response	ptr[out, alloc_pd_cmd_resp]
+	driver_data	array[int64]
+}
+
+mlx5_alloc_pd_cmd {
+	command		const[0x3, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	bytesize4[response, int16]
+	response	ptr[out, mlx5_alloc_pd_cmd_resp]
+	driver_data	array[int64]
+}
+
+alloc_pd_cmd_resp {
+	pd_handle	pd_handle
+}
+
+mlx5_alloc_pd_cmd_resp {
+	pd_handle	pd_handle
+	pdn		int32
+}
+
+dealloc_pd_cmd {
+	command		const[0x4, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0, int16]
+	pd_handle	pd_handle
+}
+
+# ah
+# ===
+
+create_ah_cmd_resp {
+	ah_handle	ah_handle
+}
+
+create_ah_cmd {
+	command		const[0x5, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	bytesize4[response, int16]
+	response	ptr[out, create_ah_cmd_resp]
+	user_handle	int64
+	pd_handle	pd_handle
+	reserved	const[0x0, int32]
+	dgid		array[int8, 16]
+	flow_label	int32
+	sgid_index	const[0x0, int8]
+	hop_limit	int8
+	traffic_class	int8
+	reserved1	int8
+	dlid		int16
+	sl		int8
+	src_path_bits	int8
+	static_rate	int8
+	is_global	int8
+	port_num	const[0x1, int8]
+	reserved2	int8
+}
+
+destroy_ah_cmd {
+	command		const[0x8, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	const[0x0, int16]
+}
+
+# mr
+# ======
+
+reg_mr_resp {
+	mr_handle	mr_handle
+	lkey		mr_lkey
+	rkey		mr_rkey
+}
+
+reg_mr_cmd {
+	command		const[0x9, int32]
+	in_words	const[0xc, int16]
+	out_words	const[0x3, int16]
+	response	ptr[out, reg_mr_resp]
+	start		ptr[in, int64]
+	length		len[start, int64]
+	hca_va		ptr[out, int64]
+	pd_handle	pd_handle
+	access_flags	flags[ib_access_flags, int32]
+	driver_data	array[int64]
+}
+
+rereg_mr_resp {
+	lkey	mr_lkey
+	rkey	mr_rkey
+}
+
+rereg_mr_cmd {
+	command		const[0xb, int32]
+	in_words	const[0xe, int16]
+	out_words	const[0x2, int16]
+	response	ptr[out, rereg_mr_resp]
+	mr_handle	mr_handle
+	flags		flags[ib_mr_rereg_flags, int32]
+	start		ptr[in, int64]
+	length		len[start, int64]
+	hca_va		ptr[out, int64]
+	pd_handle	pd_handle
+	access_flags	flags[ib_access_flags, int32]
+	driver_data	array[int64]
+}
+
+dereg_mr_cmd {
+	command		const[0xd, int32]
+	in_words	const[0x3, int16]
+	out_words	const[0x0, int16]
+	mr_handle	mr_handle
+}
+
+alloc_mw_resp {
+	mw_handle	mw_handle
+	rkey		mr_rkey
+}
+
+alloc_mw_cmd {
+	command		const[0xe, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x2, int16]
+	response	ptr[out, alloc_mw_resp]
+	pd_handle	pd_handle
+	mw_type		int8[1:2]
+	reserved0	const[0x0, int8]
+	reserved1	const[0x0, int16]
+}
+
+dealloc_mw_cmd {
+	command		const[0x10, int32]
+	in_words	const[0x4, int16]
+	out_words	const[0x0, int16]
+	mw_handle	mw_handle
+	reserved	const[0x0, int32]
+}
+
+# completion channel
+# ==================
+
+create_comp_channel_resp {
+	fd	int32
+}
+
+create_comp_channel_cmd {
+	command		const[0x11, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, create_comp_channel_resp]
+}
+
+# cq
+# ======
+
+create_cq_resp {
+	cq_handle	cq_handle
+	cqe		int32
+}
+
+mlx5_create_cq_cmd {
+	command			const[EX_CREATE_CQ_CMD, int32]
+	in_words		bytesize4[parent, int16]
+	out_words		bytesize4[response, int16]
+	response		ptr[out, create_cq_resp]
+	provider_in_words	const[0x5, int16]
+	provider_out_words	const[0x1, int16]
+	reserved0		const[0x0, int32]
+	user_handle		int64
+	cqe			int32
+	comp_vector		int32
+	comp_channel		const[0xffffffff, int32]
+	comp_mask		flags[create_cq_ex_mask, int32]
+	reserved1		const[0x0, int32]
+	buf_addr		ptr[in, array[int8, 4096]]
+	db_addr			ptr[in, array[int8, 4096]]
+	cqe_size		const[0x40, int32]
+	cqe_comp_en		int8[0:1]
+	cqe_comp_res_format	flags[mlx5_comp_cqe_res_format, int8]
+	flags			flags[mlx5_create_cq_flags, int16]
+}
+
+# this fails a lot on Mellanox HW. Leaving for others providers' sake and
+# adding mlx5_create_cq instead, which defines driver data properly.
+create_cq_cmd {
+	command		const[0x12, int32]
+	in_words	const[0x10, int16]
+	out_words	const[0x3, int16]
+	reponse		ptr[out, create_cq_resp]
+	user_handle	int64
+	cqe		int32
+	comp_vector	int32
+	comp_channel	int32
+	reserved	const[0x0, int32]
+	driver_data	array[int64]
+}
+
+create_cq_ex_cmd {
+	command			const[EX_CREATE_CQ_CMD, int32]
+	in_words		const[0x4, int16]
+	out_words		const[0x2, int16]
+	response		ptr[out, create_cq_resp]
+	provider_in_words	const[0x5, int16]
+	provider_out_words	const[0x1, int16]
+	reserved0		const[0x0, int32]
+	user_handle		int64
+	cqe			int32
+	comp_vector		const[0x0, int32]
+	comp_channel		const[0xffffffff, int32]
+	comp_mask		flags[create_cq_ex_mask, int32]
+	flags			flags[create_cq_ex_flags, int32]
+	reserved1		const[0x0, int32]
+}
+
+resize_cq_resp {
+	cqe		int32
+	reserved	const[0x0, int32]
+	driver_data	array[int64]
+}
+
+resize_cq_cmd {
+	command		const[0x13, int32]
+	in_words	const[0xa, int16]
+	out_words	const[0x2, int16]
+	response	ptr[out, resize_cq_resp]
+	cq_handle	cq_handle
+	cqe		int32
+	driver_data	array[int64]
+}
+
+destroy_cq_resp {
+	comp_events_reported	int32
+	async_events_reported	int32
+}
+
+destroy_cq_cmd {
+	command		const[0x14, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x2, int16]
+	response	ptr[out, destroy_cq_resp]
+	cq_handle	cq_handle
+	reserved	const[0x0, int32]
+}
+
+kern_wc {
+	wr_id		int64
+	status		int32
+	opcode		int32
+	vendor_err	int32
+	byte_len	int32
+	imm_data	int32
+	qp_num		int32
+	src_qp		int32
+	wc_flags	int32
+	pkey_index	int16
+	slid		int16
+	sl		int8
+	dlid_path_bits	int8
+	port_num	int8
+	reserved	const[0x0, int8]
+}
+
+poll_cq_resp {
+	count		int32
+	reserved	const[0x0, int32]
+	wc		array[kern_wc]
+}
+
+poll_cq_cmd {
+	command		const[0x15, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x2, int16]
+	response	ptr[out, poll_cq_resp]
+	cq_handle	cq_handle
+	ne		int32
+}
+
+req_notify_cq_cmd {
+	command		const[0x17, int32]
+	in_words	const[0x4, int16]
+	out_words	const[0x0, int16]
+	cq_handle	cq_handle
+	solicited	int32[0x0:0x1]
+}
+
+# qp_ex
+# =========
+
+create_qp_resp {
+	qp_handle	qp_handle
+	qpn		qp_number
+	max_send_wr	int32
+	max_recv_wr	int32
+	max_send_sge	int32
+	max_recv_sge	int32
+	max_inline_data	int32
+	reserved	const[0x0, int32]
+}
+
+create_qp_cmd {
+	command		const[0x18, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	bytesize4[response, int16]
+	response	ptr[out, create_qp_resp]
+	user_handle	int64
+	pd_handle	pd_handle
+	send_cq_handle	cq_handle
+	recv_cq_handle	cq_handle
+	srq_handle	srq_handle
+	max_send_wr	int32
+	max_recv_wr	int32
+	max_send_sge	int32
+	max_recv_sge	int32
+	max_inline_data	int32
+	sq_sig_all	int8
+	qp_type		flags[ib_qp_type, int8]
+	is_srq		int8[0:1]
+	reserved1	const[0x0, int8]
+	driver_data	array[int64]
+}
+
+mlx5_create_qp_cmd {
+	command		const[EX_CREATE_QP_CMD, int32]
+	in_words	bytesize4[parent, int16]
+	out_words	bytesize4[response, int16]
+	response	ptr[out, create_qp_resp]
+	user_handle	int64
+	pd_handle	pd_handle
+	send_cq_handle	cq_handle
+	recv_cq_handle	cq_handle
+	srq_handle	srq_handle
+	max_send_wr	int32
+	max_recv_wr	int32
+	max_send_sge	int32
+	max_recv_sge	int32
+	max_inline_data	int32
+	sq_sig_all	int8
+	qp_type		flags[ib_qp_type, int8]
+	is_srq		int8[0:1]
+	reserved0	const[0x0, int8]
+	buf_addr	ptr[in, array[int8, 4096]]
+	db_addr		ptr[in, array[int8, 4096]]
+	sq_wqe_count	int32
+	rq_wqe_count	int32
+	rq_wqe_shift	int32
+	flags		flags[mlx5_create_qp_flags, int32]
+	uidx		int32
+	bfreg_index	int32
+	sq_buf_addr	ptr[in, array[int8, 4096]]
+} [packed]
+
+mlx5_create_qp_resp {
+	qp_handle	qp_handle
+	qpn		qp_number
+	max_send_wr	int32
+	max_recv_wr	int32
+	max_send_sge	int32
+	max_recv_sge	int32
+	max_inline_data	int32
+	reserved0	const[0x0, int32]
+	uuar_index	int32
+}
+
+mlx5_create_dv_qp_cmd {
+	command		const[0x18, int32]
+	in_words	const[0x1c, int16]
+	out_words	const[0x9, int16]
+	response	ptr[out, mlx5_create_qp_resp]
+	user_handle	int64
+	pd_handle	pd_handle
+	send_cq_handle	cq_handle
+	recv_cq_handle	cq_handle
+	srq_handle	srq_handle
+	max_send_wr	int32
+	max_recv_wr	int32
+	max_send_sge	int32
+	max_recv_sge	int32
+	max_inline_data	int32
+	sq_sig_all	int8
+	qp_type		const[0xff, int8]
+	is_srq		int8
+	reserved0	const[0x0, int8]
+	driver_data	array[int64]
+	buf_addr	ptr[in, array[int8, 4096]]
+	db_addr		ptr[in, array[int8, 4096]]
+	sq_wqe_count	int32
+	rq_wqe_count	int32
+	rq_wqe_shift	int32
+	flags		int32
+	uidx		int32
+	reserved1	const[0x0, int32]
+	access_key	int64
+} [packed]
+
+destroy_qp_resp {
+	events_reported	int32
+}
+
+destroy_qp_cmd {
+	command		const[0x1b, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, destroy_qp_resp]
+	qp_handle	qp_handle
+	reserved	const[0x0, int32]
+}
+
+query_qp_resp {
+	dest_dgid		array[int8, 16]
+	dest_flow_label		int32
+	dest_dlid		int16
+	dest_reserved		const[0x0, int16]
+	dest_sgid_index		int8
+	dest_hop_limit		int8
+	dest_traffic_class	int8
+	dest_sl			int8
+	dest_src_path_bits	int8
+	dest_static_rate	int8
+	dest_is_global		int8
+	dest_port_num		int8
+	alt_dest_dgid		array[int8, 16]
+	alt_dest_flow_label	int32
+	alt_dest_dlid		int16
+	alt_dest_reserved	const[0x0, int16]
+	alt_dest_sgid_index	int8
+	alt_dest_hop_limit	int8
+	alt_dest_traffic_class	int8
+	alt_dest_sl		int8
+	alt_dest_src_path_bits	int8
+	alt_dest_static_rate	int8
+	alt_dest_is_global	int8
+	alt_dest_port_num	int8
+	max_send_wr		int32
+	max_recv_wr		int32
+	max_send_sge		int32
+	max_recv_sge		int32
+	max_inline_data		int32
+	qkey			int32
+	rq_psn			int32
+	sq_psn			int32
+	dest_qp_num		int32
+	qp_access_flags		int32
+	pkey_index		int16
+	alt_pkey_index		int16
+	qp_state		int8
+	cur_qp_state		int8
+	path_mtu		int8
+	path_mig_state		int8
+	sq_draining		int8
+	max_rd_atomic		int8
+	max_dest_rd_atomic	int8
+	min_rnr_timer		int8
+	port_num		int8
+	timeout			int8
+	retry_cnt		int8
+	rnr_retry		int8
+	alt_port_num		int8
+	alt_timeout		int8
+	sq_sig_all		int8
+	reserved0		const[0x0, int8]
+	reserved1		const[0x0, int32]
+	driver_data		array[int64]
+}
+
+query_qp_cmd {
+	command		const[0x19, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x20, int16]
+	response	ptr[out, query_qp_resp]
+	qp_handle	qp_handle
+	attr_mask	flags[ib_qp_create_flags, int32]
+	driver_data	array[int64]
+}
+
+modify_qp_cmd {
+	command			const[0x1a, int32]
+	in_words		const[0x1e, int16]
+	out_words		const[0x0, int16]
+	dest_dgid		array[int8, 16]
+	dest_flow_label		int32
+	dest_dlid		int16
+	dest_reserved		int16
+	dest_sgid_index		int8
+	dest_hop_limit		int8
+	dest_traffic_class	int8
+	dest_sl			int8
+	dest_src_path_bits	int8
+	dest_static_rate	int8
+	dest_is_global		int8
+	dest_port_num		int8
+	alt_dest_dgid		array[int8, 16]
+	alt_dest_flow_label	int32
+	alt_dest_dlid		int16
+	alt_dest_reserved	int16
+	alt_dest_sgid_index	int8
+	alt_dest_hop_limit	int8
+	alt_dest_traffic_class	int8
+	alt_dest_sl		int8
+	alt_dest_src_path_bits	int8
+	alt_dest_static_rate	int8
+	alt_dest_is_global	int8
+	alt_dest_port_num	int8
+	qp_handle		int32
+	attr_mask		int32
+	qkey			int32
+	rq_psn			int32
+	sq_psn			int32
+	dest_qp_num		int32
+	qp_access_flags		int32
+	pkey_index		int16
+	alt_pkey_index		int16
+	qp_state		int8
+	cur_qp_state		int8
+	path_mtu		int8
+	path_mig_state		int8
+	en_sqd_async_notify	int8
+	max_rd_atomic		int8
+	max_dest_rd_atomic	int8
+	min_rnr_timer		int8
+	port_num		int8
+	timeout			int8
+	retry_cnt		int8
+	rnr_retry		int8
+	alt_port_num		int8
+	alt_timeout		int8
+	reserved		const[0x0, int16]
+}
+
+# wq
+#====
+
+create_wq_resp {
+	comp_mask	int32
+	response_length	int32
+	wq_handle	wq_handle
+	max_wr		int32
+	max_sge		int32
+	wqn		int32
+}
+
+create_wq_cmd {
+	command			const[0x80000034, int32]
+	in_words		const[0x5, int16]
+	out_words		bytesize4[response, int16]
+	response		ptr[out, create_wq_resp]
+	provider_in_words	const[0x6, int16]
+	provider_out_words	const[0x1, int16]
+	cmd_hdr_reserved	const[0x0, int32]
+	comp_mask		const[0x0, int32]
+	wq_type			flags[wq_type, int32]
+	user_handle		int64
+	pd_handle		pd_handle
+	cq_handle		cq_handle
+	max_wr			int32
+	max_sge			int32
+	create_flags		flags[create_wq_flags, int32]
+	reserved		const[0x0, int32]
+}
+
+mlx5_create_wq_resp {
+	comp_mask		int32
+	response_length		const[0x6, int32]
+	wq_handle		wq_handle
+	max_wr			int32
+	max_sge			int32
+	wqn			int32
+	mlx5_response_length	bytesize4[parent, int32]
+	reserver		const[0x0, int32]
+}
+
+mlx5_create_wq_cmd {
+	command				const[0x80000034, int32]
+	in_words			const[0x5, int16]
+	out_words			bytesize4[response, int16]
+	response			ptr[out, mlx5_create_wq_resp]
+	provider_in_words		const[0x6, int16]
+	provider_out_words		const[0x1, int16]
+	cmd_hdr_reserved		const[0x0, int32]
+	comp_mask			const[0x0, int32]
+	wq_type				flags[wq_type, int32]
+	user_handle			int64
+	pd_handle			pd_handle
+	cq_handle			cq_handle
+	max_wr				int32
+	max_sge				int32
+	create_flags			flags[create_wq_flags, int32]
+	reserved			const[0x0, int32]
+	buf_addr			ptr[in, array[int8, 4096]]
+	db_addr				ptr[in, array[int8, 4096]]
+	rq_wqe_count			int32
+	rq_wqe_shift			int32
+	user_index			int32
+	flags				const[0x0, int32]
+	provider_comp_mask		const[0x0, int32]
+	single_stride_log_num_of_bytes	const[0x0, int32]
+	single_wqe_log_num_of_strides	const[0x0, int32]
+	two_byte_shift_en		int32[0:1]
+}
+
+destroy_wq_resp {
+	comp_mask	int32
+	response_length	int32
+	events_reported	int32
+	reserved	const[0x0, int32]
+}
+
+destroy_wq_cmd {
+	command			const[0x80000036, int32]
+	in_words		const[0x1, int16]
+	out_words		const[0x2, int16]
+	response		ptr[out, destroy_wq_resp]
+	provider_in_words	const[0x0, int16]
+	provider_out_words	const[0x0, int16]
+	cmd_hdr_reserved	const[0x0, int32]
+	comp_mask		int32[0x0:0xf]
+	wq_handle		wq_handle
+}
+
+mlx5_modify_wq_cmd {
+	command			const[0x80000035, int32]
+	in_words		const[0x3, int16]
+	out_words		const[0x0, int16]
+	response		const[0x0, int64]
+	provider_in_words	const[0x1, int16]
+	provider_out_words	const[0x0, int16]
+	cmd_hdr_reserved	const[0x0, int32]
+	attr_mask		flags[modify_wq_attr_flags, int32]
+	wq_handle		wq_handle
+	wq_state		int32[0:3]
+	current_wq_state	int32[0:3]
+	flags			flags[modify_wq_flags, int32]
+	flags_mask		flags[modify_wq_flags, int32]
+	comp_mask		const[0x0, int32]
+	reserved		const[0x0, int32]
+}
+
+# RSS (indirection table)
+#========================
+
+create_rwq_ind_table_resp {
+	comp_mask	int32
+	response_length	bytesize4[parent, int32]
+	ind_tbl_handle	ind_tbl_handle
+	ind_tbl_num	int32
+}
+
+# currently hard coded to use 2 WQs. Need to figure out how to
+# make it a variable.
+create_rwq_ind_table_cmd {
+	command			const[0x80000037, int32]
+	in_words		const[0x2, int16]
+	out_words		const[0x2, int16]
+	response		ptr[out, create_rwq_ind_table_resp]
+	provider_in_words	const[0x0, int16]
+	provider_out_words	const[0x0, int16]
+	cmd_hdr_reserved	const[0x0, int32]
+	comp_mask		const[0x0, int32]
+	log_ind_tbl_size	int32[1]
+	wq_handles		array[wq_handle, 2]
+}
+
+destroy_rwq_ind_table_cmd {
+	comp_mask	int32
+	ind_tbl_handle	ind_tbl_handle
+}
+
+# Send WR
+#========
+
+rdma {
+	remote_addr	int64
+	rkey		mr_rkey
+	reserved	const[0x0, int32]
+}
+
+atomic {
+	remote_addr	int64
+	compare_add	int64
+	swap		int64
+	rkey		mr_rkey
+	reserved	const[0x0, int32]
+}
+
+ud {
+	ah		int32
+	remote_qpn	int32
+	remote_qkey	int32
+	reserved	const[0x0, int32]
+}
+
+xrc {
+	remote_srqn	int32
+}
+
+wr [
+	rdma	rdma
+	atomic	atomic
+	ud	ud
+]
+
+kern_send_wr {
+	wr_id		int64
+	num_sge		int32
+	opcode		int32[0:20]
+	send_flags	flags[ib_send_flags, int32]
+	imm_data	int32
+	wr		wr
+	qp_type		xrc
+}
+
+post_send_cmd {
+	command		const[0x1c, int32]
+	in_words	const[0x8, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, post_send_resp]
+	qp_handle	qp_handle
+	wr_count	int32
+	sge_count	int32
+	wqe_size	int32
+	send_wr		array[kern_send_wr]
+}
+
+post_send_resp {
+	bad_wr	int32
+}
+
+post_recv_cmd {
+	command		const[0x1d, int32]
+	in_words	const[0x8, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, post_recv_resp]
+	qp_handle	qp_handle
+	wr_count	int32
+	sge_count	int32
+	wqe_size	int32
+	recv_wr		array[kern_recv_wr]
+}
+
+post_recv_resp {
+	bad_wr	int32
+}
+
+attach_mcast_cmd {
+	command		const[0x1e, int32]
+	in_words	const[0x8, int16]
+	out_words	const[0x0, int16]
+	gid		array[int8, 0x10]
+	qp_handle	qp_handle
+	mlid		int16
+	reserved	const[0x0, int16]
+	driver_data	array[int64]
+}
+
+detach_mcast_cmd {
+	command		const[0x1f, int32]
+	in_words	const[0x8, int16]
+	out_words	const[0x0, int16]
+	gid		array[int8, 0x10]
+	qp_handle	qp_handle
+	mlid		int16
+	reserved	const[0x0, int16]
+	driver_data	array[int64]
+}
+
+create_srq_resp {
+	srq_handle	srq_handle
+	max_wr		int32
+	max_sge		int32
+	srqn		int32
+}
+
+create_srq_cmd {
+	command		const[0x20, int32]
+	in_words	const[0xa, int16]
+	out_words	const[0x4, int16]
+	response	ptr[out, create_srq_resp]
+	user_handle	int64
+	pd_handle	pd_handle
+	max_wr		int32
+	max_sge		int32
+	srq_limit	int32
+	driver_data	array[int64]
+}
+
+mlx5_ib_create_srq_resp {
+	srqn		int32
+	reserved	const[0x0, int32]
+}
+
+mlx5_create_srq_cmd {
+	command		const[0x27, int32]
+	in_words	const[0x16, int16]
+	out_words	const[0x6, int16]
+	reserved0	const[0x0, int32]
+	response	ptr[out, mlx5_ib_create_srq_resp]
+	user_handle	int64
+	srq_type	flags[srq_type, int32]
+	pd_handle	pd_handle
+	max_wr		int32
+	max_sge		int32
+	srq_limit	int32
+	max_num_tags	int32
+	xrcd_handle	xrcd_handle
+	cq_handle	cq_handle
+	buf_addr	ptr[in, int64]
+	db_addr		ptr[in, int64]
+	flags		flags[create_srq_ex_flags, int32]
+	reserved1	const[0x0, int32]
+	uidx		int32
+	reserved2	const[0x0, int32]
+}
+
+modify_srq_cmd {
+	command		const[0x21, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x0, int16]
+	srq_handle	srq_handle
+	attr_mask	int32[0x0:0x10]
+	max_wr		int32
+	srq_limit	int32
+	driver_data	array[int64]
+}
+
+query_srq_resp {
+	max_wr		int32
+	max_sge		int32
+	srq_limit	int32
+	reserved	const[0x0, int32]
+}
+
+query_srq_cmd {
+	command		const[0x22, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x4, int16]
+	response	ptr[out, query_srq_resp]
+	srq_handle	srq_handle
+	reserved	const[0x0, int32]
+	driver_data	array[int64]
+}
+
+destroy_srq_resp {
+	events_reported	int32
+}
+
+destroy_srq_cmd {
+	command		const[0x23, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, destroy_srq_resp]
+	srq_handle	srq_handle
+	reserved	const[0x0, int32]
+}
+
+kern_recv_wr {
+	wr_id		int64
+	num_sge		int32
+	reserved	const[0x0, int32]
+}
+
+post_srq_recv_resp {
+	bad_wr	int32
+}
+
+post_srq_recv_cmd {
+	command		const[0x24, int32]
+	in_words	const[0x7, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, post_srq_recv_resp]
+	wr_count	int32
+	sge_count	int32
+	wqe_size	int32
+	recv_wr		array[kern_recv_wr]
+}
+
+open_xrcd_resp {
+	xrcd_handle	xrcd_handle
+}
+
+open_xrcd_cmd {
+	command		const[0x25, int32]
+	in_words	const[0x6, int16]
+	out_words	const[0x1, int16]
+	response	ptr[out, open_xrcd_resp]
+	fd		int32
+	oflags		int32
+	driver_data	array[int64]
+}
+
+close_xrcd_cmd {
+	command		const[0x26, int32]
+	in_words	const[0x3, int16]
+	out_words	const[0x0, int16]
+	xrcd_handle	xrcd_handle
+}
+
+# create_flow
+# ===============
+
+ib_flow_eth_filter {
+	dst_mac		array[int8, 0x6]
+	src_mac		array[int8, 0x6]
+	ether_type	int16
+	vlan_tag	int16
+}
+
+ib_flow_spec_eth {
+	type		const[IB_FLOW_SPEC_ETH, int32]
+	size		const[0x10, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_eth_filter
+	mask		ib_flow_eth_filter
+}
+
+ib_flow_ipv4_filter {
+	src_ip	int32
+	dst_ip	int32
+	proto	int8
+	tos	int8
+	ttl	int8
+	flags	flags[ib_ipv4_flags, int8]
+}
+
+ib_flow_spec_ipv4 {
+	type		const[IB_FLOW_SPEC_IPV4, int32]
+	size		const[0x20, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_ipv4_filter
+	mask		ib_flow_ipv4_filter
+}
+
+ib_flow_ipv6_filter {
+	src_ip		array[int8, 0x10]
+	dst_ip		array[int8, 0x10]
+	flow_label	int32
+	next_hdr	int8
+	traffic_class	int8
+	hop_limit	int8
+	reserved	const[0x0, int8]
+}
+
+ib_flow_spec_ipv6 {
+	type		const[IB_FLOW_SPEC_IPV6, int32]
+	size		const[0x58, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_ipv6_filter
+	mask		ib_flow_ipv6_filter
+}
+
+ib_flow_tcp_udp_filter {
+	dst_port	int16
+	src_port	int16
+}
+
+ib_flow_spec_tcp_udp {
+	type		int32[0x40:0x41]
+	size		const[0x10, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_tcp_udp_filter
+	mask		ib_flow_tcp_udp_filter
+}
+
+ib_flow_esp_filter {
+	spi	int32
+	seq	int32
+}
+
+ib_flow_spec_esp {
+	type		const[IB_FLOW_SPEC_ESP, int32]
+	size		const[0x18, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_esp_filter
+	mask		ib_flow_esp_filter
+}
+
+ib_flow_tunnel_filter {
+	tunnel_id	int32
+}
+
+ib_flow_spec_tunnel {
+	type		const[IB_FLOW_SPEC_VXLAN_TUNNEL, int32]
+	size		const[0x10, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_tunnel_filter
+	mask		ib_flow_tunnel_filter
+}
+
+ib_flow_gre_filter {
+	c_ks_res0_ver	int16
+	protocol	int16
+	key		int32
+}
+
+ib_flow_spec_gre {
+	type		const[IB_FLOW_SPEC_GRE, int32]
+	size		const[0x18, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_gre_filter
+	mask		ib_flow_gre_filter
+}
+
+ib_flow_mpls_filter {
+	tag	int32
+}
+
+ib_flow_spec_mpls {
+	type		const[IB_FLOW_SPEC_MPLS, int32]
+	size		const[0x10, int16]
+	reserved	const[0x0, int16]
+	val		ib_flow_mpls_filter
+	mask		ib_flow_mpls_filter
+}
+
+ib_flow_spec_action_tag {
+	type		const[IB_FLOW_SPEC_ACTION_TAG, int32]
+	size		const[0x10, int16]
+	reserved	const[0x0, int16]
+	tag_id		int32
+	reserved2	const[0x0, int32]
+}
+
+ib_flow_spec_action_drop {
+	type		const[IB_FLOW_SPEC_ACTION_DROP, int32]
+	size		const[0x8, int16]
+	reserved	const[0x0, int16]
+}
+
+ib_uverbs_flow_spec_action_count {
+	type		const[IB_FLOW_SPEC_ACTION_COUNT, int32]
+	size		const[0x10, int16]
+	reserved0	const[0x0, int16]
+	handle		vcontext_handle
+	reserved1	const[0x0, int32]
+}
+
+union_ib_flow_spec [
+	eth	ib_flow_spec_eth
+	ipv6	ib_flow_spec_ipv6
+	ipv4	ib_flow_spec_ipv4
+	tcp_udp	ib_flow_spec_tcp_udp
+	tunnel	ib_flow_spec_tunnel
+	gre	ib_flow_spec_gre
+	esp	ib_flow_spec_esp
+	mpls	ib_flow_spec_mpls
+	tag	ib_flow_spec_action_tag
+	drop	ib_flow_spec_action_drop
+	count	ib_uverbs_flow_spec_action_count
+]
+
+ib_uverbs_flow_attr {
+	type		int32[0:3]
+	size		bytesize[flow_specs, int16]
+	priority	int16[0:7]
+	num_of_specs	len[flow_specs, int8]
+	reserved1	const[0x0, int8]
+	reserved2	const[0x0, int8]
+	port		int8
+	flags		flags[ib_flow_flags, int32]
+	flow_specs	array[union_ib_flow_spec]
+}
+
+create_flow_cmd_ex {
+	comp_mask	const[0x0, int32]
+	qp_handle	qp_handle
+	flow_attr	ib_uverbs_flow_attr
+}
+
+create_flow_resp {
+	comp_mask	int32
+	flow_handle	flow_handle
+}
+
+create_flow_cmd {
+	command			const[IB_USER_VERBS_EX_CMD_CREATE_FLOW, int32]
+	in_words		bytesize8[flow_ex, int16]
+	out_words		const[0x1, int16]
+	response		ptr[out, create_flow_resp]
+	provider_in_words	const[0x0, int16]
+	provider_out_words	const[0x0, int16]
+	reserved		const[0x0, int32]
+	flow_ex			create_flow_cmd_ex
+}
+
+destroy_flow_cmd {
+	command			const[IB_USER_VERBS_EX_CMD_DESTROY_FLOW, int32]
+	in_words		const[0x6, int16]
+	out_words		const[0x0, int16]
+	response		const[0x0, int64]
+	provider_in_words	const[0x0, int16]
+	provider_out_words	const[0x0, int16]
+	reserved		const[0x0, int32]
+	comp_mask		const[0x0, int32]
+	flow_handle		flow_handle
+}
+
+# ioctl interface
+
+ib_uverbs_attr_flags = UVERBS_ATTR_F_MANDATORY
+ib_read_counters_flags = IB_UVERBS_READ_COUNTERS_PREFER_CACHED
+resource vcontext_handle[int32]
+
+ib_uverbs_create_counters_cmd {
+	length		bytesize8[ib_uverbs_create_counters_cmd, int16]
+	object_id	const[UVERBS_OBJECT_COUNTERS, int16]
+	method_id	const[UVERBS_METHOD_COUNTERS_CREATE, int16]
+	num_attrs	const[1, int16]
+	reserved0	const[0x0, int64]
+	driver_id	const[RDMA_DRIVER_MLX5, int32]
+	reserved1	const[0x0, int32]
+
+	attr_id		const[UVERBS_ATTR_CREATE_COUNTERS_HANDLE, int16]
+	len		int16
+	flags		flags[ib_uverbs_attr_flags, int16]
+	elem_id		int8
+	reserved2	const[0x0, int8]
+	reserved3	const[0x0, int16]
+	vcontext_handle	vcontext_handle[opt]
+	more_data	int32
+}
+
+ib_uverbs_destroy_counters_cmd {
+	length		bytesize8[ib_uverbs_destroy_counters_cmd, int16]
+	object_id	const[UVERBS_OBJECT_COUNTERS, int16]
+	method_id	const[UVERBS_METHOD_COUNTERS_DESTROY, int16]
+	num_attrs	const[1, int16]
+	reserved0	const[0x0, int64]
+	driver_id	const[RDMA_DRIVER_MLX5, int32]
+	reserved1	const[0x0, int32]
+
+	attr_id		const[UVERBS_ATTR_DESTROY_COUNTERS_HANDLE, int16]
+	len		int16
+	flags		flags[ib_uverbs_attr_flags, int16]
+	elem_id		int8
+	reserved2	const[0x0, int8]
+	reserved3	const[0x0, int16]
+	vcontext_handle	vcontext_handle
+	more_data	int32
+}
+
+# in UVERBS_ATTR_READ_COUNTERS_FLAGS, assuming that sizeof(uint32_t is 4)
+ib_uverbs_read_counters_cmd {
+	length		bytesize8[ib_uverbs_read_counters_cmd, int16]
+	object_id	const[UVERBS_OBJECT_COUNTERS, int16]
+	method_id	const[UVERBS_METHOD_COUNTERS_READ, int16]
+	num_attrs	const[0x3, int16]
+	reserved0	const[0x0, int64]
+	driver_id	const[RDMA_DRIVER_MLX5, int32]
+	reserved1	const[0x0, int32]
+
+	attr_id0	const[UVERBS_ATTR_READ_COUNTERS_HANDLE, int16]
+	len0		int16
+	flags0		flags[ib_uverbs_attr_flags, int16]
+	reserved2	const[0x0, int16]
+	vcontext_handle	vcontext_handle
+	more_data0	int32
+
+	attr_id1	const[UVERBS_ATTR_READ_COUNTERS_BUFF, int16]
+	len1		bytesize8[data, int16]
+	flags1		flags[ib_uverbs_attr_flags, int16]
+	reserved3	const[0x0, int16]
+	data		buffer[out]
+
+	attr_id2	const[UVERBS_ATTR_READ_COUNTERS_FLAGS, int16]
+	len2		const[0x4, int16]
+	flags2		flags[ib_uverbs_attr_flags, int16]
+	reserved4	const[0x0, int16]
+	flags		flags[ib_read_counters_flags, int32]
+	more_data1	int32
+}
+
+# commands
+ioctl$CREATE_COUNTERS(fd fd_rdma, cmd const[RDMA_VERBS_IOCTL], arg ptr[inout, ib_uverbs_create_counters_cmd])
+ioctl$DESTROY_COUNTERS(fd fd_rdma, cmd const[RDMA_VERBS_IOCTL], arg ptr[in, ib_uverbs_destroy_counters_cmd])
+ioctl$READ_COUNTERS(fd fd_rdma, cmd const[RDMA_VERBS_IOCTL], arg ptr[inout, ib_uverbs_read_counters_cmd])
+
+# device
+syz_open_dev$ibv_device(dev ptr[in, string["/dev/infiniband/uverbs0"]], id intptr, flags flags[rdma_dev_open_flags]) fd_rdma
+write$MLX5_GET_CONTEXT(fd fd_rdma, buf ptr[inout, mlx5_get_context_cmd], len len[buf])
+close$ibv_device(fd fd_rdma)
+write$QUERY_DEVICE_EX(fd fd_rdma, buf ptr[inout, query_device_cmd_ex], len len[buf])
+
+# query_port
+write$QUERY_PORT(fd fd_rdma, buf ptr[inout, query_port_cmd], len len[buf])
+
+# pd
+write$ALLOC_PD(fd fd_rdma, buf ptr[inout, alloc_pd_cmd], len len[buf])
+write$MLX5_ALLOC_PD(fd fd_rdma, buf ptr[inout, mlx5_alloc_pd_cmd], len len[buf])
+#dealloc_pd
+write$DEALLOC_PD(fd fd_rdma, buf ptr[inout, dealloc_pd_cmd], len len[buf])
+
+#ah
+write$CREATE_AH(fd fd_rdma, buf ptr[inout, create_ah_cmd], len len[buf])
+write$DESTROY_AH(fd fd_rdma, buf ptr[inout, destroy_ah_cmd], len len[buf])
+
+# mr
+write$REG_MR(fd fd_rdma, buf ptr[inout, reg_mr_cmd], len len[buf])
+write$DEREG_MR(fd fd_rdma, buf ptr[inout, rereg_mr_cmd], len len[buf])
+write$REREG_MR(fd fd_rdma, buf ptr[inout, dereg_mr_cmd], len len[buf])
+
+# mw
+write$ALLOC_MW(fd fd_rdma, buf ptr[inout, alloc_mw_cmd], len len[buf])
+write$DEALLOC_MW(fd fd_rdma, buf ptr[inout, dealloc_mw_cmd], len len[buf])
+
+#comp_channel
+write$CREATE_COMP_CHANNEL(fd fd_rdma, buf ptr[inout, create_comp_channel_cmd], len len[buf])
+
+# cq
+write$CREATE_CQ(fd fd_rdma, buf ptr[inout, create_cq_cmd], len len[buf])
+write$MLX5_CREATE_CQ(fd fd_rdma, buf ptr[inout, mlx5_create_cq_cmd], len len[buf])
+write$CREATE_CQ_EX(fd fd_rdma, buf ptr[inout, create_cq_ex_cmd], len len[buf])
+write$RESIZE_CQ(fd fd_rdma, buf ptr[inout, resize_cq_cmd], len len[buf])
+write$DESTROY_CQ(fd fd_rdma, buf ptr[inout, destroy_cq_cmd], len len[buf])
+write$POLL_CQ(fd fd_rdma, buf ptr[inout, poll_cq_cmd], len len[buf])
+write$REQ_NOTIFY_CQ(fd fd_rdma, buf ptr[inout, req_notify_cq_cmd], len len[buf])
+
+# qp
+write$CREATE_QP(fd fd_rdma, buf ptr[inout, create_qp_cmd], len len[buf])
+
+# qp_ex
+write$MLX5_CREATE_QP(fd fd_rdma, buf ptr[inout, mlx5_create_qp_cmd], len len[buf])
+write$MLX5_CREATE_DV_QP(fd fd_rdma, buf ptr[inout, mlx5_create_dv_qp_cmd], len len[buf])
+write$DESTROY_QP(fd fd_rdma, buf ptr[inout, destroy_qp_cmd], len len[buf])
+write$QUERY_QP(fd fd_rdma, buf ptr[inout, query_qp_cmd], len len[buf])
+write$MODIFY_QP(fd fd_rdma, buf ptr[inout, modify_qp_cmd], len len[buf])
+write$POST_SEND(fd fd_rdma, buf ptr[inout, post_send_cmd], len len[buf])
+write$POST_RECV(fd fd_rdma, buf ptr[inout, post_recv_cmd], len len[buf])
+write$ATTACH_MCAST(fd fd_rdma, buf ptr[in, attach_mcast_cmd], len len[buf])
+write$DETACH_MCAST(fd fd_rdma, buf ptr[in, detach_mcast_cmd], len len[buf])
+write$CREATE_SRQ(fd fd_rdma, buf ptr[inout, create_srq_cmd], len len[buf])
+write$MLX5_CREATE_SRQ(fd fd_rdma, buf ptr[inout, mlx5_create_srq_cmd], len len[buf])
+write$MODIFY_SRQ(fd fd_rdma, buf ptr[in, modify_srq_cmd], len len[buf])
+write$QUERY_SRQ(fd fd_rdma, buf ptr[inout, query_srq_cmd], len len[buf])
+write$DESTROY_SRQ(fd fd_rdma, buf ptr[inout, destroy_srq_cmd], len len[buf])
+write$POST_SRQ_RECV(fd fd_rdma, buf ptr[inout, post_srq_recv_cmd], len len[buf])
+write$OPEN_XRCD(fd fd_rdma, buf ptr[inout, open_xrcd_cmd], len len[buf])
+write$CLOSE_XRCD(fd fd_rdma, buf ptr[inout, close_xrcd_cmd], len len[buf])
+
+# wq
+write$CREATE_WQ(fd fd_rdma, buf ptr[inout, create_wq_cmd], len len[buf])
+write$MLX5_CREATE_WQ(fd fd_rdma, buf ptr[inout, mlx5_create_wq_cmd], len len[buf])
+write$MLX5_MODIFY_WQ(fd fd_rdma, buf ptr[inout, mlx5_modify_wq_cmd], len len[buf])
+write$DESTROY_WQ(fd fd_rdma, buf ptr[inout, destroy_wq_cmd], len len[buf])
+
+# RSS
+write$CREATE_RWQ_IND_TBL(fd fd_rdma, buf ptr[inout, create_rwq_ind_table_cmd], len len[buf])
+write$DESTROY_RWQ_IND_TBL(fd fd_rdma, buf ptr[inout, destroy_rwq_ind_table_cmd], len len[buf])
+
+# create_flow
+write$CREATE_FLOW(fd fd_rdma, buf ptr[inout, create_flow_cmd], len len[buf])
+write$DESTROY_FLOW(fd fd_rdma, buf ptr[inout, destroy_flow_cmd], len len[buf])