components/open-fabrics/rds-tools/rds.h
author Boris Chiu <boris.chiu@oracle.com>
Tue, 01 Jul 2014 14:49:32 -0700
branchs11-update
changeset 3195 cf6a5a756b74
permissions -rw-r--r--
15695368 SUNBT7017250 man page from rds-stress has incorect entry for rdma bytes transmi 17489343 Update solaris rds-tools to 2.0.7-1.12 18382333 libibverbs ibv_cmd_get_context() fails when mcxnex driver UAR space is exhausted 18422470 modify librdmacm on solaris to avoid opening device context per hca on startup 18728045 Include local copy of rds.h file to build NRM rds-tools

/*
 * This file contains definitions used in OFED defined user/kernel
 * interfaces. These are imported from the OFED header <linux/rds.h>. Oracle
 * elects to have and use the contents of <linux/rds.h> under and governed
 * by the OpenIB.org BSD license (see below for full license text). However,
 * the following notice accompanied the original version of this file:
 */
/*
 * Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
/*
 * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved.
 */
#ifndef _SYS_RDS_H
#define	_SYS_RDS_H

#ifdef __cplusplus
extern "C" {
#endif

#if !(defined(__SVR4) && defined(__sun))
#include <linux/types.h>
#else
#include <sys/types.h>
#include <sys/socket.h>
#endif

/*
 * These sparse annotated types shouldn't be in any user
 * visible header file. We should clean this up rather
 * than kludging around them.
 */
#if !(defined(__SVR4) && defined(__sun))
#ifndef __KERNEL__
#define	__be16	u_int16_t
#define	__be32	u_int32_t
#define	__be64	u_int64_t
#endif
#else
#define	u_int8_t	uint8_t
#define	u_int16_t	uint16_t
#define	u_int32_t	uint32_t
#define	u_int64_t	uint64_t
#endif

#define	RDS_IB_ABI_VERSION		0x301

#define	AF_RDS	AF_INET_OFFLOAD
#define	PF_RDS	AF_INET_OFFLOAD
#define	SOL_RDS	272

/*
 * setsockopt/getsockopt for SOL_RDS
 */
#define	RDS_CANCEL_SENT_TO	1
#define	RDS_GET_MR			2
#define	RDS_FREE_MR			3
/* deprecated: RDS_BARRIER 4 */
#define	RDS_RECVERR			5
#define	RDS_CONG_MONITOR		6
#define	RDS_GET_MR_FOR_DEST		7
#define	RDS_CONN_RESET			8

/* New ioctls for qos */
#define	SIOCRDSSETTOS	11000
#define	SIOCRDSGETTOS	11001

typedef		u_int8_t	rds_tos_t;

/*
 * Control message types for SOL_RDS.
 *
 * CMSG_RDMA_ARGS (sendmsg)
 *	Request a RDMA transfer to/from the specified
 *	memory ranges.
 *	The cmsg_data is a struct rds_rdma_args.
 * RDS_CMSG_RDMA_DEST (recvmsg, sendmsg)
 *	Kernel informs application about intended
 *	source/destination of a RDMA transfer
 * RDS_CMSG_RDMA_MAP (sendmsg)
 *	Application asks kernel to map the given
 *	memory range into a IB MR, and send the
 *	R_Key along in an RDS extension header.
 *	The cmsg_data is a struct rds_get_mr_args,
 *	the same as for the GET_MR setsockopt.
 * RDS_CMSG_RDMA_STATUS (recvmsg)
 *	Returns the status of a completed RDMA operation.
 */
#define	RDS_CMSG_RDMA_ARGS		1
#define	RDS_CMSG_RDMA_DEST		2
#define	RDS_CMSG_RDMA_MAP		3
#define	RDS_CMSG_RDMA_STATUS		4
#define	RDS_CMSG_CONG_UPDATE		5
#define	RDS_CMSG_ATOMIC_FADD		6
#define	RDS_CMSG_ATOMIC_CSWP		7
#define	RDS_CMSG_MASKED_ATOMIC_FADD	8
#define	RDS_CMSG_MASKED_ATOMIC_CSWP	9
#define	RDS_CMSG_ASYNC_SEND		10


/* private ioctl cmds */
#define	RDS_INFO_FIRST			10000
#define	RDS_INFO_COUNTERS		10000
#define	RDS_INFO_CONNECTIONS		10001
/* 10002 aka RDS_INFO_FLOWS is deprecated */
#define	RDS_INFO_SEND_MESSAGES		10003
#define	RDS_INFO_RETRANS_MESSAGES	10004
#define	RDS_INFO_RECV_MESSAGES		10005
#define	RDS_INFO_SOCKETS		10006
#define	RDS_INFO_TCP_SOCKETS		10007
#define	RDS_INFO_IB_CONNECTIONS		10008
#define	RDS_INFO_CONNECTION_STATS	10009
#define	RDS_INFO_IWARP_CONNECTIONS	10010
#define	RDS_INFO_LAST			10010

#if defined(__SVR4) && defined(__sun)
struct rds_info_arg {
	uint64_t	lenp;
	uint64_t	datap;
};
#endif

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_counter {
	u_int8_t	name[32];
	u_int64_t	value;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_counter {
	u_int8_t	name[32];
	u_int64_t	value;
};
#endif

#define	RDS_INFO_CONNECTION_FLAG_SENDING	0x01
#define	RDS_INFO_CONNECTION_FLAG_CONNECTING	0x02
#define	RDS_INFO_CONNECTION_FLAG_CONNECTED	0x04
#define	RDS_INFO_CONNECTION_FLAG_ERROR		0x08

#define	TRANSNAMSIZ	16

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_connection {
	u_int64_t	next_tx_seq;
	u_int64_t	next_rx_seq;
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int8_t	transport[TRANSNAMSIZ];		/* null term ascii */
	u_int8_t	flags;
	u_int8_t	tos;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_connection {
	u_int64_t	next_tx_seq;
	u_int64_t	next_rx_seq;
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int8_t	transport[TRANSNAMSIZ];		/* null term ascii */
	u_int8_t	flags;
	u_int8_t	tos;
};
#endif

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_flow {
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int32_t	bytes;
	u_int16_t	lport;
	u_int16_t	fport;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_flow {
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int32_t	bytes;
	u_int16_t	lport;
	u_int16_t	fport;
};
#endif

#define	RDS_INFO_MESSAGE_FLAG_ACK		0x01
#define	RDS_INFO_MESSAGE_FLAG_FAST_ACK		0x02

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_message {
	u_int64_t	seq;
	u_int32_t	len;
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int16_t	lport;
	u_int16_t	fport;
	u_int8_t	flags;
	u_int8_t	tos;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_message {
	u_int64_t	seq;
	u_int32_t	len;
	u_int32_t	laddr;
	u_int32_t	faddr;
	u_int16_t	lport;
	u_int16_t	fport;
	u_int8_t	flags;
	u_int8_t	tos;
};
#endif

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_socket {
	u_int32_t	sndbuf;
	u_int32_t	bound_addr;
	u_int32_t	connected_addr;
	u_int16_t	bound_port;
	u_int16_t	connected_port;
	u_int32_t	rcvbuf;
	u_int64_t	inum;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_socket {
	u_int32_t	sndbuf;
	u_int32_t	bound_addr;
	u_int32_t	connected_addr;
	u_int16_t	bound_port;
	u_int16_t	connected_port;
	u_int32_t	rcvbuf;
	u_int64_t	inum;
};
#endif

#ifndef __lock_lint
#pragma pack(1)
struct rds_info_tcp_socket {
	u_int32_t	local_addr;
	u_int16_t	local_port;
	u_int32_t	peer_addr;
	u_int16_t	peer_port;
	u_int64_t	hdr_rem;
	u_int64_t	data_rem;
	u_int32_t	last_sent_nxt;
	u_int32_t	last_expected_una;
	u_int32_t	last_seen_una;
} __attribute__((packed));
#pragma pack()
#else
struct rds_info_tcp_socket {
	u_int32_t	local_addr;
	u_int16_t	local_port;
	u_int32_t	peer_addr;
	u_int16_t	peer_port;
	u_int64_t	hdr_rem;
	u_int64_t	data_rem;
	u_int32_t	last_sent_nxt;
	u_int32_t	last_expected_una;
	u_int32_t	last_seen_una;
} __attribute__((packed));
#endif

#define	RDS_IB_GID_LEN	16
struct rds_info_rdma_connection {
	u_int32_t	src_addr;
	u_int32_t	dst_addr;
	uint8_t		src_gid[RDS_IB_GID_LEN];
	uint8_t		dst_gid[RDS_IB_GID_LEN];

	uint32_t	max_send_wr;
	uint32_t	max_recv_wr;
	uint32_t	max_send_sge;
	uint32_t	rdma_mr_max;
	uint32_t	rdma_mr_size;
	uint8_t		tos;
	uint8_t		sl;
};

/*
 * Congestion monitoring.
 * Congestion control in RDS happens at the host connection
 * level by exchanging a bitmap marking congested ports.
 * By default, a process sleeping in poll() is always woken
 * up when the congestion map is updated.
 * With explicit monitoring, an application can have more
 * fine-grained control.
 * The application installs a 64bit mask value in the socket,
 * where each bit corresponds to a group of ports.
 * When a congestion update arrives, RDS checks the set of
 * ports that are now uncongested against the list bit mask
 * installed in the socket, and if they overlap, we queue a
 * cong_notification on the socket.
 *
 * To install the congestion monitor bitmask, use RDS_CONG_MONITOR
 * with the 64bit mask.
 * Congestion updates are received via RDS_CMSG_CONG_UPDATE
 * control messages.
 *
 * The correspondence between bits and ports is
 *	1 << (portnum % 64)
 */
#define	RDS_CONG_MONITOR_SIZE	64
#define	RDS_CONG_MONITOR_BIT(port)  \
	(((unsigned int) port) % RDS_CONG_MONITOR_SIZE)
#define	RDS_CONG_MONITOR_MASK(port) (1ULL << RDS_CONG_MONITOR_BIT(port))

/*
 * RDMA related types
 */

/*
 * This encapsulates a remote memory location.
 * In the current implementation, it contains the R_Key
 * of the remote memory region, and the offset into it
 * (so that the application does not have to worry about
 * alignment).
 */
typedef u_int64_t	rds_rdma_cookie_t;

struct rds_iovec {
	u_int64_t	addr;
	u_int64_t	bytes;
};

struct rds_get_mr_args {
	struct rds_iovec vec;
	u_int64_t	cookie_addr;
	uint64_t	flags;
};

struct rds_get_mr_for_dest_args {
	struct sockaddr_storage dest_addr;
	struct rds_iovec	vec;
	u_int64_t		cookie_addr;
	uint64_t		flags;
};


struct rds_free_mr_args {
	rds_rdma_cookie_t cookie;
	u_int64_t	flags;
};

struct rds_rdma_args {
	rds_rdma_cookie_t cookie;
	struct rds_iovec remote_vec;
	u_int64_t	local_vec_addr;
	u_int64_t	nr_local;
	u_int64_t	flags;
	u_int64_t	user_token;
};

struct rds_rdma_notify {
	u_int64_t	user_token;
	int32_t		status;
};

struct rds_reset {
	u_int8_t	tos;
	struct in_addr	src;
	struct in_addr	dst;
};

struct rds_asend_args {
	u_int64_t	user_token;
	u_int64_t	flags;
};

struct rds_rdma_send_notify {
	u_int64_t	user_token;
	int32_t		status;
};

#define	RDS_RDMA_SUCCESS		0
#define	RDS_RDMA_REMOTE_ERROR		1
#define	RDS_RDMA_SEND_CANCELED		2
#define	RDS_RDMA_SEND_DROPPED		3
#define	RDS_RDMA_SEND_OTHER_ERROR	4

/* Retain these to build pre-NRM rds-tools */
#define	RDS_RDMA_CANCELED		RDS_RDMA_SEND_CANCELED
#define	RDS_RDMA_DROPPED		RDS_RDMA_SEND_DROPPED
#define	RDS_RDMA_OTHER_ERROR		RDS_RDMA_SEND_OTHER_ERROR

/*
 * Common set of flags for all RDMA related structs
 */
#define	RDS_RDMA_READWRITE	0x0001
#define	RDS_RDMA_FENCE		0x0002	/* use FENCE for immediate send */
#define	RDS_RDMA_INVALIDATE	0x0004	/* invalidate R_Key after freeing MR */
#define	RDS_RDMA_USE_ONCE	0x0008	/* free MR after use */
#define	RDS_RDMA_DONTWAIT	0x0010	/* Don't wait in SET_BARRIER */
#define	RDS_RDMA_NOTIFY_ME	0x0020	/* Notify when operation completes */
#define	RDS_RDMA_SILENT		0x0040	/* Do not interrupt remote */
#define	RDS_RDMA_REMOTE_COMPLETE 0x0080	/* Notify when data is available */
#define	RDS_SEND_NOTIFY_ME	0x0100	/* Notify when operation completes */

#ifdef	__cplusplus
}
#endif

#endif /* _SYS_RDS_H */