extend support for CUDA features

Makefile.am

@@ -19,12 +19,12 @@ libperftest_a_SOURCES = src/get_clock.c src/perftest_communication.c src/perftes
 noinst_HEADERS = src/get_clock.h src/perftest_communication.h src/perftest_parameters.h src/perftest_resources.h
 
 bin_PROGRAMS = ib_send_bw ib_send_lat ib_write_lat ib_write_bw ib_read_lat ib_read_bw ib_atomic_lat ib_atomic_bw
-bin_SCRIPTS = run_perftest_loopback
+bin_SCRIPTS = run_perftest_loopback run_perftest_multi_devices
 
 if HAVE_RAW_ETH
 libperftest_a_SOURCES += src/raw_ethernet_resources.c
 noinst_HEADERS += src/raw_ethernet_resources.h
-bin_PROGRAMS += raw_ethernet_bw raw_ethernet_lat
+bin_PROGRAMS += raw_ethernet_bw raw_ethernet_lat raw_ethernet_burst_lat raw_ethernet_fs_rate
 else
 libperftest_a_SOURCES +=
 noinst_HEADERS +=
@@ -70,6 +70,12 @@ raw_ethernet_bw_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5)
 raw_ethernet_lat_SOURCES = src/raw_ethernet_send_lat.c
 raw_ethernet_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5)
 
+raw_ethernet_burst_lat_SOURCES = src/raw_ethernet_send_burst_lat.c
+raw_ethernet_burst_lat_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5)
+
+raw_ethernet_fs_rate_SOURCES = src/raw_ethernet_fs_rate.c
+raw_ethernet_fs_rate_LDADD = libperftest.a $(LIBMATH) $(LIBMLX4) $(LIBMLX5)
+
 else
 raw_ethernet_bw_SOURCES =
 raw_ethernet_bw_LDADD =

configure.ac

@@ -11,7 +11,7 @@
 # $HEADER$
 dnl Process this file with autoconf to produce a configure script.
 
-AC_INIT([perftest],[5.6],[[email protected]])
+AC_INIT([perftest],[5.60],[[email protected]])
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_MACRO_DIR([m4])
@@ -21,7 +21,7 @@ m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
 AC_SUBST(MAJOR_VERSION)
 AC_SUBST(MINOR_VERSION)
 
-AC_ARG_VAR(CUDA_H_PATH, help-string)
+AC_ARG_VAR(CUDA_H_PATH, path to CUDA Toolkit include directory (enables CUDA support))
 AC_ARG_VAR(RANLIB, ranlib tool)
 
 AC_ARG_ENABLE([verbs_exp],
@@ -193,15 +193,49 @@ fi
 AC_TRY_LINK([
 #include <infiniband/verbs.h>
 #include <infiniband/verbs_exp.h>],
-        [int x = IBV_EXP_ACCESS_ON_DEMAND;],[HAVE_ODP=yes], [HAVE_ODP=no])
-AM_CONDITIONAL([HAVE_ODP],[test "x$HAVE_ODP" = "xyes"])
-if [test $HAVE_ODP = yes] && [test $HAVE_VERBS_EXP = yes]; then
-        AC_DEFINE([HAVE_ODP], [1], [Have ODP support])
+        [int x = IBV_EXP_FLOW_ATTR_SNIFFER;],[HAVE_SNIFFER_EXP=yes], [HAVE_SNIFFER_EXP=no])
+AM_CONDITIONAL([HAVE_SNIFFER_EXP],[test "x$HAVE_SNIFFER_EXP" = "xyes"])
+if [test $HAVE_SNIFFER_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
+        AC_DEFINE([HAVE_SNIFFER_EXP], [1], [Enable Sniffer Flow Specification])
+fi
+
+AC_TRY_LINK([
+#include <infiniband/verbs.h>],
+        [int x = IBV_FLOW_ATTR_SNIFFER;],[HAVE_SNIFFER=yes], [HAVE_SNIFFER=no])
+AM_CONDITIONAL([HAVE_SNIFFER],[test "x$HAVE_SNIFFER" = "xyes"])
+if [test $HAVE_SNIFFER = yes] && [test $HAVE_VERBS_EXP = no]; then
+        AC_DEFINE([HAVE_SNIFFER], [1], [Enable Sniffer Flow Specification])
+fi
+
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+	[struct ibv_device_attr_ex *attr_ex; int x = attr_ex->comp_mask;],[HAVE_EX=yes], [HAVE_EX=no])
+AM_CONDITIONAL([HAVE_EX],[test "x$HAVE_EX" = "xyes"])
+if [test $HAVE_EX = yes]; then
+	AC_DEFINE([HAVE_EX], [1], [Have EX support])
+fi
+
+AC_TRY_LINK([
+#include <infiniband/verbs.h>],
+        [int x = IBV_ACCESS_ON_DEMAND;],[HAVE_EX_ODP=yes], [HAVE_EX_ODP=no])
+AM_CONDITIONAL([HAVE_EX_ODP],[test "x$HAVE_EX_ODP" = "xyes"])
+if [test $HAVE_EX_ODP = yes] && [test $HAVE_EX = yes]; then
+        AC_DEFINE([HAVE_EX_ODP], [1], [Have Extended  ODP support])
+fi
+
+AC_TRY_LINK([
+#include <infiniband/verbs.h>
+#include <infiniband/verbs_exp.h>],
+        [int x = IBV_EXP_ACCESS_ON_DEMAND;],[HAVE_EXP_ODP=yes], [HAVE_EXP_ODP=no])
+AM_CONDITIONAL([HAVE_EXP_ODP],[test "x$HAVE_EXP_ODP" = "xyes"])
+if [test $HAVE_EXP_ODP = yes] && [test $HAVE_VERBS_EXP = yes]; then
+	if  [test $HAVE_EX_ODP = no] ; then
+		AC_DEFINE([HAVE_EXP_ODP], [1], [Have Experimental ODP support])
+	fi
 fi
 
 if [test "$CUDA_H_PATH" ]; then
 	AC_DEFINE([HAVE_CUDA], [1], [Enable CUDA feature])
-	AC_DEFINE_UNQUOTED([CUDA_PATH], "$CUDA_H_PATH" , [Enable CUDA feature])
+	AC_DEFINE_UNQUOTED([CUDA_PATH], "$CUDA_H_PATH" , [Path to CUDA toolkit include directory])
 	LIBS=$LIBS" -lcuda"
 fi
 
@@ -223,7 +257,36 @@ if [test $HAVE_GID_ATTR = yes] && [test $HAVE_VERBS_EXP = yes]; then
         AC_DEFINE([HAVE_GID_ATTR], [1], [Enable GID Attribute query feature])
 fi
 
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+	[struct ibv_exp_qp_attr *attr; int x = attr->rate_limit;],[HAVE_PACKET_PACING_EXP=yes], [HAVE_PACKET_PACING_EXP=no])
+AM_CONDITIONAL([HAVE_PACKET_PACING_EXP],[test "x$HAVE_PACKET_PACING_EXP" = "xyes"])
+if [test $HAVE_PACKET_PACING_EXP = yes] && [test $HAVE_VERBS_EXP = yes]; then
+	AC_DEFINE([HAVE_PACKET_PACING_EXP], [1], [Have PACKET_PACING_EXP support])
+fi
+
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+	[struct ibv_qp_attr *attr; int x = attr->rate_limit;],[HAVE_PACKET_PACING=yes], [HAVE_PACKET_PACING=no])
+AM_CONDITIONAL([HAVE_PACKET_PACING],[test "x$HAVE_PACKET_PACING" = "xyes"])
+if [test $HAVE_PACKET_PACING = yes]; then
+	AC_DEFINE([HAVE_PACKET_PACING], [1], [Have PACKET_PACING support])
+fi
+
+AC_TRY_LINK([#include <infiniband/verbs.h>],
+	[int x = IBV_OOO_RW_DATA_PLACEMENT;], [HAVE_OOO_ATTR=yes], [HAVE_OOO_ATTR=no])
+AM_CONDITIONAL([HAVE_OOO_ATTR], [test "x$HAVE_OOO_ATTR" = "xyes"])
+if [test $HAVE_OOO_ATTR = yes]; then
+	AC_DEFINE([HAVE_OOO_ATTR], [1], [Have Out of order data placement support])
+fi
+
+AC_TRY_LINK([#include <infiniband/verbs_exp.h>],
+	[int x = IBV_EXP_OOO_SUPPORT_RW_DATA_PLACEMENT;], [HAVE_EXP_OOO_ATTR=yes], [HAVE_EXP_OOO_ATTR=no])
+AM_CONDITIONAL([HAVE_EXP_OOO_ATTR], [test "x$HAVE_EXP_OOO_ATTR" = "xyes"])
+if [test $HAVE_EXP_OOO_ATTR = yes]; then
+	AC_DEFINE([HAVE_EXP_OOO_ATTR], [1], [Have Experimental Out of order data placement support])
+fi
+
 CFLAGS="-g -Wall -D_GNU_SOURCE -O3"
+LIBS=$LIBS" -lpthread"
 AC_SUBST([LIBUMAD])
 AC_SUBST([LIBMATH])
 AC_CONFIG_FILES([Makefile])

debian/changelog

@@ -1,4 +1,4 @@
-perftest (3.0-3.1) stable; urgency=low
+perftest (4.2-0.0) stable; urgency=low
 
   * Initial release For Ubuntu (Closes: #182805)
 

perftest.spec

@@ -1,10 +1,10 @@
 Name:           perftest
 Summary:        IB Performance tests
-Version:        3.0
-Release:        4.0
+Version: 4.2
+Release: 0.2.g4f56894
 License:        BSD 3-Clause, GPL v2 or later
 Group:          Productivity/Networking/Diagnostic
-Source:         http://www.openfabrics.org/downloads/%{name}-%{version}.tar.gz
+Source: http://www.openfabrics.org/downloads/perftest-4.2-0.2.g4f56894.tar.gz
 Url:            http://www.openfabrics.org
 BuildRoot:      %{_tmppath}/%{name}-%{version}-build
 BuildRequires:  libibverbs-devel librdmacm-devel libibumad-devel

run_perftest_multi_devices

@@ -0,0 +1,182 @@
+#!/bin/sh
+# Script to launch a multi device test on seperate processes.
+
+GET_HELP=0
+HAVE_CORES=0
+HAVE_DEVICES=0
+HAVE_IB_PORTS=0
+HAVE_GID_INDEXES=0
+HAVE_REMOTE_HOST=0
+HAVE_CMD=0
+BASE_TCP_PORT=15000
+
+
+function force_dependencies {
+	if [[ $GET_HELP -eq 1 ]]
+	then
+		print_usage
+		exit
+	fi
+
+	#mandatory flags
+	if [[ $HAVE_DEVICES -eq 0 ]]
+	then
+		echo "-d/--devices flag is mandatory"
+		exit
+	fi
+
+	if [[ $HAVE_CMD -eq 0 ]]
+	then
+		echo "-C/--cmd flag is mandatory"
+		exit
+	fi
+
+	#optional flags
+	#check that all arguments have enough params.
+	if [[ $HAVE_CORES -eq 1 ]]
+	then
+		if [[ ${#CORES_LIST[@]} -ne $EXPECTED_PARAMS ]]
+		then
+			echo "number of cores should be equal to number of devices (cores for each device)"
+			exit
+		fi
+	fi
+
+	if [[ $HAVE_IB_PORTS -eq 1 ]]
+	then
+		if [[ ${#IB_PORTS_LIST[@]} -ne $EXPECTED_PARAMS ]]
+		then
+			echo "number of ib ports should be equal to number of devices (ib_port for each device)"
+			exit
+		fi
+	fi
+
+	if [[ $HAVE_GID_INDEXES -eq 1 ]]
+	then
+		if [[ ${#GID_INDEXES_LIST[@]} -ne $EXPECTED_PARAMS ]]
+		then
+			echo "number of gid indexes should be equal to number of devices (gid_index for each device)"
+			exit
+		fi
+	fi
+}
+
+function run_commands {
+	for (( I=0 ; I < $EXPECTED_PARAMS ; I++ ))
+	do
+		cmd=""
+		if [[ $HAVE_CORES -eq 1 ]]
+		then
+			cmd="taskset -c ${CORES_LIST[$I]}"
+		fi
+
+		#mandatory:
+		cmd="$cmd $TEST_CMD -d ${DEVICE_LIST[$I]} -p $(($BASE_TCP_PORT+I))"
+
+		#optional:
+		if [[ $HAVE_IB_PORTS -eq 1 ]]
+		then
+			cmd="$cmd -i ${IB_PORTS_LIST[$I]}"
+		fi
+
+		if [[ $HAVE_GID_INDEXES -eq 1 ]]
+		then
+			cmd="$cmd -x ${GID_INDEXES_LIST[$I]}"
+		fi
+
+		if [[ $HAVE_REMOTE_HOST -eq 1 ]]
+		then
+			cmd="$cmd $REMOTE_HOST"
+		fi
+
+		if [[ $I -ne $(($EXPECTED_PARAMS-1)) ]]
+		then
+			cmd="$cmd &"
+		fi
+
+		eval $cmd
+	done
+}
+
+function print_usage {
+	echo -e "\nUsage:"
+	echo "    Server side: run_perftest_multi_device --devices dev1,dev2 --cmd \"<perftest command>\" [optional_flags]"
+	echo "    Client side: run_perftest_multi_device --devices dev1,dev2 --cmd \"<perftest command>\" --remote <server_name> [optional_flags]"
+	echo ""
+	echo "    ** Please make sure that <perftest command> does not include the <server_name> on both sides. **"
+	echo "       This should be added only by --remote flag on the Client side."
+
+	echo -e "\nMandatory flags:"
+	echo "    -d, --devices       List of IB devices, seperated by comma. This will override '-d, --ib-dev' flag if existed in the perftest command."
+	echo "                        i.e. --devices dev1,dev2"
+
+	echo "    -C, --cmd           A valid perftest command."
+	echo "                        i.e. --cmd \"ib_write_bw --size 64 --duration 3\""
+
+	echo -e "\nOptional flags:"
+	echo "    -c,  --cores        Pin each device to a specific core using taskset"
+	echo "                        i.e. --cores 0,1  - This will pin dev1 command to core 0 and dev2 command to core 1"
+
+	echo "    -i,  --ib_ports     Choose ib_port for each device. This will override '-i, --ib-port' flag if existed in the perftest command."
+	echo "                        i.e. --ib_ports 1,2  - dev1 will work with port 1 and dev2 will work with port 2"
+
+	echo "    -x,  --gid_indexes  Choose gid_index for each device. This will override '-x, --gid-index' flag if existed in the perftest command."
+	echo "                        i.e. --gid_indexes 3,7  - dev1 will work with gid_index 3 and dev2 will work with gid_index 7"
+
+	echo "    -r,  --remote       Sets the remote host to connect. This will set it as the client side."
+	echo "                        i.e. --remote <server_host_name> , or --remote <server_ip>."
+}
+
+#parser
+while [[ $# -ge 1 ]]
+do
+	key="$1"
+
+	case $key in
+		-h|--help)
+		GET_HELP=1
+		shift
+		;;
+		-c|--cores)
+		CORES_LIST=($(echo "$2" | tr "," " "))
+		HAVE_CORES=1
+		shift # past argument
+		;;
+		-d|--devices)
+		DEVICE_LIST=($(echo "$2" | tr "," " "))
+		HAVE_DEVICES=1
+		EXPECTED_PARAMS=${#DEVICE_LIST[@]}
+		shift # past argument
+		;;
+		-i|--ib_ports)
+		IB_PORTS_LIST=($(echo "$2" | tr "," " "))
+		HAVE_IB_PORTS=1
+		shift # past argument
+		;;
+		-x|--gid_indexes)
+		GID_INDEXES_LIST=($(echo "$2" | tr "," " "))
+		HAVE_GID_INDEXES=1
+		shift
+		;;
+		-C|--cmd)
+		TEST_CMD="$2"
+		HAVE_CMD=1
+		shift
+		;;
+		-r|--remote)
+		REMOTE_HOST="$2"
+		HAVE_REMOTE_HOST=1
+		shift
+		;;
+		*)
+		# unknown option - ignore
+		;;
+	esac
+	shift
+done
+
+force_dependencies
+
+run_commands
+
+exit