[Resolve #25] Merge changes from #25 into libfabric-sarus branch

.gitignore

@@ -0,0 +1,49 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# CMake
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+bin/
+configuration/
+volumes/
+containers/config/htpasswd

benchmarks/warm_benchmark.cpp

@@ -33,9 +33,11 @@ int main(int argc, char ** argv)
   rfaas::devices::deserialize(in_dev);
   in_dev.close();
 
+  #ifdef USE_GNI_AUTH
   rdmalib::Configuration::get_instance().configure_cookie(
     rfaas::devices::instance()._configuration.authentication_credential
   );
+  #endif
 
   // Read benchmark settings
   std::ifstream benchmark_cfg{opts.json_config};

config/executor_manager.json

@@ -1,13 +1,13 @@
 {
   "config": {
     "rdma_device": "",
-    "rdma_device_port": ,
+    "rdma_device_port": 10004,
     "resource_manager_address": "",
     "resource_manager_port": 0,
     "resource_manager_secret": 0
   },
   "executor": {
-    "sandbox-type": "process",
+    "sandbox_type": "process",
     "repetitions": 100,
     "warmup_iters": 0,
     "pin_threads": false
@@ -16,30 +16,57 @@
     {
       "key": "sarus",
       "value": {
-        "devices": [
-          "/dev/kgni0", "dev/kdreg"
-        ],
-        "mounts": [
-          "/opt/cray",
-          "/tmp/drcc.sock",
-          "/etc/opt/cray/rdma-credentials",
-          "/etc/alternatives/cray-ugni",
-          "/etc/alternatives/cray-xpmem",
-          "/etc/alternatives/cray-alps",
-          "/etc/alternatives/cray-udreg",
-          "/etc/alternatives/cray-wlm_detect"
-        ],
-        "mount_filesystem": [
-          "/scratch/snx3000/{user}"
-        ],
-        "env": [
-          {
-            "key": "LD_LIBRARY_PATH",
-            "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/"
-          }
-        ]
+        "index": 1,
+        "data": {
+          "user": "mcopik",
+          "name": "spcleth/hpc-disagg:rfaas-executor-daint",
+          "devices": [
+            "/dev/kgni0", "dev/kdreg"
+          ],
+          "mounts": [
+            "/opt/cray",
+            "/tmp/drcc.sock",
+            "/etc/opt/cray/rdma-credentials",
+            "/etc/alternatives/cray-ugni",
+            "/etc/alternatives/cray-xpmem",
+            "/etc/alternatives/cray-alps",
+            "/etc/alternatives/cray-udreg",
+            "/etc/alternatives/cray-wlm_detect"
+          ],
+          "mount_filesystem": [
+            "/scratch/snx3000/{user}"
+          ],
+          "env": [
+            {
+              "key": "LD_LIBRARY_PATH",
+              "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/"
+            }
+          ]
+        }
+      }
+    },
+    {
+      "key": "docker",
+      "value": {
+        "index": 0,
+        "data": {
+          "image": "rfaas-registry/rfaas-base",
+          "network": "mynet",
+          "ip": "172.31.82.202",
+          "volume": "/home/ubuntu/rfaas/containers/opt",
+          "registry_ip": "172.31.82.200",
+          "registry_port": 5000
+        }
+      }
+    },
+    {
+      "key": "singularity",
+      "value": {
+        "index": 2,
+        "data": {
+          "container": "rfaas-container-singularity"
+        }
       }
     }
   ]
 }
-

config/executor_manager_backup.json

@@ -0,0 +1,63 @@
+{
+  "config": {
+    "rdma_device": "",
+    "rdma_device_port": ,
+    "resource_manager_address": "",
+    "resource_manager_port": 0,
+    "resource_manager_secret": 0
+  },
+  "executor": {
+    "sandbox-type": "process",
+    "repetitions": 100,
+    "warmup_iters": 0,
+    "pin_threads": false
+  },
+  "sandbox-configuration": {
+    "sandboxes": [
+      {
+        "key": "sarus",
+        "value": {
+          "user": "mcopik",
+          "name": "spcleth/hpc-disagg:rfaas-executor-daint",
+          "devices": [
+            "/dev/kgni0", "dev/kdreg"
+          ],
+          "mounts": [
+            "/opt/cray",
+            "/tmp/drcc.sock",
+            "/etc/opt/cray/rdma-credentials",
+            "/etc/alternatives/cray-ugni",
+            "/etc/alternatives/cray-xpmem",
+            "/etc/alternatives/cray-alps",
+            "/etc/alternatives/cray-udreg",
+            "/etc/alternatives/cray-wlm_detect"
+          ],
+          "mount_filesystem": [
+            "/scratch/snx3000/{user}"
+          ],
+          "env": [
+            {
+              "key": "LD_LIBRARY_PATH",
+              "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/"
+            }
+          ]
+        }
+      },
+      {
+        "key": "docker",
+        "value": {
+          "devices": [
+            "dev1", "dev2"
+          ],
+          "image": "rfaas-registry/rfaas-base",
+          "network": "mynet",
+          "ip": "172.31.82.202",
+          "volume": "/home/ubuntu/rfaas/containers/opt",
+          "registry_ip": "172.31.82.200",
+          "registry_port": 5000
+        }
+      }
+    ]
+  }
+}
+

containers/build_base.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t rfaas-base:latest - < rfaas-base.Dockerfile

containers/init_docker.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# This script builds the rfaas-base docker container, pushes it to the
+# registry, sets up a docker volume, and generates example configuration 
+# json which should then be updated in config/executor_manager.json.
+# The script also configures a docker network for suitable use with
+# docker_rdma_sriov
+
+# NOTE: Run this script from repo root, and make sure the sriov docker plugin
+# is installed
+
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "usage: ./init_docker.sh <REGISTRY IP> <REGISTRY PORT> <NETWORK MODE> [<NETWORK DEVICE> <SUBNET>]"
+    exit
+fi
+
+REG_IP=$1    # IP or name of the docker registry
+REG_PORT=$2  # Port of the docker registry
+NET_MODE=$3  # Docker networking mode -- sriov or host
+DEVICE=$4    # The RDMA adapter to use for networking
+SUBNET=$5    # Subnet for the docker network
+
+IMG_NAME=rfaas-base
+REG_IMG=$REG_IP:$REG_PORT/$IMG_NAME
+
+# Build the docker container, login and push to the registry
+docker build -t $IMG_NAME - < containers/rfaas-base.Dockerfile
+echo "built rfaas-base image"
+docker login $REG_IP:$REG_PORT
+echo "logged into docker daemon"
+
+if docker push $REG_IMG; then
+    echo "ERROR: make sure a docker registry is actually running on $REG_IP:$REG_PORT.
+    Start one with scripts/run_registry.sh"
+    exit
+else
+    echo "pushed rfaas-base image to $REG_IMG"
+fi
+
+# Set up docker network
+net_name=testnet
+if ["$NET_MODE" = "sriov"]; then
+    docker network create -d sriov --subnet=$SUBNET -o netdevice=$DEVICE $net_name
+elif ["$NET_MODE" = "host"]; then
+    net_name="host"
+else
+    echo "ERROR: invalid networking mode $NET_MODE. Valid options are sriov and host."
+    exit
+fi
+echo "set up docker network"
+
+# Configure volume
+volume=$(pwd)/volumes/rfaas-test/opt # Do not put a trailing slash
+mkdir -p $volume/bin
+cp bin/executor $volume/bin
+cp examples/libfunctions.so $volume
+
+# Print json to be updated
+config=$(jq -n --arg use_docker "true" \
+    --arg image "$REG_IMG" \
+    --arg network "$net_name" \
+    --arg ip "<ip of container (any available ip within $SUBNET)>" \
+    --arg volume $volume \
+    --arg registry_ip "$REG_IP" \
+    --arg registry_port "$REG_PORT" \
+    '{
+        "image": $image,
+        "network": $network,
+        "ip": $ip,
+        "volume": $volume,
+        "registry_ip": $registry_ip,
+        "registry_port": $registry_port
+    }'
+)
+
+echo "Update config/executor_manager.json with"
+echo "$config"
+

containers/rfaas-base.Dockerfile

@@ -0,0 +1,14 @@
+# Container version should be same ubuntu version as where `executor`
+# was built (due to glibc versioning)
+
+FROM ubuntu:22.04
+
+#ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y && apt-get upgrade -y \
+    && apt-get install -y \
+    libibverbs-dev librdmacm-dev
+
+RUN mkdir -p /opt/bin
+WORKDIR "/opt/bin"
+

rdmalib/include/rdmalib/buffer.hpp

@@ -61,10 +61,11 @@ namespace rdmalib {
       #endif
       #ifdef USE_LIBFABRIC
       void *lkey() const;
+      uint64_t rkey() const;
       #else
       uint32_t lkey() const;
+      uint32_t rkey() const;
       #endif
-      uint64_t rkey() const;
       ScatterGatherElement sge(uint32_t size, uint32_t offset) const;
     };
 
@@ -79,6 +80,12 @@ namespace rdmalib {
     // When accessing the remote buffer, we might not need to know the size.
     RemoteBuffer(uintptr_t addr, uint64_t rkey, uint32_t size = 0);
 
+    #ifdef USE_LIBFABRIC
+    RemoteBuffer(uintptr_t addr, uint64_t rkey, uint32_t size);
+    #else
+    RemoteBuffer(uintptr_t addr, uint32_t rkey, uint32_t size);
+    #endif
+
     template<class Archive>
     void serialize(Archive & ar)
     {

rdmalib/include/rdmalib/rdmalib.hpp

@@ -25,6 +25,7 @@ extern "C" {
 
 #include <rdmalib/buffer.hpp>
 #include <rdmalib/connection.hpp>
+#include <mutex>
 
 namespace rdmalib {
 
@@ -45,7 +46,9 @@ namespace rdmalib {
     ~Configuration();
 
     std::once_flag _access_flag;
+    #ifdef USE_GNI_AUTH
     drc_info_handle_t _credential_info;
+    #endif
     uint64_t _cookie;
     uint32_t _credential;
     bool _is_configured;

rdmalib/lib/buffer.cpp

@@ -247,6 +247,12 @@ namespace rdmalib {
     size(0)
   {}
 
+  RemoteBuffer::RemoteBuffer(uintptr_t addr, uint64_t rkey, uint32_t size):
+    addr(addr),
+    rkey(rkey),
+    size(size)
+  {}
+
   #ifdef USE_LIBFABRIC
   RemoteBuffer::RemoteBuffer(uintptr_t addr, uint64_t rkey, uint32_t size):
     addr(addr),