Better, more configurable Docker support

.gitignore

@@ -0,0 +1,49 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# CMake
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps
+bin/
+configuration/
+volumes/
+containers/config/htpasswd

CMakeLists.txt

@@ -7,6 +7,7 @@ set (CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
+
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined")
 #set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined")
 #find_program(CLANG_TIDY_COMMAND NAMES clang-tidy clang-tidy-13 clang-tidy-12 clang-tidy-11 clang-tidy-10)
@@ -22,7 +23,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
 string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -g -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ")
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ")
-string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra ")
+string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra")
+#string(APPEND CMAKE_CXX_FLAGS " -v -lstdc++")
 
 ###
 # Mandatory: devices configuration file.

config/executor_manager.json

@@ -1,16 +1,23 @@
 {
   "config": {
-    "rdma_device": "",
+    "rdma_device": "eth0",
     "rdma_device_port": 10000,
-    "resource_manager_address": "",
-    "resource_manager_port": 0,
-    "resource_manager_secret": 0
+    "resource_manager_address": "172.31.82.200",
+    "resource_manager_port": 3000,
+    "resource_manager_secret": 12345
   },
   "executor": {
-    "use_docker": false,
     "repetitions": 100,
     "warmup_iters": 0,
-    "pin_threads": false
+    "pin_threads": false,
+    "docker": {
+      "use_docker": true,
+      "image": "rfaas-registry/rfaas-base",
+      "network": "mynet",
+      "ip": "172.31.82.202",
+      "volume": "/home/ubuntu/rfaas/containers/opt",
+      "registry_ip": "172.31.82.200",
+      "registry_port": 5000
+    }
   }
 }
-

containers/build_base.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -t rfaas-base:latest - < rfaas-base.Dockerfile

containers/init_docker.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# This script builds the rfaas-base docker container, pushes it to the
+# registry, sets up a docker volume, and generates example configuration 
+# json which should then be updated in config/executor_manager.json.
+# The script also configures a docker network for suitable use with
+# docker_rdma_sriov
+
+# NOTE: Run this script from repo root, and make sure the sriov docker plugin
+# is installed
+
+set -e
+
+if [ $# -lt 3 ]; then
+    echo "usage: ./init_docker.sh <REGISTRY IP> <REGISTRY PORT> <SUBNET>"
+    exit
+fi
+
+REG_IP=$1    # IP or name of the docker registry
+REG_PORT=$2  # Port of the docker registry
+SUBNET=$3    # Subnet for the docker network
+
+IMG_NAME=rfaas-base
+REG_IMG=$REG_IP:$REG_PORT/$IMG_NAME
+
+# Build the docker container, login and push to the registry
+sudo docker build -t $IMG_NAME - < containers/rfaas-base.Dockerfile
+echo "built rfaas-base image"
+sudo docker login $REG_IP:$REG_PORT
+echo "logged into docker daemon"
+
+if sudo docker push $REG_IMG; then
+    echo "ERROR: make sure a docker registry is actually running on $REG_IP:$REG_PORT.
+    Start one with scripts/run_registry.sh"
+    exit
+else
+    echo "pushed rfaas-base image to $REG_IMG"
+fi
+
+# Set up docker network
+net_name=testnet
+#sudo docker network create -d sriov --subnet=$SUBNET -o netdevice=$DEVICE $net_name
+echo "set up docker network"
+
+# Configure volume
+volume=$(pwd)/volumes/rfaas-test/opt # Do not put a trailing slash
+mkdir -p $volume/bin
+cp bin/executor $volume/bin
+cp examples/libfunctions.so $volume
+
+# Print json to be updated
+config=$(jq -n --arg use_docker "true" \
+    --arg image "$REG_IMG" \
+    --arg network "$net_name" \
+    --arg ip "<ip of container (un-used ip within $SUBNET)>" \
+    --arg volume $volume \
+    --arg registry_ip "$REG_IP" \
+    --arg registry_port "$REG_PORT" \
+    '{"docker": {
+        "use_docker": true,
+        "image": $image,
+        "network": $network,
+        "ip": $ip,
+        "volume": $volume,
+        "registry_ip": $registry_ip,
+        "registry_port": $registry_port
+    }}'
+)
+
+echo "Update config/executor_manager.json with"
+echo "$config"
+

containers/rfaas-base.Dockerfile

@@ -0,0 +1,14 @@
+# Container version should be same ubuntu version as where `executor`
+# was built (due to glibc versioning)
+
+FROM ubuntu:22.04
+
+#ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y && apt-get upgrade -y \
+    && apt-get install -y \
+    libibverbs-dev librdmacm-dev
+
+RUN mkdir -p /opt/bin
+WORKDIR "/opt/bin"
+

containers/test/net.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sudo docker network create -d sriov --subnet=172.31.80.0/20 -o netdevice=eth0 mynet

containers/test/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+#DOCKER=docker
+DOCKER=docker_rdma_sriov
+
+sudo $DOCKER run --rm --net=host -i --volume /home/ubuntu/rfaas/containers/opt:/opt rfaas-base /opt/executor $1
+
+#sudo docker exec -it base /bin/bash

rfaas/include/rfaas/rfaas.hpp

@@ -1,4 +1,3 @@
-
 #include <rfaas/connection.hpp>
 #include <rfaas/devices.hpp>
 #include <rfaas/executor.hpp>

scripts/push_image.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Push an image to the local docker registry
+
+if [ $# -lt 3 ]; then
+    echo "usage: ./push_image.sh <IMAGE NAME> <REGISTRY NAME or IP> <REGISTRY PORT>";
+    exit
+fi
+
+IMAGE=$1
+IP=$2
+PORT=$3
+docker push $IP:$PORT/$IMAGE

scripts/registry.yaml

@@ -0,0 +1,20 @@
+version: "3"
+
+services:
+  registry:
+    image: registry:2
+    container_name: rfaas-registry
+    ports:
+      - "5000:5000"
+    environment:
+      REGISTRY_AUTH: htpasswd
+      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
+      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
+      REGISTRY_STORAGE_DELETE_ENABLED: "true"
+      #REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
+      #REGISTRY_HTTP_TLS_KEY: /certs/domain.unencrypted.key
+      #REGISTRY_HTTP_SECRET: supersecrettext
+    volumes:
+      - /home/ubuntu/rfaas/containers/registry:/var/lib/registry
+      - /home/ubuntu/rfaas/containers/config:/auth
+      #- /home/ubuntu/rfaas/containers/config/certs:/certs

scripts/run_registry.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Run this script on the server where you want the docker registry to be hosted
+# Recommended to host the registry on the same server as the executor manager
+
+# NOTE: Run this script from the repo root
+
+PORT=5000
+NAME="rfaas-registry"
+
+set -e
+
+# Make password file if it doesn't already exist
+cfg=containers/config
+mkdir -p $cfg
+if [ -s $cfg/htpasswd ]; then
+    echo "htpasswd exists"
+else
+    sudo htpasswd -Bc $cfg/htpasswd $USER
+    echo "created htpasswd file"
+fi
+
+# Generate certs to use TLS (if they dont already exist)
+## if [ -s $cfg/certs/domain.key ]; then
+##     echo "using certs in $cfg/certs"
+## else
+##     mkdir -p $cfg/certs
+##     openssl genpkey -algorithm RSA -out $cfg/certs/domain.key -aes256
+## 
+##     openssl req -new               \
+##         -key $cfg/certs/domain.key \
+##         -out $cfg/certs/domain.csr \
+##         -addext 'subjectAltName = IP:172.31.82.200'
+## 
+##     openssl x509 -req -days 365        \
+##         -in $cfg/certs/domain.csr      \
+##         -signkey $cfg/certs/domain.key \
+##         -out $cfg/certs/domain.crt
+##     
+##     openssl rsa -in $cfg/certs/domain.key -out $cfg/certs/domain.unencrypted.key
+##     echo "generated certs in $cfg/certs"
+## fi
+
+# Start registry
+sudo docker-compose -f scripts/registry.yaml up -d
+echo "started docker registry $NAME on port $PORT"
+

server/executor/opts.cpp

@@ -25,11 +25,11 @@ namespace server {
       ("r,repetitions", "Repetitions to execute", cxxopts::value<int>()->default_value("1"))
       ("f,file", "Output server status.", cxxopts::value<std::string>())
       ("v,verbose", "Verbose output", cxxopts::value<bool>()->default_value("false"))
-      ("mgr-address", "Use selected address", cxxopts::value<std::string>())
-      ("mgr-port", "Use selected port", cxxopts::value<int>())
-      ("mgr-secret", "Use selected port", cxxopts::value<int>())
-      ("mgr-buf-addr", "Use selected port", cxxopts::value<uint64_t>())
-      ("mgr-buf-rkey", "Use selected port", cxxopts::value<uint32_t>())
+      ("mgr-address", "Use selected address to executor manager", cxxopts::value<std::string>())
+      ("mgr-port", "Use selected port to executor manager", cxxopts::value<int>())
+      ("mgr-secret", "Use selected secret", cxxopts::value<int>())
+      ("mgr-buf-addr", "Use selected buf addr", cxxopts::value<uint64_t>())
+      ("mgr-buf-rkey", "Use selected rkey", cxxopts::value<uint32_t>())
     ;
     auto parsed_options = options.parse(argc, argv);
 

server/executor_manager/client.hpp

@@ -26,7 +26,7 @@ namespace rfaas::executor_manager {
     rdmalib::Buffer<rdmalib::AllocationRequest> allocation_requests;
     rdmalib::RecvBuffer rcv_buffer;
     std::unique_ptr<ActiveExecutor> executor;
-    rdmalib::Buffer<Accounting> accounting;
+    rdmalib::Buffer<Accounting> accounting; // Timing data
     uint32_t allocation_time;
     bool _active;
 

server/executor_manager/executor_process.cpp

@@ -81,7 +81,7 @@ namespace rfaas::executor_manager {
       executor_pin_threads = std::to_string(0);//counter++);
     else
       executor_pin_threads = std::to_string(exec.pin_threads);
-    bool use_docker = exec.use_docker;
+    bool use_docker = exec.docker.use_docker;
 
     std::string mgr_port = std::to_string(conn.port);
     std::string mgr_secret = std::to_string(conn.secret);
@@ -92,6 +92,11 @@ namespace rfaas::executor_manager {
     if(mypid < 0) {
       spdlog::error("Fork failed! {}", mypid);
     }
+
+    // Child runs actual process (exec)
+    // Either runs docker or the executor binary
+    // Executor binary offloads work to a node, which is why it
+    // has an address and port field (along with other cli opts)
     if(mypid == 0) {
       mypid = getpid();
       auto out_file = ("executor_" + std::to_string(mypid));
@@ -129,46 +134,20 @@ namespace rfaas::executor_manager {
           exit(1);
         }
       } else {
-        //const char * argv[] = {
-        //  "docker_rdma_sriov", "run",
-        //  "--rm",
-        //  "--net=mynet", "-i", //"-it",
-        //  // FIXME: make configurable
-        //  "--ip=148.187.105.220",
-        //  // FIXME: make configurable
-        //  "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt",
-        //  // FIXME: make configurable
-        //  "rdma-test",
-        //  "/opt/bin/executor",
-        //  "-a", client_addr.c_str(),
-        //  "-p", client_port.c_str(),
-        //  "--polling-mgr", "thread",
-        //  "-r", executor_repetitions.c_str(),
-        //  "-x", executor_recv_buf.c_str(),
-        //  "-s", client_in_size.c_str(),
-        //  "--pin-threads", "true",
-        //  "--fast", client_cores.c_str(),
-        //  "--warmup-iters", executor_warmups.c_str(),
-        //  "--max-inline-data", executor_max_inline.c_str(),
-        //  "--func-size", client_func_size.c_str(),
-        //  "--timeout", client_timeout.c_str(),
-        //  "--mgr-address", conn.addr.c_str(),
-        //  "--mgr-port", mgr_port.c_str(),
-        //  "--mgr-secret", mgr_secret.c_str(),
-        //  "--mgr-buf-addr", mgr_buf_addr.c_str(),
-        //  "--mgr-buf-rkey", mgr_buf_rkey.c_str(),
-        //  nullptr
-        //};
+        std::string ip_arg = "--ip=" + exec.docker.ip;
+        std::string volume_arg = exec.docker.volume + ":/opt";
+        std::string net_arg = "--net=" + exec.docker.network;
+        std::string registry_port = std::to_string(exec.docker.registry_port);
+        std::string docker_image = exec.docker.registry_ip + ":" + registry_port
+            + "/" + exec.docker.image;
+
         const char * argv[] = {
           "docker_rdma_sriov", "run",
-          "--rm",
-          "--net=mynet", "-i", //"-it",
-          // FIXME: make configurable
-          "--ip=148.187.105.250",
-          // FIXME: make configurable
-          "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt",
-          // FIXME: make configurable
-          "rdma-test",
+          "--rm", "-i",
+          net_arg.c_str(),
+          ip_arg.c_str(),
+          "--volume", volume_arg.c_str(),
+          docker_image.c_str(),
           "/opt/bin/executor",
           "-a", client_addr.c_str(),
           "-p", client_port.c_str(),