diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8364c9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +bin/ +configuration/ +volumes/ +containers/config/htpasswd diff --git a/CMakeLists.txt b/CMakeLists.txt index 37ccf58..b68ca70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,7 @@ set (CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) + #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined") #set(CMAKE_LINKER_FLAGS "${CMAKE_LINKER_FLAGS} -fno-omit-frame-pointer -fsanitize=undefined") #find_program(CLANG_TIDY_COMMAND NAMES clang-tidy clang-tidy-13 clang-tidy-12 clang-tidy-11 clang-tidy-10) @@ -22,7 +23,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) string(REPLACE "-DNDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -g -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ") string(APPEND CMAKE_CXX_FLAGS_DEBUG " -DSPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_DEBUG ") -string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra ") +string(APPEND CMAKE_CXX_FLAGS " -Wall -Wextra") +#string(APPEND CMAKE_CXX_FLAGS " -v -lstdc++") ### # Mandatory: devices configuration file. diff --git a/benchmarks/cold_benchmarker b/benchmarks/cold_benchmarker new file mode 100755 index 0000000..dabde78 Binary files /dev/null and b/benchmarks/cold_benchmarker differ diff --git a/benchmarks/cpp_interface b/benchmarks/cpp_interface new file mode 100755 index 0000000..79636ef Binary files /dev/null and b/benchmarks/cpp_interface differ diff --git a/benchmarks/parallel_invocations b/benchmarks/parallel_invocations new file mode 100755 index 0000000..8e91cd2 Binary files /dev/null and b/benchmarks/parallel_invocations differ diff --git a/benchmarks/warm_benchmarker b/benchmarks/warm_benchmarker new file mode 100755 index 0000000..a1590fd Binary files /dev/null and b/benchmarks/warm_benchmarker differ diff --git a/config/executor_manager.json b/config/executor_manager.json index 6e4cf8b..d1f36a9 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -1,16 +1,23 @@ { "config": { - "rdma_device": "", + "rdma_device": "eth0", "rdma_device_port": 10000, - "resource_manager_address": "", - "resource_manager_port": 0, - "resource_manager_secret": 0 + "resource_manager_address": "172.31.82.200", + "resource_manager_port": 3000, + "resource_manager_secret": 12345 }, "executor": { - "use_docker": false, "repetitions": 100, "warmup_iters": 0, - "pin_threads": false + "pin_threads": false, + "docker": { + "use_docker": true, + "image": "rfaas-registry/rfaas-base", + "network": "mynet", + "ip": "172.31.82.202", + "volume": "/home/ubuntu/rfaas/containers/opt", + "registry_ip": "172.31.82.200", + "registry_port": 5000 + } } } - diff --git a/containers/build_base.sh b/containers/build_base.sh new file mode 100755 index 0000000..2b2657a --- /dev/null +++ b/containers/build_base.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build -t rfaas-base:latest - < rfaas-base.Dockerfile diff --git a/containers/init_docker.sh b/containers/init_docker.sh new file mode 100755 index 0000000..5f976fc --- /dev/null +++ b/containers/init_docker.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# This script builds the rfaas-base docker container, pushes it to the +# registry, sets up a docker volume, and generates example configuration +# json which should then be updated in config/executor_manager.json. +# The script also configures a docker network for suitable use with +# docker_rdma_sriov + +# NOTE: Run this script from repo root, and make sure the sriov docker plugin +# is installed + +set -e + +if [ $# -lt 3 ]; then + echo "usage: ./init_docker.sh " + exit +fi + +REG_IP=$1 # IP or name of the docker registry +REG_PORT=$2 # Port of the docker registry +SUBNET=$3 # Subnet for the docker network + +IMG_NAME=rfaas-base +REG_IMG=$REG_IP:$REG_PORT/$IMG_NAME + +# Build the docker container, login and push to the registry +sudo docker build -t $IMG_NAME - < containers/rfaas-base.Dockerfile +echo "built rfaas-base image" +sudo docker login $REG_IP:$REG_PORT +echo "logged into docker daemon" + +if sudo docker push $REG_IMG; then + echo "ERROR: make sure a docker registry is actually running on $REG_IP:$REG_PORT. + Start one with scripts/run_registry.sh" + exit +else + echo "pushed rfaas-base image to $REG_IMG" +fi + +# Set up docker network +net_name=testnet +#sudo docker network create -d sriov --subnet=$SUBNET -o netdevice=$DEVICE $net_name +echo "set up docker network" + +# Configure volume +volume=$(pwd)/volumes/rfaas-test/opt # Do not put a trailing slash +mkdir -p $volume/bin +cp bin/executor $volume/bin +cp examples/libfunctions.so $volume + +# Print json to be updated +config=$(jq -n --arg use_docker "true" \ + --arg image "$REG_IMG" \ + --arg network "$net_name" \ + --arg ip "" \ + --arg volume $volume \ + --arg registry_ip "$REG_IP" \ + --arg registry_port "$REG_PORT" \ + '{"docker": { + "use_docker": true, + "image": $image, + "network": $network, + "ip": $ip, + "volume": $volume, + "registry_ip": $registry_ip, + "registry_port": $registry_port + }}' +) + +echo "Update config/executor_manager.json with" +echo "$config" + diff --git a/containers/opt/bin/executor b/containers/opt/bin/executor new file mode 100755 index 0000000..b3c0b18 Binary files /dev/null and b/containers/opt/bin/executor differ diff --git a/containers/opt/libfunctions.so b/containers/opt/libfunctions.so new file mode 100755 index 0000000..e95a369 Binary files /dev/null and b/containers/opt/libfunctions.so differ diff --git a/containers/rfaas-base.Dockerfile b/containers/rfaas-base.Dockerfile new file mode 100644 index 0000000..401297e --- /dev/null +++ b/containers/rfaas-base.Dockerfile @@ -0,0 +1,14 @@ +# Container version should be same ubuntu version as where `executor` +# was built (due to glibc versioning) + +FROM ubuntu:22.04 + +#ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && apt-get upgrade -y \ + && apt-get install -y \ + libibverbs-dev librdmacm-dev + +RUN mkdir -p /opt/bin +WORKDIR "/opt/bin" + diff --git a/containers/test/net.sh b/containers/test/net.sh new file mode 100755 index 0000000..bea6c96 --- /dev/null +++ b/containers/test/net.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +sudo docker network create -d sriov --subnet=172.31.80.0/20 -o netdevice=eth0 mynet diff --git a/containers/test/run.sh b/containers/test/run.sh new file mode 100755 index 0000000..da215d1 --- /dev/null +++ b/containers/test/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +#DOCKER=docker +DOCKER=docker_rdma_sriov + +sudo $DOCKER run --rm --net=host -i --volume /home/ubuntu/rfaas/containers/opt:/opt rfaas-base /opt/executor $1 + +#sudo docker exec -it base /bin/bash diff --git a/examples/libfunctions.so b/examples/libfunctions.so new file mode 100755 index 0000000..6d1e263 Binary files /dev/null and b/examples/libfunctions.so differ diff --git a/rfaas/include/rfaas/rfaas.hpp b/rfaas/include/rfaas/rfaas.hpp index 6eba012..af3370d 100644 --- a/rfaas/include/rfaas/rfaas.hpp +++ b/rfaas/include/rfaas/rfaas.hpp @@ -1,4 +1,3 @@ - #include #include #include diff --git a/scripts/push_image.sh b/scripts/push_image.sh new file mode 100755 index 0000000..40beeb3 --- /dev/null +++ b/scripts/push_image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Push an image to the local docker registry + +if [ $# -lt 3 ]; then + echo "usage: ./push_image.sh "; + exit +fi + +IMAGE=$1 +IP=$2 +PORT=$3 +docker push $IP:$PORT/$IMAGE diff --git a/scripts/registry.yaml b/scripts/registry.yaml new file mode 100644 index 0000000..3ed7a10 --- /dev/null +++ b/scripts/registry.yaml @@ -0,0 +1,20 @@ +version: "3" + +services: + registry: + image: registry:2 + container_name: rfaas-registry + ports: + - "5000:5000" + environment: + REGISTRY_AUTH: htpasswd + REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd + REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm + REGISTRY_STORAGE_DELETE_ENABLED: "true" + #REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt + #REGISTRY_HTTP_TLS_KEY: /certs/domain.unencrypted.key + #REGISTRY_HTTP_SECRET: supersecrettext + volumes: + - /home/ubuntu/rfaas/containers/registry:/var/lib/registry + - /home/ubuntu/rfaas/containers/config:/auth + #- /home/ubuntu/rfaas/containers/config/certs:/certs diff --git a/scripts/run_registry.sh b/scripts/run_registry.sh new file mode 100755 index 0000000..d3cb0eb --- /dev/null +++ b/scripts/run_registry.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Run this script on the server where you want the docker registry to be hosted +# Recommended to host the registry on the same server as the executor manager + +# NOTE: Run this script from the repo root + +PORT=5000 +NAME="rfaas-registry" + +set -e + +# Make password file if it doesn't already exist +cfg=containers/config +mkdir -p $cfg +if [ -s $cfg/htpasswd ]; then + echo "htpasswd exists" +else + sudo htpasswd -Bc $cfg/htpasswd $USER + echo "created htpasswd file" +fi + +# Generate certs to use TLS (if they dont already exist) +## if [ -s $cfg/certs/domain.key ]; then +## echo "using certs in $cfg/certs" +## else +## mkdir -p $cfg/certs +## openssl genpkey -algorithm RSA -out $cfg/certs/domain.key -aes256 +## +## openssl req -new \ +## -key $cfg/certs/domain.key \ +## -out $cfg/certs/domain.csr \ +## -addext 'subjectAltName = IP:172.31.82.200' +## +## openssl x509 -req -days 365 \ +## -in $cfg/certs/domain.csr \ +## -signkey $cfg/certs/domain.key \ +## -out $cfg/certs/domain.crt +## +## openssl rsa -in $cfg/certs/domain.key -out $cfg/certs/domain.unencrypted.key +## echo "generated certs in $cfg/certs" +## fi + +# Start registry +sudo docker-compose -f scripts/registry.yaml up -d +echo "started docker registry $NAME on port $PORT" + diff --git a/server/executor/opts.cpp b/server/executor/opts.cpp index 5e5adc8..9dfabf2 100644 --- a/server/executor/opts.cpp +++ b/server/executor/opts.cpp @@ -25,11 +25,11 @@ namespace server { ("r,repetitions", "Repetitions to execute", cxxopts::value()->default_value("1")) ("f,file", "Output server status.", cxxopts::value()) ("v,verbose", "Verbose output", cxxopts::value()->default_value("false")) - ("mgr-address", "Use selected address", cxxopts::value()) - ("mgr-port", "Use selected port", cxxopts::value()) - ("mgr-secret", "Use selected port", cxxopts::value()) - ("mgr-buf-addr", "Use selected port", cxxopts::value()) - ("mgr-buf-rkey", "Use selected port", cxxopts::value()) + ("mgr-address", "Use selected address to executor manager", cxxopts::value()) + ("mgr-port", "Use selected port to executor manager", cxxopts::value()) + ("mgr-secret", "Use selected secret", cxxopts::value()) + ("mgr-buf-addr", "Use selected buf addr", cxxopts::value()) + ("mgr-buf-rkey", "Use selected rkey", cxxopts::value()) ; auto parsed_options = options.parse(argc, argv); diff --git a/server/executor_manager/client.hpp b/server/executor_manager/client.hpp index b535ad4..a649d61 100644 --- a/server/executor_manager/client.hpp +++ b/server/executor_manager/client.hpp @@ -26,7 +26,7 @@ namespace rfaas::executor_manager { rdmalib::Buffer allocation_requests; rdmalib::RecvBuffer rcv_buffer; std::unique_ptr executor; - rdmalib::Buffer accounting; + rdmalib::Buffer accounting; // Timing data uint32_t allocation_time; bool _active; diff --git a/server/executor_manager/executor_process.cpp b/server/executor_manager/executor_process.cpp index afdbf2d..50f701b 100644 --- a/server/executor_manager/executor_process.cpp +++ b/server/executor_manager/executor_process.cpp @@ -81,7 +81,7 @@ namespace rfaas::executor_manager { executor_pin_threads = std::to_string(0);//counter++); else executor_pin_threads = std::to_string(exec.pin_threads); - bool use_docker = exec.use_docker; + bool use_docker = exec.docker.use_docker; std::string mgr_port = std::to_string(conn.port); std::string mgr_secret = std::to_string(conn.secret); @@ -92,6 +92,11 @@ namespace rfaas::executor_manager { if(mypid < 0) { spdlog::error("Fork failed! {}", mypid); } + + // Child runs actual process (exec) + // Either runs docker or the executor binary + // Executor binary offloads work to a node, which is why it + // has an address and port field (along with other cli opts) if(mypid == 0) { mypid = getpid(); auto out_file = ("executor_" + std::to_string(mypid)); @@ -129,46 +134,20 @@ namespace rfaas::executor_manager { exit(1); } } else { - //const char * argv[] = { - // "docker_rdma_sriov", "run", - // "--rm", - // "--net=mynet", "-i", //"-it", - // // FIXME: make configurable - // "--ip=148.187.105.220", - // // FIXME: make configurable - // "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // // FIXME: make configurable - // "rdma-test", - // "/opt/bin/executor", - // "-a", client_addr.c_str(), - // "-p", client_port.c_str(), - // "--polling-mgr", "thread", - // "-r", executor_repetitions.c_str(), - // "-x", executor_recv_buf.c_str(), - // "-s", client_in_size.c_str(), - // "--pin-threads", "true", - // "--fast", client_cores.c_str(), - // "--warmup-iters", executor_warmups.c_str(), - // "--max-inline-data", executor_max_inline.c_str(), - // "--func-size", client_func_size.c_str(), - // "--timeout", client_timeout.c_str(), - // "--mgr-address", conn.addr.c_str(), - // "--mgr-port", mgr_port.c_str(), - // "--mgr-secret", mgr_secret.c_str(), - // "--mgr-buf-addr", mgr_buf_addr.c_str(), - // "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - // nullptr - //}; + std::string ip_arg = "--ip=" + exec.docker.ip; + std::string volume_arg = exec.docker.volume + ":/opt"; + std::string net_arg = "--net=" + exec.docker.network; + std::string registry_port = std::to_string(exec.docker.registry_port); + std::string docker_image = exec.docker.registry_ip + ":" + registry_port + + "/" + exec.docker.image; + const char * argv[] = { "docker_rdma_sriov", "run", - "--rm", - "--net=mynet", "-i", //"-it", - // FIXME: make configurable - "--ip=148.187.105.250", - // FIXME: make configurable - "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // FIXME: make configurable - "rdma-test", + "--rm", "-i", + net_arg.c_str(), + ip_arg.c_str(), + "--volume", volume_arg.c_str(), + docker_image.c_str(), "/opt/bin/executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), diff --git a/server/executor_manager/executor_process.hpp b/server/executor_manager/executor_process.hpp index c5f266a..de4a7a2 100644 --- a/server/executor_manager/executor_process.hpp +++ b/server/executor_manager/executor_process.hpp @@ -45,6 +45,8 @@ namespace rfaas::executor_manager { virtual std::tuple check() const = 0; }; + // Ac actual process which can be spawned/is executing + // uses fork/exec pattern struct ProcessExecutor : public ActiveExecutor { pid_t _pid; diff --git a/server/executor_manager/manager.cpp b/server/executor_manager/manager.cpp index 1e4197a..2eda2f4 100644 --- a/server/executor_manager/manager.cpp +++ b/server/executor_manager/manager.cpp @@ -185,7 +185,7 @@ namespace rfaas::executor_manager { if(cores > 0) { spdlog::info( "Client {} requests executor with {} threads, it should connect to {}:{}," - "it should have buffer of size {}, func buffer {}, and hot timeout {}", + "it should have buffer of size {}, func buffer {}, hot timeout {}", i, client.allocation_requests.data()[id].cores, client.allocation_requests.data()[id].listen_address, client.allocation_requests.data()[id].listen_port, diff --git a/server/executor_manager/opts.cpp b/server/executor_manager/opts.cpp index c1712d0..e9e24eb 100644 --- a/server/executor_manager/opts.cpp +++ b/server/executor_manager/opts.cpp @@ -13,9 +13,15 @@ namespace rfaas::executor_manager { ("device-database", "JSON configuration of devices.", cxxopts::value()) ("skip-resource-manager", "Ignore resource manager and don't connect to it.", cxxopts::value()->default_value("false")) ("v,verbose", "Verbose output", cxxopts::value()->default_value("false")) + ("h,help", "Print usage") ; auto parsed_options = options.parse(argc, argv); + if (parsed_options.count("help")) { + std::cout << options.help() << std::endl; + exit(0); + } + Options result; result.json_config = parsed_options["config"].as(); result.device_database = parsed_options["device-database"].as(); diff --git a/server/executor_manager/settings.cpp b/server/executor_manager/settings.cpp index a5fcc14..1a4673e 100644 --- a/server/executor_manager/settings.cpp +++ b/server/executor_manager/settings.cpp @@ -11,7 +11,7 @@ namespace rfaas::executor_manager { cereal::JSONInputArchive archive_in(in); archive_in(cereal::make_nvp("config", settings)); archive_in(cereal::make_nvp("executor", settings.exec)); - + // read RDMA device details rfaas::device_data * dev = rfaas::devices::instance().device(settings.rdma_device); if(!dev) { diff --git a/server/executor_manager/settings.hpp b/server/executor_manager/settings.hpp index 5347c07..78b1d38 100644 --- a/server/executor_manager/settings.hpp +++ b/server/executor_manager/settings.hpp @@ -11,20 +11,43 @@ namespace rfaas::executor_manager { - struct ExecutorSettings + struct DockerSettings { bool use_docker; + std::string image; + std::string network; + std::string ip; + std::string volume; + std::string registry_ip; + int registry_port; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(use_docker), CEREAL_NVP(image), + CEREAL_NVP(network), CEREAL_NVP(ip), + CEREAL_NVP(volume), + CEREAL_NVP(registry_ip), CEREAL_NVP(registry_port) + ); + } + }; + + struct ExecutorSettings + { int repetitions; int warmup_iters; int recv_buffer_size; int max_inline_data; bool pin_threads; + struct DockerSettings docker; + template - void load(Archive & ar ) + void load(Archive & ar) { ar( - CEREAL_NVP(use_docker), CEREAL_NVP(repetitions), + CEREAL_NVP(docker), CEREAL_NVP(repetitions), CEREAL_NVP(warmup_iters), CEREAL_NVP(pin_threads) ); } diff --git a/server/resource_manager/manager.cpp b/server/resource_manager/manager.cpp index 21eb909..713b843 100644 --- a/server/resource_manager/manager.cpp +++ b/server/resource_manager/manager.cpp @@ -35,6 +35,8 @@ namespace rfaas::resource_manager { } + // Listen for incoming RDMA connections + // If there is one, accept it and add it to the queue void Manager::listen_rdma() { while(!_shutdown.load()) { @@ -66,6 +68,7 @@ namespace rfaas::resource_manager { spdlog::info("Background thread stops waiting for rdmacm events"); } + // For each connection in the queue, add it to the vector of connections void Manager::process_rdma() { std::vector vec;