Merge branch 'main' into valgrindClean

awslabs · Feb 6, 2024 · fb63327 · fb63327
2 parents fb027a0 + da10323
commit fb63327
Show file tree

Hide file tree

Showing 139 changed files with 9,126 additions and 1,330 deletions.
diff --git a/.github/build.sh b/.github/build.sh
@@ -2,7 +2,6 @@
 
 compileMode="default"
 stMode=false
-sharedLibs=false
 OPENSSL_VERSION=1.1.1n
 
 # Check if first argument is compile mode
@@ -13,9 +12,6 @@ if [ "$compileModeArgument" = "compile-mode" ]; then
     st_component_mode)
     compileMode="st_component_mode"
     ;;
-    shared_lib_mode)
-    sharedLibs=true
-    ;;
     armhf_cross_mode)
     compileMode="armhf_cross_mode"
     ;;
@@ -25,18 +21,6 @@ if [ "$compileModeArgument" = "compile-mode" ]; then
     aarch64_cross_mode)
     compileMode="aarch64_cross_mode"
     ;;
-    armhf_cross_mode_shared_libs)
-    compileMode="armhf_cross_mode"
-    sharedLibs=true
-    ;;
-    mips_cross_mode_shared_libs)
-    compileMode="mips_cross_mode"
-    sharedLibs=true
-    ;;
-    aarch64_cross_mode_shared_libs)
-    compileMode="aarch64_cross_mode"
-    sharedLibs=true
-    ;;
     st_armhf_cross_mode)
     compileMode="armhf_cross_mode"
     stMode=true
@@ -126,10 +110,6 @@ case $compileMode in
     if [ "$stMode" = true ]; then
       # Set CMake flags for ST mode
       cmake -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-armhf.cmake -DBUILD_SDK=ON -DEXCLUDE_JOBS=ON -DEXCLUDE_DD=ON -DEXCLUDE_FP=ON -DDISABLE_MQTT=ON -DEXCLUDE_SENSOR_PUBLISH=ON ../
-    elif [ "$sharedLibs" = true ]; then
-      cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-armhf.cmake -DBUILD_SDK=ON ../
-      make install DESTDIR=./shared_install_dir
-      chmod 0777 ./shared_install_dir
     else
       cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-armhf.cmake ../
     fi
@@ -156,13 +136,7 @@ case $compileMode in
     make -j 4
     make install
     cd ..
-    if [ "$sharedLibs" = true ]; then
-      cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-mips.cmake -DS2N_NO_PQ=ON ../
-      make install DESTDIR=./shared_install_dir
-      chmod 0777 ./shared_install_dir
-    else
-      cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-mips.cmake -DS2N_NO_PQ=ON ../
-    fi
+    cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-mips.cmake -DS2N_NO_PQ=ON ../
     cmake --build . --target aws-iot-device-client
     cmake --build . --target test-aws-iot-device-client
     exit $?
@@ -189,25 +163,63 @@ case $compileMode in
     if [ "$stMode" = true ]; then
       # Set CMake flags for ST mode
       cmake -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-aarch64.cmake -DBUILD_SDK=ON -DEXCLUDE_JOBS=ON -DEXCLUDE_DD=ON -DEXCLUDE_FP=ON -DDISABLE_MQTT=ON -DEXCLUDE_SENSOR_PUBLISH=ON ../
-    elif [ "$sharedLibs" = true ]; then
-      cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-aarch64.cmake -DBUILD_SDK=ON ../
-      make install DESTDIR=./shared_install_dir
-      chmod 0777 ./shared_install_dir
     else
       cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-aarch64.cmake ../
     fi
     cmake --build . --target aws-iot-device-client
     cmake --build . --target test-aws-iot-device-client
     exit $?
     ;;
+    #################################
+    powerpc64_cross_mode)
+    apt-get update
+    apt-get install --assume-yes software-properties-common
+    apt-get install --assume-yes build-essential
+    apt-get install --assume-yes gcc-powerpc-linux-gnu
+    apt-get install --assume-yes g++-powerpc-linux-gnu
+    apt-get install --assume-yes gdb-multiarch
+    wget --ca-certificate=/etc/ssl/certs/ca-certificates.crt https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+    tar -xvzf openssl-${OPENSSL_VERSION}.tar.gz
+    export INSTALL_DIR=/usr/lib/aarch64-linux-gnu
+    cd openssl-${OPENSSL_VERSION}
+    ./Configure linux-aarch64 shared \
+      --prefix=$INSTALL_DIR --openssldir=$INSTALL_DIR/openssl \
+      --cross-compile-prefix=/usr/bin/powerpc-linux-gnu-
+    make depend
+    make -j 4
+    make install
+    cd ..
+    cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-ppc64.cmake ../
+    cmake --build . --target aws-iot-device-client
+    cmake --build . --target test-aws-iot-device-client
+    exit $?
+    ;;
+    #################################
+    powerpc64le_cross_mode)
+    apt-get update
+    apt-get install --assume-yes software-properties-common
+    apt-get install --assume-yes build-essential
+    apt-get install --assume-yes gcc-powerpc64le-linux-gnu
+    apt-get install --assume-yes g++-powerpc64le-linux-gnu
+    apt-get install --assume-yes gdb-multiarch
+    wget --ca-certificate=/etc/ssl/certs/ca-certificates.crt https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+    tar -xvzf openssl-${OPENSSL_VERSION}.tar.gz
+    export INSTALL_DIR=/usr/lib/aarch64-linux-gnu
+    cd openssl-${OPENSSL_VERSION}
+    ./Configure linux-aarch64 shared \
+      --prefix=$INSTALL_DIR --openssldir=$INSTALL_DIR/openssl \
+      --cross-compile-prefix=/usr/bin/powerpc64le-linux-gnu-
+    make depend
+    make -j 4
+    make install
+    cd ..
+    cmake -DCMAKE_TOOLCHAIN_FILE=../cmake-toolchain/Toolchain-ppc64.cmake ../
+    cmake --build . --target aws-iot-device-client
+    cmake --build . --target test-aws-iot-device-client
+    exit $?
+    ;;
     *)
-    if [ "$sharedLibs" = false ]; then
       cmake ../ -DBUILD_SDK=OFF -DBUILD_TEST_DEPS=OFF -DLINK_DL=ON
-    else
-      cmake ../ -DBUILD_SHARED_LIBS=ON -DBUILD_SDK=ON -DLINK_DL=ON
-      make install DESTDIR=./shared_install_dir
-      chmod 0777 ./shared_install_dir
-    fi
     ;;
 esac
 

diff --git a/.github/docker-images/Dockerfile b/.github/docker-images/Dockerfile
@@ -0,0 +1,31 @@
+# This is a multi-stage Dockerfile used for building minimum sized images of Device Client from your current directory.
+ARG OS
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE} AS deploy
+
+###############################################################################
+# Copy and build device client
+###############################################################################
+COPY . /root/aws-iot-device-client
+RUN mkdir -p /root/aws-iot-device-client/build \
+    && cd /root/aws-iot-device-client/build \
+    && cmake -DCMAKE_BUILD_TYPE=Release .. \
+    && cmake --build . --target aws-iot-device-client
+
+ENTRYPOINT ["/root/aws-iot-device-client/build/aws-iot-device-client"]
+
+FROM ${OS} AS minimum_size
+
+COPY --from=deploy /root/aws-iot-device-client/build/aws-iot-device-client ./aws-iot-device-client
+COPY --from=deploy /root/aws-iot-device-client/sample-job-handlers /root/.aws-iot-device-client/jobs
+
+COPY ./.github/docker-images/oss-compliance /root/oss-compliance
+RUN HOME_DIR=/root \
+    && chmod +x ${HOME_DIR}/oss-compliance/generate-oss-compliance.sh \
+    && chmod +x ${HOME_DIR}/oss-compliance/test/test-oss-compliance.sh \
+    && bash ${HOME_DIR}/oss-compliance/generate-oss-compliance.sh ${HOME_DIR} \
+    && rm -rf ${HOME_DIR}/oss-compliance*
+
+RUN chmod 700 /root/.aws-iot-device-client/jobs/*.sh
+
+ENTRYPOINT ["/aws-iot-device-client"]
diff --git a/.github/docker-images/README.md b/.github/docker-images/README.md
@@ -0,0 +1,23 @@
+# Docker Images
+
+The AWS IoT Device Client provides several Docker images to simplify testing and development.
+
+## Base Images
+
+The base images are built from the Docker files located in the [base-images](./base-images) directory. These images contain
+the dependencies of the Device Client only and are intended to be used to build & run the Device Client. Pre-built base
+images are publicly available through AWS ECR in the [aws-iot-device-client-base-images repository](https://gallery.ecr.aws/aws-iot-device-client/aws-iot-device-client-base-images).
+
+## AWS IoT Device Client Images
+
+Docker images containing Device Client binaries are also available through ECR. These images are built using this [Dockerfile](./Dockerfile)
+and are multi-stage builds that are minimum sized. Images are built from both the pre-release and main branches. The pre-release
+images are available for testing and experimenting with features which are still in development from the [pre-release repository](https://gallery.ecr.aws/aws-iot-device-client/pre-release).
+Images built from the main branch are available from the [aws-iot-device-client repository](https://gallery.ecr.aws/aws-iot-device-client/aws-iot-device-client).
+
+## Integration Test Images
+
+Before releasing new features on the main branch end-to-end tests are performed in the pre-release branch. As part of this
+process the Docker files located in the [integration-tests](./integration-tests) directory are used to build and run the
+end-to-end tests. In order to make it easy to recreate any test failures locally, these images are pushed to the [integration-tests repository](https://gallery.ecr.aws/aws-iot-device-client/integration-tests).
+These images contain both the pre-release version of the Device Client and the integration test binary.
diff --git a/.github/docker-images/amazonlinux/Dockerfile → ...ages/device-client/amazonlinux/Dockerfile b/.github/docker-images/amazonlinux/Dockerfile → ...ages/device-client/amazonlinux/Dockerfile
@@ -1,6 +1,6 @@
-FROM amazonlinux:latest
+FROM amazonlinux:2.0.20230307.0 as base
 
-ARG OPENSSL_VERSION=1.1.1n
+ARG OPENSSL_VERSION=3.0.8
 
 ###############################################################################
 # Install prereqs
@@ -11,37 +11,20 @@ RUN yum -y update \
     bzip2 \
     git \
     wget \
-    curl \
+    perl \
     sudo \
     make \
     gcc \
     gcc-c++ \
     softhsm \
+    zlib-devel \
+    perl-IPC-Cmd \
+    perl-Pod-Html \
     && yum clean all \
     && rm -rf /var/cache/yum
 
 ###############################################################################
-# Install cppcheck
-# cppcheck package is not available from amazonlinux default repositories
-# Install and enable epel repository.
-# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/add-repositories.html
-###############################################################################
-RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-RUN yum --enablerepo=epel install -y cppcheck
-
-###############################################################################
-# Install pre-built CMake
-###############################################################################
-WORKDIR /tmp
-RUN curl -sSL https://github.com/Kitware/CMake/releases/download/v3.10.0/cmake-3.10.0.tar.gz -o cmake-3.10.0.tar.gz \
-    && tar -zxvf cmake-3.10.0.tar.gz \
-    && cd cmake-3.10.0 \
-    && ./bootstrap \
-    && make -j 4 \
-    && make install
-
-###############################################################################
-# Install OpenSSL 1.1.1
+# Install OpenSSL 3.0.8
 ###############################################################################
 WORKDIR /tmp
 RUN wget https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
@@ -51,19 +34,29 @@ RUN wget https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
     && make \
     && sudo make install
 
+# Install pre-built CMake
+###############################################################################
+WORKDIR /tmp
+RUN curl -sSL https://github.com/Kitware/CMake/releases/download/v3.24.4/cmake-3.24.4.tar.gz -o cmake-3.24.4.tar.gz \
+        && tar -zxvf cmake-3.24.4.tar.gz \
+        && cd cmake-3.24.4 \
+        && ./bootstrap \
+        && make -j 4 \
+        && make install
+
 ###############################################################################
 # Clone and build Google Test
 ###############################################################################
 WORKDIR /tmp
-RUN curl -sSL https://github.com/google/googletest/archive/release-1.10.0.tar.gz -o release-1.10.0.tar.gz \
-    && tar xf release-1.10.0.tar.gz \
-    && cd googletest-release-1.10.0 \
+RUN curl -sSL https://github.com/google/googletest/archive/release-1.12.0.tar.gz -o release-1.12.0.tar.gz \
+    && tar xf release-1.12.0.tar.gz \
+    && cd googletest-release-1.12.0 \
     && cmake -DBUILD_SHARED_LIBS=ON . \
     && make \
     && cp -a googletest/include/gtest /usr/include/ \
     && cp -a googlemock/include/gmock /usr/include/ \
     && cp -a lib/* /usr/lib64/ \
-    && rm -f /tmp/release-1.10.0.tar.gz
+    && rm -f /tmp/release-1.11.0.tar.gz
 
 ###############################################################################
 # Clone and build valgrind
@@ -76,6 +69,18 @@ RUN wget https://sourceware.org/pub/valgrind/valgrind-3.19.0.tar.bz2 \
     && make \
     && make install
 
+###############################################################################
+# Install cppcheck
+###############################################################################
+WORKDIR /tmp
+RUN wget https://github.com/danmar/cppcheck/archive/2.9.tar.gz \
+    && tar zxvf 2.9.tar.gz \
+    && cd cppcheck-2.9 \
+    && mkdir build \
+    && cd build \
+    && cmake .. \
+    && make install
+
 # ###############################################################################
 # # Install Aws Iot Device Sdk Cpp v2
 # ###############################################################################
@@ -84,14 +89,34 @@ RUN mkdir sdk-cpp-workspace \
     && cd sdk-cpp-workspace \
     && git clone https://github.com/aws/aws-iot-device-sdk-cpp-v2.git \
     && cd aws-iot-device-sdk-cpp-v2 \
-    && git checkout ac3ba3774b031dde1b988e698880d6064d53b9d9 \
+    && git checkout 74c8b683ebe5b1cbf484f6acaa281f56aaa63948 \
     && git submodule update --init --recursive \
     && cd .. \
     && mkdir aws-iot-device-sdk-cpp-v2-build \
     && cd aws-iot-device-sdk-cpp-v2-build \
     && cmake -DCMAKE_INSTALL_PREFIX="/usr" -DUSE_OPENSSL=ON -DBUILD_DEPS=ON ../aws-iot-device-sdk-cpp-v2 \
     && cmake --build . --target install
 
-ADD entry-script.sh /home/entry-script
+COPY ./.github/docker-images/oss-compliance /root/oss-compliance
+RUN HOME_DIR=/root \
+    && chmod +x ${HOME_DIR}/oss-compliance/generate-oss-compliance.sh \
+    && chmod +x ${HOME_DIR}/oss-compliance/test/test-oss-compliance.sh \
+    && bash ${HOME_DIR}/oss-compliance/generate-oss-compliance.sh ${HOME_DIR} \
+    && rm -rf ${HOME_DIR}/oss-compliance*
+
+ADD .github/docker-images/entry-script.sh /home/entry-script
 RUN chmod a+x /home/entry-script
 ENTRYPOINT ["/home/entry-script"]
+
+FROM base AS deploy
+
+###############################################################################
+# Copy and build device client
+###############################################################################
+COPY . /root/aws-iot-device-client
+RUN mkdir -p /root/aws-iot-device-client/build \
+    && cd /root/aws-iot-device-client/build \
+    && cmake .. \
+    && cmake --build . --target aws-iot-device-client
+
+ENTRYPOINT ["/root/aws-iot-device-client/build/aws-iot-device-client"]