From cb816417d30e9676ae68dae9beb88077bd85aa14 Mon Sep 17 00:00:00 2001 From: Harsh Gandhi <63674702+HarshGandhi-AWS@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:57:20 -0800 Subject: [PATCH 1/3] Updated the OpenSSL, CMake and Device SDK version and added dependent libs for local proxy integration tests (#441) * updated the OpenSSL version and added dependent lib * updated SDK version of integ tests * updated SDK tag * updated CMake version for installing SDK * removed libssl-dev which was causing CMake built failure * typo * cmake prereq * installing openssl * installing openssl --- .../device-client/amazonlinux/Dockerfile | 2 + .../base-images/device-client/ubi8/Dockerfile | 2 + .../integration-tests/amazonlinux/Dockerfile | 4 +- .../integration-tests/ubi8/Dockerfile | 4 +- .../integration-tests/ubuntu/Dockerfile | 2 +- .../integration-tests/amazonlinux/Dockerfile | 20 +++++++ .../integration-tests/ubi8/Dockerfile | 21 ++++++++ .../integration-tests/ubuntu/Dockerfile | 52 ++++++++++++++++++- integration-tests/CMakeLists.txt | 2 +- integration-tests/CMakeLists.txt.gtest | 2 +- integration-tests/CMakeLists.txt.libawscpp | 4 +- 11 files changed, 107 insertions(+), 8 deletions(-) diff --git a/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile b/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile index 6a90821b7..c991087ef 100644 --- a/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile +++ b/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile @@ -18,6 +18,8 @@ RUN yum -y update \ gcc-c++ \ softhsm \ zlib-devel \ + perl-IPC-Cmd \ + perl-Pod-Html \ && yum clean all \ && rm -rf /var/cache/yum diff --git a/.github/docker-images/base-images/device-client/ubi8/Dockerfile b/.github/docker-images/base-images/device-client/ubi8/Dockerfile index 97a69b9f7..a91e940b9 100644 --- a/.github/docker-images/base-images/device-client/ubi8/Dockerfile +++ b/.github/docker-images/base-images/device-client/ubi8/Dockerfile @@ -18,6 +18,8 @@ RUN yum -y update \ make \ gcc \ gcc-c++ \ + perl-IPC-Cmd \ + perl-Pod-Html \ && yum clean all \ && rm -rf /var/cache/yum diff --git a/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile b/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile index b420072bc..62ab3151b 100644 --- a/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile +++ b/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile @@ -1,6 +1,6 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2.0.20230307.0 -ARG OPENSSL_VERSION=1.1.1n +ARG OPENSSL_VERSION=3.0.8 RUN yum -y update && yum -y install \ autoconf \ @@ -14,6 +14,8 @@ RUN yum -y update && yum -y install \ perl \ softhsm \ tar \ + perl-IPC-Cmd \ + perl-Pod-Html \ wget diff --git a/.github/docker-images/base-images/integration-tests/ubi8/Dockerfile b/.github/docker-images/base-images/integration-tests/ubi8/Dockerfile index c3808f734..a02ac3098 100644 --- a/.github/docker-images/base-images/integration-tests/ubi8/Dockerfile +++ b/.github/docker-images/base-images/integration-tests/ubi8/Dockerfile @@ -1,6 +1,6 @@ FROM registry.access.redhat.com/ubi8/ubi -ARG OPENSSL_VERSION=1.1.1n +ARG OPENSSL_VERSION=3.0.8 RUN yum -y update && yum -y install \ autoconf \ @@ -13,6 +13,8 @@ RUN yum -y update && yum -y install \ make \ perl \ tar \ + perl-IPC-Cmd \ + perl-Pod-Html \ wget diff --git a/.github/docker-images/base-images/integration-tests/ubuntu/Dockerfile b/.github/docker-images/base-images/integration-tests/ubuntu/Dockerfile index a9a8bf2a2..edcf5c0f6 100644 --- a/.github/docker-images/base-images/integration-tests/ubuntu/Dockerfile +++ b/.github/docker-images/base-images/integration-tests/ubuntu/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:18.04 -ARG OPENSSL_VERSION=1.1.1n +ARG OPENSSL_VERSION=3.0.8 RUN apt-get update && apt-get install -y \ wget \ diff --git a/.github/docker-images/integration-tests/amazonlinux/Dockerfile b/.github/docker-images/integration-tests/amazonlinux/Dockerfile index c8e606a73..7cec11895 100644 --- a/.github/docker-images/integration-tests/amazonlinux/Dockerfile +++ b/.github/docker-images/integration-tests/amazonlinux/Dockerfile @@ -2,6 +2,21 @@ ARG BASE_IMAGE ARG PRE_RELEASE_IMAGE FROM ${BASE_IMAGE} AS build_integration_tests +############################################################################### +# Install pre-built CMake +############################################################################### +WORKDIR /tmp +RUN curl -sSL https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0.tar.gz -o cmake-3.28.0.tar.gz \ + && tar -zxvf cmake-3.28.0.tar.gz \ + && cd cmake-3.28.0 \ + && ./bootstrap \ + && make -j 2 \ + && make install + +############################################################################### +# Install Prerequisites +############################################################################### + RUN yum -y update && yum -y install \ autoconf \ libtool \ @@ -14,10 +29,15 @@ RUN mkdir -p /root/integration-tests/build \ && cmake .. -DCMAKE_BUILD_TYPE=DEBUG \ && cmake --build . --target aws-iot-device-client-integration-tests +############################################################################### # Actual docker image +############################################################################### + FROM ${PRE_RELEASE_IMAGE} AS runner +############################################################################### # Install openssl for libssl dependency. +############################################################################### RUN yum update -y && yum install -y libatomic libcurl-devel openssl-devel libuuid-devel wget && \ rm -rf /var/cache/yum && \ yum clean all diff --git a/.github/docker-images/integration-tests/ubi8/Dockerfile b/.github/docker-images/integration-tests/ubi8/Dockerfile index 65a5df602..f76ae7cc4 100644 --- a/.github/docker-images/integration-tests/ubi8/Dockerfile +++ b/.github/docker-images/integration-tests/ubi8/Dockerfile @@ -2,6 +2,21 @@ ARG BASE_IMAGE ARG PRE_RELEASE_IMAGE FROM ${BASE_IMAGE} AS build_integration_tests +############################################################################### +# Install pre-built CMake +############################################################################### +WORKDIR /tmp +RUN curl -sSL https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0.tar.gz -o cmake-3.28.0.tar.gz \ + && tar -zxvf cmake-3.28.0.tar.gz \ + && cd cmake-3.28.0 \ + && ./bootstrap \ + && make -j 2 \ + && make install + +############################################################################### +# Install Prerequisites +############################################################################### + RUN #yum update -y && yum install -y libcurl-devel openssl-devel libuuid-devel make gcc gcc-c++ git zlib-devel COPY ./integration-tests /root/integration-tests @@ -10,10 +25,16 @@ RUN mkdir -p /root/integration-tests/build \ && cmake .. -DCMAKE_BUILD_TYPE=DEBUG \ && cmake --build . --target aws-iot-device-client-integration-tests +############################################################################### # Actual docker image +############################################################################### + FROM ${PRE_RELEASE_IMAGE} AS runner +############################################################################### # Install openssl for libssl dependency. +############################################################################### + RUN yum update -y && yum install -y libatomic libcurl-devel openssl-devel libuuid-devel wget && \ rm -rf /var/cache/yum && \ yum clean all diff --git a/.github/docker-images/integration-tests/ubuntu/Dockerfile b/.github/docker-images/integration-tests/ubuntu/Dockerfile index b72fb3e40..83e37a0da 100644 --- a/.github/docker-images/integration-tests/ubuntu/Dockerfile +++ b/.github/docker-images/integration-tests/ubuntu/Dockerfile @@ -3,13 +3,58 @@ ARG BASE_IMAGE ARG PRE_RELEASE_IMAGE FROM ${OS} AS build_localproxy +ARG OPENSSL_VERSION=3.0.8 + +############################################################################### +# Install prereqs for CMake +############################################################################### +RUN apt-get update -qq \ + && apt-get -y install \ + git \ + clang-format \ + curl \ + build-essential \ + wget \ + libc6-dbg \ + softhsm \ + zlib1g-dev \ + && apt-get clean + + +############################################################################### +# Install OpenSSL 3.0.8 +############################################################################### +WORKDIR /tmp +RUN wget https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ + && tar -zxvf openssl-${OPENSSL_VERSION}.tar.gz \ + && cd openssl-${OPENSSL_VERSION} \ + && ./config \ + && make \ + && make install \ + && ldconfig + +############################################################################### +# Install pre-built CMake +############################################################################### +WORKDIR /tmp +RUN curl -sSL https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0.tar.gz -o cmake-3.28.0.tar.gz \ + && tar -zxvf cmake-3.28.0.tar.gz \ + && cd cmake-3.28.0 \ + && ./bootstrap \ + && make -j 2 \ + && make install + +############################################################################### # Install Prerequisites +############################################################################### RUN apt update && apt upgrade -y && \ apt install -y git libboost-all-dev autoconf automake \ wget libtool curl make g++ unzip cmake libssl-dev +############################################################################### # Install Dependencies +############################################################################### RUN mkdir /home/dependencies WORKDIR /home/dependencies @@ -74,7 +119,10 @@ RUN mkdir -p /root/integration-tests/build \ && cmake .. -DCMAKE_BUILD_TYPE=DEBUG \ && cmake --build . --target aws-iot-device-client-integration-tests +############################################################################### # Actual docker image +############################################################################### + FROM ${PRE_RELEASE_IMAGE} AS runner RUN apt-get update && apt-get install -y libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev libpulse-dev wget ssh ca-certificates wget apt-utils @@ -82,8 +130,10 @@ RUN mkdir -p /home/aws-iot-securetunneling-localproxy/certs && \ cd /home/aws-iot-securetunneling-localproxy/certs && \ wget https://www.amazontrust.com/repository/AmazonRootCA1.pem && \ openssl rehash ./ +############################################################################### +# Copy the binaries from builder stages. +############################################################################### -# # Copy the binaries from builder stages. COPY --from=build_localproxy /home/aws-iot-securetunneling-localproxy/localproxy /localproxy COPY --from=build_integration_tests /root/integration-tests/build/aws-iot-device-client-integration-tests /aws-iot-device-client-integration-tests COPY --from=build_integration_tests /root/integration-tests/source/tunneling/test-tunnel.sh /test-tunnel.sh diff --git a/integration-tests/CMakeLists.txt b/integration-tests/CMakeLists.txt index ac9465df5..89bb3fd2d 100644 --- a/integration-tests/CMakeLists.txt +++ b/integration-tests/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.13) project(aws-iot-device-client-integration-tests) set(OPENSSL_USE_STATIC_LIBS TRUE) diff --git a/integration-tests/CMakeLists.txt.gtest b/integration-tests/CMakeLists.txt.gtest index becd6bd68..39a73ca47 100644 --- a/integration-tests/CMakeLists.txt.gtest +++ b/integration-tests/CMakeLists.txt.gtest @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.13) project(googletest-download NONE) diff --git a/integration-tests/CMakeLists.txt.libawscpp b/integration-tests/CMakeLists.txt.libawscpp index 30ef18de7..638eb5938 100644 --- a/integration-tests/CMakeLists.txt.libawscpp +++ b/integration-tests/CMakeLists.txt.libawscpp @@ -1,10 +1,10 @@ -cmake_minimum_required(VERSION 3.10) +cmake_minimum_required(VERSION 3.13) project(aws-sdk-cpp NONE) include(ExternalProject) ExternalProject_Add(aws-sdk-cpp GIT_REPOSITORY https://github.com/aws/aws-sdk-cpp.git - GIT_TAG 1.11.56 + GIT_TAG 1.11.219 LIST_SEPARATOR "|" CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=iot|iotsecuretunneling From 3441d1622b0f753919fcc3fb599f47834eb9cbca Mon Sep 17 00:00:00 2001 From: Harsh Gandhi Date: Wed, 13 Dec 2023 15:17:50 -0800 Subject: [PATCH 2/3] updated amazon linux OS version for CI tests --- .../base-images/device-client/amazonlinux/Dockerfile | 2 +- .../base-images/integration-tests/amazonlinux/Dockerfile | 2 +- .github/workflows/e2e-ci.yml | 4 ++-- .github/workflows/release-ci.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile b/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile index c991087ef..6905abab6 100644 --- a/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile +++ b/.github/docker-images/base-images/device-client/amazonlinux/Dockerfile @@ -1,4 +1,4 @@ -FROM amazonlinux:2.0.20230307.0 as base +FROM amazonlinux:2023.2.20231113.0 as base ARG OPENSSL_VERSION=3.0.8 diff --git a/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile b/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile index 60cec4093..abf3e7c4e 100644 --- a/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile +++ b/.github/docker-images/base-images/integration-tests/amazonlinux/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/amazonlinux/amazonlinux:2.0.20230307.0 +FROM public.ecr.aws/amazonlinux/amazonlinux:2023.2.20231113.0 ARG OPENSSL_VERSION=3.0.8 diff --git a/.github/workflows/e2e-ci.yml b/.github/workflows/e2e-ci.yml index ad9376558..896abd9e0 100644 --- a/.github/workflows/e2e-ci.yml +++ b/.github/workflows/e2e-ci.yml @@ -203,7 +203,7 @@ jobs: file: .github/docker-images/Dockerfile context: . build-args: | - OS=amazonlinux:2022.0.20221207.4 + OS=amazonlinux:2023.2.20231113.0 BASE_IMAGE=public.ecr.aws/${{ env.ECR_BASE_REPO }}:x86_64-amazonlinux-latest push: true tags: | @@ -241,7 +241,7 @@ jobs: file: .github/docker-images/Dockerfile context: . build-args: | - OS=amazonlinux:2022.0.20221207.4 + OS=amazonlinux:2023.2.20231113.0 BASE_IMAGE=public.ecr.aws/${{ env.ECR_BASE_REPO }}:aarch64-amazonlinux-latest push: true tags: | diff --git a/.github/workflows/release-ci.yml b/.github/workflows/release-ci.yml index 985a2782e..debe37343 100644 --- a/.github/workflows/release-ci.yml +++ b/.github/workflows/release-ci.yml @@ -214,7 +214,7 @@ jobs: file: .github/docker-images/Dockerfile context: . build-args: | - OS=amazonlinux:latest + OS=amazonlinux:2023.2.20231113.0 BASE_IMAGE=public.ecr.aws/${{ env.ECR_BASE_REPO }}:x86_64-amazonlinux-latest push: true tags: | @@ -256,7 +256,7 @@ jobs: file: .github/docker-images/Dockerfile context: . build-args: | - OS=amazonlinux:latest + OS=amazonlinux:2023.2.20231113.0 BASE_IMAGE=public.ecr.aws/${{ env.ECR_BASE_REPO }}:aarch64-amazonlinux-latest push: true tags: | From 760f153977d3315e852e6dcad5bae7b50da828d2 Mon Sep 17 00:00:00 2001 From: Harsh Gandhi Date: Wed, 13 Dec 2023 16:53:24 -0800 Subject: [PATCH 3/3] removed redundant code --- .../integration-tests/ubuntu/Dockerfile | 2 +- integration-tests/source/GTestMain.cpp | 13 ------------- .../tunneling/SecureTunnelingIntegrationTests.cpp | 6 ------ 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/docker-images/integration-tests/ubuntu/Dockerfile b/.github/docker-images/integration-tests/ubuntu/Dockerfile index 380e07b96..6974d3975 100644 --- a/.github/docker-images/integration-tests/ubuntu/Dockerfile +++ b/.github/docker-images/integration-tests/ubuntu/Dockerfile @@ -129,7 +129,7 @@ RUN apt-get update && apt-get install -y libcurl4-openssl-dev libssl-dev uuid-de RUN mkdir -p /home/aws-iot-securetunneling-localproxy/certs && \ cd /home/aws-iot-securetunneling-localproxy/certs && \ - wget https://www.amazontrust.com/repository/AmazonRootCA1.pem && \ + wget --no-check-certificate https://www.amazontrust.com/repository/AmazonRootCA1.pem && \ openssl rehash ./ ############################################################################### # Copy the binaries from builder stages. diff --git a/integration-tests/source/GTestMain.cpp b/integration-tests/source/GTestMain.cpp index c8875bc27..2325a2365 100644 --- a/integration-tests/source/GTestMain.cpp +++ b/integration-tests/source/GTestMain.cpp @@ -122,18 +122,6 @@ class GlobalEnvironment : public ::testing::Environment { public: ~GlobalEnvironment() override {} - // cppcheck-suppress unusedFunction - void SetUp() override - { - Aws::InitAPI(options); - { - Aws::Client::ClientConfiguration clientConfig; - clientConfig.region = REGION; - resourceHandler = - std::shared_ptr(new IntegrationTestResourceHandler(clientConfig)); - } - } - // cppcheck-suppress unusedFunction void SetUp() override { @@ -146,7 +134,6 @@ class GlobalEnvironment : public ::testing::Environment // cppcheck-suppress unusedFunction void TearDown() override { - resourceHandler->CleanUp(); if (CLEAN_UP) { printf("Clean up thingName: %s\n", THING_NAME.c_str()); diff --git a/integration-tests/source/tunneling/SecureTunnelingIntegrationTests.cpp b/integration-tests/source/tunneling/SecureTunnelingIntegrationTests.cpp index d3daa7920..6910fd392 100644 --- a/integration-tests/source/tunneling/SecureTunnelingIntegrationTests.cpp +++ b/integration-tests/source/tunneling/SecureTunnelingIntegrationTests.cpp @@ -33,12 +33,6 @@ class TestSecureTunnelingFeature : public ::testing::Test { if (!SKIP_ST) { - ClientConfiguration clientConfig; - resourceHandler = - unique_ptr(new IntegrationTestResourceHandler(clientConfig)); - Aws::IoTSecureTunneling::Model::OpenTunnelResult openTunnelResult = resourceHandler->OpenTunnel(THING_NAME); - tunnelId = openTunnelResult.GetTunnelId(); - sourceToken = openTunnelResult.GetSourceAccessToken(); Aws::IoTSecureTunneling::Model::OpenTunnelResult openTunnelResult = resourceHandler->OpenTunnel(THING_NAME); tunnelId = openTunnelResult.GetTunnelId();