From a3c342e14eaf7b43fd9ba9e1a2dc046dfc1388ac Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 10:15:57 -0500 Subject: [PATCH 01/12] MIGRATIONS-1296 - Fix mend security issues - part 1 Signed-off-by: Omar Khasawneh --- .../captureKafkaOffloader/build.gradle | 2 +- TrafficCapture/testUtilities/build.gradle | 7 ++++--- TrafficCapture/trafficReplayer/build.gradle | 16 ++++++++-------- test/requirements.txt | 2 +- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index e8de2244b..b48df1ab8 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation project(':captureOffloader') implementation 'org.projectlombok:lombok:1.18.26' implementation 'com.google.protobuf:protobuf-java:3.22.2' - implementation 'org.apache.kafka:kafka-clients:3.5.0' + implementation 'org.apache.kafka:kafka-clients:3.5.1' implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.7' implementation 'org.slf4j:slf4j-api:2.0.7' testImplementation project(':captureProtobufs') diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle index 645a13750..52387414d 100644 --- a/TrafficCapture/testUtilities/build.gradle +++ b/TrafficCapture/testUtilities/build.gradle @@ -29,7 +29,7 @@ spotbugs { } checkstyle { - toolVersion = '10.9.3' + toolVersion = '10.12.1' configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml') System.setProperty('checkstyle.cache.file', String.format('%s/%s', buildDir, 'checkstyle.cachefile')) @@ -37,6 +37,7 @@ checkstyle { repositories { mavenCentral() + maven { url 'https://www.bouncycastle.org/repo/' } } dependencies { @@ -44,8 +45,8 @@ dependencies { implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' implementation group: 'io.netty', name: 'netty-all', version: '4.1.89.Final' implementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' - implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.68' - implementation group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version: '1.68' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.74' + implementation group: 'org.bouncycastle', name: 'bcpkix-jdk18on', version: '1.74' implementation group: 'org.projectlombok', name: 'lombok', version: '1.18.22' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 7583dc1f8..82d837699 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -19,7 +19,7 @@ buildscript { plugins { id 'org.opensearch.migrations.java-application-conventions' id "com.github.spotbugs" version "4.7.3" -// id 'checkstyle' + id 'checkstyle' id 'org.owasp.dependencycheck' version '8.2.1' id "io.freefair.lombok" version "8.0.1" } @@ -28,12 +28,12 @@ spotbugs { includeFilter = new File(rootDir, 'config/spotbugs/spotbugs-include.xml') } -//checkstyle { -// toolVersion = '10.9.3' -// configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml') -// System.setProperty('checkstyle.cache.file', String.format('%s/%s', -// buildDir, 'checkstyle.cachefile')) -//} +checkstyle { + toolVersion = '10.12.1' + configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml') + System.setProperty('checkstyle.cache.file', String.format('%s/%s', + buildDir, 'checkstyle.cachefile')) +} repositories { mavenCentral() @@ -55,7 +55,7 @@ dependencies { implementation group: 'org.json', name: 'json', version: '20230227' implementation group: 'org.projectlombok', name: 'lombok', version: '1.18.22' - implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.0' + implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.1' implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' diff --git a/test/requirements.txt b/test/requirements.txt index 3a293bea2..e23befe0b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -certifi==2023.5.7 +certifi==2023.7.22 charset-normalizer==3.1.0 idna==3.4 iniconfig==2.0.0 From 235cd49df5ee4c6f45f9890d7a8a6642e42a5e39 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 10:28:41 -0500 Subject: [PATCH 02/12] Update checkstyle version to latest Signed-off-by: Omar Khasawneh --- TrafficCapture/testUtilities/build.gradle | 2 +- TrafficCapture/trafficReplayer/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle index 52387414d..7821ba68c 100644 --- a/TrafficCapture/testUtilities/build.gradle +++ b/TrafficCapture/testUtilities/build.gradle @@ -29,7 +29,7 @@ spotbugs { } checkstyle { - toolVersion = '10.12.1' + toolVersion = '10.12.3' configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml') System.setProperty('checkstyle.cache.file', String.format('%s/%s', buildDir, 'checkstyle.cachefile')) diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 82d837699..fb89b7c1d 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -29,7 +29,7 @@ spotbugs { } checkstyle { - toolVersion = '10.12.1' + toolVersion = '10.12.3' configFile = new File(rootDir, 'config/checkstyle/checkstyle.xml') System.setProperty('checkstyle.cache.file', String.format('%s/%s', buildDir, 'checkstyle.cachefile')) From 0f9794503c8de483a288a992f5e879afdb7048ff Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 10:44:53 -0500 Subject: [PATCH 03/12] fix aws-msk-iam-auth related mend issue Signed-off-by: Omar Khasawneh --- TrafficCapture/captureKafkaOffloader/build.gradle | 2 +- TrafficCapture/trafficReplayer/build.gradle | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/captureKafkaOffloader/build.gradle b/TrafficCapture/captureKafkaOffloader/build.gradle index b48df1ab8..122c4fb2b 100644 --- a/TrafficCapture/captureKafkaOffloader/build.gradle +++ b/TrafficCapture/captureKafkaOffloader/build.gradle @@ -14,7 +14,7 @@ dependencies { implementation 'org.projectlombok:lombok:1.18.26' implementation 'com.google.protobuf:protobuf-java:3.22.2' implementation 'org.apache.kafka:kafka-clients:3.5.1' - implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.7' + implementation 'software.amazon.msk:aws-msk-iam-auth:1.1.9' implementation 'org.slf4j:slf4j-api:2.0.7' testImplementation project(':captureProtobufs') testImplementation 'org.mockito:mockito-core:4.6.1' diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index fb89b7c1d..30c76a618 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -60,7 +60,7 @@ dependencies { implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.20.0' implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j2-impl', version: '2.20.0' implementation group: 'org.slf4j', name: 'slf4j-api', version: '2.0.7' - implementation group: 'software.amazon.msk', name: 'aws-msk-iam-auth', version: '1.1.7' + implementation group: 'software.amazon.msk', name: 'aws-msk-iam-auth', version: '1.1.9' testImplementation project(':testUtilities') testImplementation group: 'org.apache.httpcomponents.client5', name: 'httpclient5', version: '5.2.1' From cf7711a46791c462cd0056c437b72c5fe0656c79 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 12:05:52 -0500 Subject: [PATCH 04/12] actually update spotbugs Signed-off-by: Omar Khasawneh --- TrafficCapture/testUtilities/build.gradle | 2 ++ TrafficCapture/trafficReplayer/build.gradle | 2 ++ 2 files changed, 4 insertions(+) diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle index 7821ba68c..01faf1d62 100644 --- a/TrafficCapture/testUtilities/build.gradle +++ b/TrafficCapture/testUtilities/build.gradle @@ -41,6 +41,8 @@ repositories { } dependencies { + spotbugs 'com.github.spotbugs:spotbugs:4.7.3' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.0' implementation group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' implementation group: 'io.netty', name: 'netty-all', version: '4.1.89.Final' diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 30c76a618..e15178a06 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -40,6 +40,8 @@ repositories { } dependencies { + spotbugs 'com.github.spotbugs:spotbugs:4.7.3' + implementation project(':captureProtobufs') implementation 'software.amazon.awssdk:sdk-core:2.20.102' From 293bb8d7424d323100c8f97484434c3bb8e9b524 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 12:30:03 -0500 Subject: [PATCH 05/12] apache related mend fixes Signed-off-by: Omar Khasawneh --- .../opensearch/migrations/common/CommonUtils.groovy | 2 +- TrafficCapture/trafficReplayer/build.gradle | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy b/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy index 1864adaa4..6a932b925 100644 --- a/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy +++ b/TrafficCapture/buildSrc/src/main/groovy/org/opensearch/migrations/common/CommonUtils.groovy @@ -75,7 +75,7 @@ class CommonConfigurations { static void applyCommonConfigurations(Project project) { project.configurations.all { resolutionStrategy.dependencySubstitution { - substitute module('org.apache.xmlgraphics:batik-codec') using module('org.apache.xmlgraphics:batik-all:1.15') + substitute module('org.apache.xmlgraphics:batik-codec') using module('org.apache.xmlgraphics:batik-all:1.17') } } } diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index e15178a06..8cac314c5 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -70,6 +70,17 @@ dependencies { testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1' } +configurations.all { + resolutionStrategy.eachDependency { DependencyResolveDetails details -> + if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-text') { + details.useVersion '1.10.0' + } + if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') { + details.useVersion '6.7.0' + } + } +} + application { mainClass = 'org.opensearch.migrations.replay.TrafficReplayer' } From 3dc7b38f3208abd527ff14066845e87b653436d5 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 15:53:08 -0500 Subject: [PATCH 06/12] upgrading aws-cdk-lib version in hopes of fixing mend issue Signed-off-by: Omar Khasawneh --- deployment/cdk/opensearch-service-migration/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index 089173145..8f2fbfd26 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-kafka": "^3.354.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.84.0", + "aws-cdk-lib": "2.88.0", "aws-lambda": "^1.0.7", "constructs": "^10.0.0", "source-map-support": "^0.5.21" From 076aa9c8697d7ab6436ea88d0b412c514e761662 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 16:00:47 -0500 Subject: [PATCH 07/12] upgrading aws-cdk-lib version in hopes of fixing mend issue Signed-off-by: Omar Khasawneh --- deployment/cdk/opensearch-service-migration/package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index b2d198550..8b6958c7a 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -11,7 +11,7 @@ "@aws-sdk/client-kafka": "^3.354.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.84.0", + "aws-cdk-lib": "2.88.0", "aws-lambda": "^1.0.7", "constructs": "^10.0.0", "source-map-support": "^0.5.21" From 70ec83786e51a77774a7a52c05eee5267d1ed35a Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 16:08:17 -0500 Subject: [PATCH 08/12] reverting aws-cdk-lib upgrade because it didn't fix mend issue Signed-off-by: Omar Khasawneh --- deployment/cdk/opensearch-service-migration/package-lock.json | 2 +- deployment/cdk/opensearch-service-migration/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/package-lock.json b/deployment/cdk/opensearch-service-migration/package-lock.json index 8b6958c7a..b2d198550 100644 --- a/deployment/cdk/opensearch-service-migration/package-lock.json +++ b/deployment/cdk/opensearch-service-migration/package-lock.json @@ -11,7 +11,7 @@ "@aws-sdk/client-kafka": "^3.354.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.88.0", + "aws-cdk-lib": "2.84.0", "aws-lambda": "^1.0.7", "constructs": "^10.0.0", "source-map-support": "^0.5.21" diff --git a/deployment/cdk/opensearch-service-migration/package.json b/deployment/cdk/opensearch-service-migration/package.json index 8f2fbfd26..089173145 100644 --- a/deployment/cdk/opensearch-service-migration/package.json +++ b/deployment/cdk/opensearch-service-migration/package.json @@ -23,7 +23,7 @@ "@aws-sdk/client-kafka": "^3.354.0", "@aws-sdk/client-lambda": "^3.359.0", "@types/aws-lambda": "^8.10.117", - "aws-cdk-lib": "2.88.0", + "aws-cdk-lib": "2.84.0", "aws-lambda": "^1.0.7", "constructs": "^10.0.0", "source-map-support": "^0.5.21" From f34a25edd694b0c719846ac7092a4b2b6ca907e8 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 16:28:36 -0500 Subject: [PATCH 09/12] removing unnecessary repo in gradle file Signed-off-by: Omar Khasawneh --- TrafficCapture/testUtilities/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/TrafficCapture/testUtilities/build.gradle b/TrafficCapture/testUtilities/build.gradle index 01faf1d62..39ca32efa 100644 --- a/TrafficCapture/testUtilities/build.gradle +++ b/TrafficCapture/testUtilities/build.gradle @@ -37,7 +37,6 @@ checkstyle { repositories { mavenCentral() - maven { url 'https://www.bouncycastle.org/repo/' } } dependencies { From 423ae1189ea1f45030f3dce891be44370b9943dc Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Fri, 1 Sep 2023 16:53:34 -0500 Subject: [PATCH 10/12] specify a minimum version for some transitive dependencies per @lewijacn recommendation Signed-off-by: Omar Khasawneh --- TrafficCapture/trafficReplayer/build.gradle | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 8cac314c5..8dfafb296 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -73,10 +73,24 @@ dependencies { configurations.all { resolutionStrategy.eachDependency { DependencyResolveDetails details -> if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-text') { - details.useVersion '1.10.0' + def requestedVersion = details.requested.version + def targetVersion = '1.10.0' + + if (requestedVersion > targetVersion) { + details.useVersion requestedVersion + } else { + details.useVersion targetVersion + } } if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') { - details.useVersion '6.7.0' + def requestedVersion = details.requested.version + def targetVersion = '6.7.0' + + if (requestedVersion > targetVersion) { + details.useVersion requestedVersion + } else { + details.useVersion targetVersion + } } } } From dc9ded405d36a35a071e19e299e96cce44307068 Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Mon, 4 Sep 2023 22:12:42 -0500 Subject: [PATCH 11/12] add string parsing to compare version manually and accurately choose the right version of the dependency Signed-off-by: Omar Khasawneh --- TrafficCapture/trafficReplayer/build.gradle | 26 +++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index 8dfafb296..dce215181 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -70,25 +70,31 @@ dependencies { testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1' } +def compareVersions(String v1, String v2) { + def partsOfv1 = v1.split('\\.')*.toInteger() + def partsOfv2 = v2.split('\\.')*.toInteger() + + for (int i = 0; i < 3; i++) { + if (partsOfv1[i] < partsOfv2[i]) { + return -1 + } else if (partsOfv1[i] > partsOfv2[i]) { + return 1 + } + } + return 0 +} + configurations.all { resolutionStrategy.eachDependency { DependencyResolveDetails details -> if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-text') { - def requestedVersion = details.requested.version def targetVersion = '1.10.0' - - if (requestedVersion > targetVersion) { - details.useVersion requestedVersion - } else { + if (compareVersions(details.requested.version, targetVersion) == -1) { details.useVersion targetVersion } } if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') { - def requestedVersion = details.requested.version def targetVersion = '6.7.0' - - if (requestedVersion > targetVersion) { - details.useVersion requestedVersion - } else { + if (compareVersions(details.requested.version, targetVersion) == -1) { details.useVersion targetVersion } } From a875b380f5ccb410eca2752a4e5b36be0ef1fe5a Mon Sep 17 00:00:00 2001 From: Omar Khasawneh Date: Tue, 5 Sep 2023 13:14:46 -0500 Subject: [PATCH 12/12] updated version comparison function Signed-off-by: Omar Khasawneh --- TrafficCapture/trafficReplayer/build.gradle | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/TrafficCapture/trafficReplayer/build.gradle b/TrafficCapture/trafficReplayer/build.gradle index dce215181..5f9822e3a 100644 --- a/TrafficCapture/trafficReplayer/build.gradle +++ b/TrafficCapture/trafficReplayer/build.gradle @@ -70,31 +70,30 @@ dependencies { testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1' } -def compareVersions(String v1, String v2) { - def partsOfv1 = v1.split('\\.')*.toInteger() - def partsOfv2 = v2.split('\\.')*.toInteger() +def isRequestedVersionOlder(String requested, String target) { + def requestedParts = requested.split('\\.')*.toInteger() + def targetParts = target.split('\\.')*.toInteger() for (int i = 0; i < 3; i++) { - if (partsOfv1[i] < partsOfv2[i]) { - return -1 - } else if (partsOfv1[i] > partsOfv2[i]) { - return 1 + if (requestedParts[i] < targetParts[i]) { + return false + } else if (requestedParts[i] > targetParts[i]) { + return true } } - return 0 } configurations.all { resolutionStrategy.eachDependency { DependencyResolveDetails details -> if (details.requested.group == 'org.apache.commons' && details.requested.name == 'commons-text') { def targetVersion = '1.10.0' - if (compareVersions(details.requested.version, targetVersion) == -1) { + if (isRequestedVersionOlder(details.requested.version, targetVersion) != true) { details.useVersion targetVersion } } if (details.requested.group == 'org.apache.bcel' && details.requested.name == 'bcel') { def targetVersion = '6.7.0' - if (compareVersions(details.requested.version, targetVersion) == -1) { + if (isRequestedVersionOlder(details.requested.version, targetVersion) != true) { details.useVersion targetVersion } }