From af09430d949331bc417495277b453071bd27bd28 Mon Sep 17 00:00:00 2001 From: Przemyslaw Czuj Date: Thu, 18 May 2023 23:34:15 +0200 Subject: [PATCH] JPERF-1106: Gather Apache2 logs --- CHANGELOG.md | 13 ++++ .../awsinfrastructure/FailSafeRunnable.kt | 26 +++++++ .../api/jira/DataCenterFormula.kt | 5 ++ .../tools/awsinfrastructure/api/jira/Jira.kt | 39 ++++++++--- .../awsinfrastructure/api/jira/StartedNode.kt | 5 +- .../loadbalancer/ApacheProxyLoadBalancer.kt | 14 +++- .../loadbalancer/DiagnosableLoadBalancer.kt | 8 +++ .../LoadBalancerMeasurementSource.kt | 23 +++++++ .../awsinfrastructure/FailSafeRunnableTest.kt | 67 +++++++++++++++++++ 9 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt create mode 100644 src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt create mode 100644 src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 445d8849..6ba72219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,19 @@ Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] [Unreleased]: https://github.com/atlassian/aws-infrastructure/compare/release-3.0.0...master +## Added +- Add `DiagnosableLoadBalancer`. +- Add ability to `gatherEvidence` from `ApacheProxyLoadBalancer` by making it `DiagnosableLoadBalancer`. +- Make it possible to specify `extraMeasurementSources` inside `Jira`, so that it's possible to `gatherResults` from any of Jira hosted integrations, e.g. load balancer or plugins. +- Let `Jira` produced by `DataCenterFormula` gather logs of `ApacheProxyLoadBalancer`. Resolve [JPERF-1106]. +- Make `StartedNode` a `MeasurementSource`. + +## Fixed +- Let every `MeasurementSource` inside `Jira` finish its gathering even if one of them fails. Fix [JPERF-1114]. + +[JPERF-1106]: https://ecosystem.atlassian.net/browse/JPERF-1106 +[JPERF-1114]: https://ecosystem.atlassian.net/browse/JPERF-1114 + ## [3.0.0] - 2023-05-25 [3.0.0]: https://github.com/atlassian/aws-infrastructure/compare/release-2.29.0...release-3.0.0 diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt new file mode 100644 index 00000000..e4eeea06 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnable.kt @@ -0,0 +1,26 @@ +package com.atlassian.performance.tools.awsinfrastructure + +internal class FailSafeRunnable( + private val delegates: Iterable +) : Runnable { + override fun run() { + val exceptions = delegates.mapNotNull { + try { + it.run() + null + } catch (e: Exception) { + e + } + } + + when { + exceptions.isEmpty() -> return + exceptions.size == 1 -> throw exceptions[0] + else -> { + val root = Exception("Multiple exceptions were thrown and are added suppressed into this one") + exceptions.forEach { root.addSuppressed(it) } + throw root + } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt index cd7c1baa..c9afd1d0 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/DataCenterFormula.kt @@ -11,6 +11,7 @@ import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M4ExtraLar import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheEc2LoadBalancerFormula import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheProxyLoadBalancer +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.LoadBalancerFormula import com.atlassian.performance.tools.awsinfrastructure.api.network.Network import com.atlassian.performance.tools.awsinfrastructure.api.network.NetworkFormula @@ -19,6 +20,7 @@ import com.atlassian.performance.tools.awsinfrastructure.jira.DataCenterNodeForm import com.atlassian.performance.tools.awsinfrastructure.jira.DiagnosableNodeFormula import com.atlassian.performance.tools.awsinfrastructure.jira.StandaloneNodeFormula import com.atlassian.performance.tools.awsinfrastructure.jira.home.SharedHomeFormula +import com.atlassian.performance.tools.awsinfrastructure.loadbalancer.LoadBalancerMeasurementSource.Extension.asMeasurementSource import com.atlassian.performance.tools.concurrency.api.AbruptExecutorService import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.app.Apps @@ -326,6 +328,8 @@ class DataCenterFormula private constructor( loadBalancer.waitUntilHealthy(Duration.ofMinutes(5)) } + val loadBalancerResultsSource = (provisionedLoadBalancer.loadBalancer as? DiagnosableLoadBalancer) + ?.asMeasurementSource(resultsTransport.location) val jira = Jira.Builder( nodes = nodes, jiraHome = RemoteLocation( @@ -336,6 +340,7 @@ class DataCenterFormula private constructor( address = loadBalancer.uri ) .jmxClients(jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) }) + .extraMeasurementSources(listOfNotNull(loadBalancerResultsSource)) .build() logger.info("$jira is set up, will expire ${jiraStack.expiry}") diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt index 307032f0..b39989ac 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/Jira.kt @@ -1,40 +1,49 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira import com.atlassian.performance.tools.awsinfrastructure.api.CustomDatasetSource +import com.atlassian.performance.tools.awsinfrastructure.FailSafeRunnable import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation import com.atlassian.performance.tools.concurrency.api.submitWithLogContext import com.atlassian.performance.tools.infrastructure.api.MeasurementSource import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.JmxClient -import com.atlassian.performance.tools.jvmtasks.api.TaskTimer.time import com.google.common.util.concurrent.ThreadFactoryBuilder import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import java.net.URI import java.util.concurrent.Executors +/** + * @param extraMeasurementSources source of results/diagnostics of: reverse proxy, Crowd, LDAP, DVCS, DB, Jira plugins + * or anything that can be integrated with Jira as part of web application provisioning + */ class Jira private constructor( private val nodes: List, val jiraHome: RemoteLocation, val database: RemoteLocation, val address: URI, - val jmxClients: List = emptyList() + val jmxClients: List, + private val extraMeasurementSources: List ) : MeasurementSource { private val logger: Logger = LogManager.getLogger(this::class.java) override fun gatherResults() { - if (nodes.isEmpty()) { - logger.warn("No Jira nodes known to JPT, not downloading node results") + val firstNode = nodes.firstOrNull() + val measurementSources = extraMeasurementSources + nodes + listOfNotNull(firstNode?.let { AnalyticsLogsSource(it) }) + if (measurementSources.isEmpty()) { + logger.warn("No result sources known to Jira, can't download anything") return } val executor = Executors.newFixedThreadPool( - nodes.size.coerceAtMost(4), + measurementSources.size.coerceAtMost(4), ThreadFactoryBuilder() .setNameFormat("results-gathering-thread-%d") .build() ) - nodes.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } } - .forEach { it.get() } - time("gather analytics") { nodes.firstOrNull()?.gatherAnalyticLogs() } + FailSafeRunnable( + measurementSources.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } } + .map { Runnable { it.get() } } + ).run() + executor.shutdownNow() } @@ -55,14 +64,26 @@ class Jira private constructor( private val address: URI ) { private var jmxClients: List = emptyList() + private var extraMeasurementSources: List = emptyList() + fun jmxClients(jmxClients: List) = apply { this.jmxClients = jmxClients } + fun extraMeasurementSources(extraMeasurementSources: List) = apply { this.extraMeasurementSources = extraMeasurementSources } fun build() = Jira( nodes = nodes, jiraHome = jiraHome, database = database, address = address, - jmxClients = jmxClients + jmxClients = jmxClients, + extraMeasurementSources = extraMeasurementSources ) } + + private class AnalyticsLogsSource( + private val node: StartedNode + ) : MeasurementSource { + override fun gatherResults() { + node.gatherAnalyticLogs() + } + } } diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt index f778dc3c..71d61152 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/StartedNode.kt @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira import com.atlassian.performance.tools.aws.api.Storage import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli +import com.atlassian.performance.tools.infrastructure.api.MeasurementSource import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess import com.atlassian.performance.tools.ssh.api.Ssh @@ -15,10 +16,10 @@ class StartedNode( private val jiraPath: String, private val monitoringProcesses: List, private val ssh: Ssh -) { +) : MeasurementSource { private val resultsDirectory = "results" - fun gatherResults() { + override fun gatherResults() { ssh.newConnection().use { shell -> monitoringProcesses.forEach { it.stop(shell) } val nodeResultsDirectory = "$resultsDirectory/'$name'" diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt index 02fe6812..1111df1b 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheProxyLoadBalancer.kt @@ -1,7 +1,8 @@ package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli import com.atlassian.performance.tools.infrastructure.api.Sed -import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction import com.atlassian.performance.tools.ssh.api.Ssh @@ -14,7 +15,7 @@ class ApacheProxyLoadBalancer private constructor( private val ssh: Ssh, ipAddress: String, httpPort: Int -) : LoadBalancer { +) : DiagnosableLoadBalancer { override val uri: URI = URI("http://${ipAddress}:$httpPort/") @@ -81,6 +82,15 @@ class ApacheProxyLoadBalancer private constructor( connection.execute("echo \"$line\" | sudo tee -a $APACHE_CONFIG_PATH") } + override fun gatherEvidence(location: StorageLocation) { + ssh.newConnection().use { connection -> + val resultsDir = "/tmp/s3-results" + connection.execute("mkdir -p $resultsDir") + connection.execute("cp -R /var/log/apache2 $resultsDir") + AwsCli().upload(location, connection, resultsDir, Duration.ofMinutes(1)) + } + } + class Builder( private val ssh: Ssh ) { diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt new file mode 100644 index 00000000..1b33ae8e --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/DiagnosableLoadBalancer.kt @@ -0,0 +1,8 @@ +package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer + +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer + +interface DiagnosableLoadBalancer : LoadBalancer { + fun gatherEvidence(location: StorageLocation) +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt new file mode 100644 index 00000000..fc9837a4 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/loadbalancer/LoadBalancerMeasurementSource.kt @@ -0,0 +1,23 @@ +package com.atlassian.performance.tools.awsinfrastructure.loadbalancer + +import com.atlassian.performance.tools.aws.api.StorageLocation +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer +import com.atlassian.performance.tools.infrastructure.api.MeasurementSource + +internal class LoadBalancerMeasurementSource( + private val loadBalancer: DiagnosableLoadBalancer, + private val target: StorageLocation +) : MeasurementSource { + internal object Extension { + fun DiagnosableLoadBalancer.asMeasurementSource( + target: StorageLocation + ) = LoadBalancerMeasurementSource( + loadBalancer = this, + target = target + ) + } + + override fun gatherResults() { + loadBalancer.gatherEvidence(target) + } +} \ No newline at end of file diff --git a/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt new file mode 100644 index 00000000..c24f92de --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/FailSafeRunnableTest.kt @@ -0,0 +1,67 @@ +package com.atlassian.performance.tools.awsinfrastructure + +import org.assertj.core.api.Assertions.assertThat +import org.junit.Test + +class FailSafeRunnableTest { + @Test + fun shouldExecuteAllEvenIfFirstFails() { + var executed1 = false + var executed2 = false + var executed3 = false + val runnable = FailSafeRunnable( + listOf( + Runnable { executed1 = true; throw Exception("Fail 1") }, + Runnable { executed2 = true; throw Exception("Fail 2") }, + Runnable { executed3 = true; throw Exception("Fail 3") } + ) + ) + + try { + runnable.run() + } catch (e: Exception) { + // Expected and ignored, so that we can go to asserts + } + + assertThat(executed1).isTrue() + assertThat(executed2).isTrue() + assertThat(executed3).isTrue() + } + + @Test + fun shouldThrowAllFailures() { + val runnable = FailSafeRunnable( + listOf( + Runnable { throw Exception("Banana") }, + Runnable { throw Exception("Apple") }, + Runnable { throw Exception("Pear") }, + Runnable { throw Exception("Peach") } + ) + ) + + val exception = try { + runnable.run() + null + } catch (e: Exception) { + e + } + + val allExceptions = listOf(exception!!) + exception.suppressed.toList() + val allMessages = allExceptions.map { it.message } + assertThat(allMessages).contains("Banana", "Apple", "Pear", "Peach") + } + + + @Test + fun shouldExecuteAll() { + val allIndexes = Array(20) { it } + val finishedIndexes = mutableListOf() + val runnable = FailSafeRunnable( + allIndexes.map { index -> Runnable { finishedIndexes.add(index) } } + ) + + runnable.run() + + assertThat(finishedIndexes).contains(*allIndexes) + } +} \ No newline at end of file