From f7acef7072419b17f302800b4726e382eefaf829 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 4 Mar 2024 10:32:22 -0800 Subject: [PATCH 01/41] X-Smart-Branch-Parent: master From 8bc3ded2be8001baa4e7d36092ee74c8c5dd405d Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 5 Mar 2024 16:07:20 -0800 Subject: [PATCH 02/41] Trying to get core dumps from CI artifacts. --- .github/workflows/integration-tests-vm-type.yml | 1 + collector/collector.cpp | 1 + integration-tests/suites/common/collector_manager.go | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-vm-type.yml b/.github/workflows/integration-tests-vm-type.yml index c7da3db47e..f938fec700 100644 --- a/.github/workflows/integration-tests-vm-type.yml +++ b/.github/workflows/integration-tests-vm-type.yml @@ -117,3 +117,4 @@ jobs: path: | ${{ github.workspace }}/integration-tests/container-logs/**/* ${{ github.workspace }}/integration-tests/performance-logs/**/* + /tmp/core.out diff --git a/collector/collector.cpp b/collector/collector.cpp index 36d4dc1104..c14cae4fc6 100644 --- a/collector/collector.cpp +++ b/collector/collector.cpp @@ -130,6 +130,7 @@ void initialChecks() { } int main(int argc, char** argv) { + abort(); initialChecks(); CollectorArgs* args = CollectorArgs::getInstance(); diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index 7903d9568a..ab6e6e4ca8 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -49,7 +49,7 @@ func NewCollectorManager(e Executor, name string) *CollectorManager { "GRPC_SERVER": "localhost:9999", "COLLECTION_METHOD": collectionMethod, "COLLECTOR_PRE_ARGUMENTS": collectorOptions.PreArguments, - "ENABLE_CORE_DUMP": "false", + "ENABLE_CORE_DUMP": "true", } if !collectorOptions.Offline { From 117891000e92dc7a92ff8f61deb8c808fa26ef3a Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 6 Mar 2024 14:32:39 -0800 Subject: [PATCH 03/41] Ansible should get the core dumps and move them to the host machine --- .github/workflows/integration-tests-vm-type.yml | 2 +- .../roles/run-test-target/tasks/test-collection-method.yml | 6 ++++++ collector/collector.cpp | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-vm-type.yml b/.github/workflows/integration-tests-vm-type.yml index f938fec700..70ff9e4795 100644 --- a/.github/workflows/integration-tests-vm-type.yml +++ b/.github/workflows/integration-tests-vm-type.yml @@ -117,4 +117,4 @@ jobs: path: | ${{ github.workspace }}/integration-tests/container-logs/**/* ${{ github.workspace }}/integration-tests/performance-logs/**/* - /tmp/core.out + ${{ github.workspace }}/integration-tests/core.out diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 897a7644e7..c2a5835cf4 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -127,6 +127,12 @@ dest: "{{ logs_root }}/integration-test.log" delegate_to: localhost + - name: Fetch core dump + fetch: + src: "/tmp/core.out" + dest: "{{ logs_root }}/" + flat: true + - name: Report community.general.make: chdir: "{{ integration_tests_root }}" diff --git a/collector/collector.cpp b/collector/collector.cpp index c14cae4fc6..dfe84d4f36 100644 --- a/collector/collector.cpp +++ b/collector/collector.cpp @@ -130,7 +130,6 @@ void initialChecks() { } int main(int argc, char** argv) { - abort(); initialChecks(); CollectorArgs* args = CollectorArgs::getInstance(); @@ -146,6 +145,7 @@ int main(int argc, char** argv) { config.InitCollectorConfig(args); setCoreDumpLimit(config.IsCoreDumpEnabled()); + abort(); auto& startup_diagnostics = StartupDiagnostics::GetInstance(); From 63173a2653b044d958c9caacece187ff6c6d7176 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 6 Mar 2024 16:04:13 -0800 Subject: [PATCH 04/41] Added debugging --- integration-tests/suites/common/collector_manager.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index ab6e6e4ca8..d504db4631 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -242,11 +242,17 @@ func (c *CollectorManager) stopContainer(name string) error { // Sets the path to where core dumps are saved to. This is specified in the core_pattern file // The core_pattern file is backed up, because we don't want to permanently change it func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { + fmt.Println("In SetCoreDumpPath") if !config.HostInfo().IsLocal() { corePatternFile := "/proc/sys/kernel/core_pattern" corePatternBackupFile := "/tmp/core_pattern_backup" cmdBackupCorePattern := []string{"sudo", "cp", corePatternFile, corePatternBackupFile} cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "sudo", "tee", corePatternFile} + + fmt.Println(corePatternFile) + fmt.Println(corePatternBackupFile) + fmt.Println(cmdBackupCorePattern) + fmt.Println(cmdSetCoreDumpPath) var err error _, err = c.executor.Exec(cmdBackupCorePattern...) if err != nil { @@ -264,6 +270,7 @@ func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { // Restores the backed up core_pattern file, which sets the location where core dumps are written to. func (c *CollectorManager) RestoreCoreDumpPath() error { + fmt.Println("In RestoreCoreDumpPath") corePatternFile := "/proc/sys/kernel/core_pattern" corePatternBackupFile := "/tmp/core_pattern_backup" // cat is used to restore the backup instead of mv, becuase mv is not allowed. @@ -279,7 +286,9 @@ func (c *CollectorManager) RestoreCoreDumpPath() error { // If the integration test is run on a remote host the core dump needs to be copied from the remote host // to the local maching func (c *CollectorManager) GetCoreDump(coreDumpFile string) error { + fmt.Println("In GetCoreDump") if c.env["ENABLE_CORE_DUMP"] == "true" && !config.HostInfo().IsLocal() { + fmt.Println("Getting core dump") cmd := []string{"sudo", "chmod", "755", coreDumpFile} c.executor.Exec(cmd...) c.executor.CopyFromHost(coreDumpFile, coreDumpFile) From 72b403455e82602ae57ab8b1a1a48ac397496f25 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 6 Mar 2024 19:58:08 -0800 Subject: [PATCH 05/41] Moved the location of abort. Temporarily removed NetworkStatusNotifierTest --- collector/collector.cpp | 1 - collector/lib/NetworkStatusNotifier.cpp | 1 + collector/test/NetworkStatusNotifierTest.cpp | 339 ------------------- 3 files changed, 1 insertion(+), 340 deletions(-) delete mode 100644 collector/test/NetworkStatusNotifierTest.cpp diff --git a/collector/collector.cpp b/collector/collector.cpp index dfe84d4f36..36d4dc1104 100644 --- a/collector/collector.cpp +++ b/collector/collector.cpp @@ -145,7 +145,6 @@ int main(int argc, char** argv) { config.InitCollectorConfig(args); setCoreDumpLimit(config.IsCoreDumpEnabled()); - abort(); auto& startup_diagnostics = StartupDiagnostics::GetInstance(); diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index d944b30b93..35bd09f2e0 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -313,6 +313,7 @@ void NetworkStatusNotifier::RunSingleAfterglow(IDuplexClientWriter -#include -#include - -#include - -#include "internalapi/sensor/network_connection_iservice.grpc.pb.h" - -#include "CollectorArgs.h" -#include "CollectorConfig.h" -#include "DuplexGRPC.h" -#include "NetworkStatusNotifier.h" -#include "Utility.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -namespace collector { - -namespace { - -using grpc_duplex_impl::Result; -using grpc_duplex_impl::Status; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::ReturnPointee; -using ::testing::UnorderedElementsAre; - -/* Semaphore: A subset of the semaphore feature. - We use it to wait (with a timeout) for the NetworkStatusNotifier service thread to achieve the test scenario. - (C++20 comes with a compatible class (std::counting_semaphore) and we probably want to use this once we - update the C++ version). */ -class Semaphore { - public: - Semaphore(int value = 1) : value_(value) {} - - void release() { - std::unique_lock lock(mutex_); - value_++; - cond_.notify_one(); - } - - void acquire() { - std::unique_lock lock(mutex_); - - while (value_ <= 0) - cond_.wait(lock); - value_--; - } - - template - bool try_acquire_for(const std::chrono::duration& rel_time) { - auto deadline = std::chrono::steady_clock::now() + rel_time; - std::unique_lock lock(mutex_); - - while (value_ <= 0) - if (cond_.wait_until(lock, deadline) == std::cv_status::timeout) - return false; - value_--; - return true; - } - - private: - int value_; - std::condition_variable cond_; - std::mutex mutex_; -}; - -class MockCollectorConfig : public collector::CollectorConfig { - public: - MockCollectorConfig() : collector::CollectorConfig() {} - - void DisableAfterglow() { - enable_afterglow_ = false; - } -}; - -class MockConnScraper : public IConnScraper { - public: - MOCK_METHOD(bool, Scrape, (std::vector * connections, std::vector* listen_endpoints), (override)); -}; - -class MockDuplexClientWriter : public IDuplexClientWriter { - public: - MOCK_METHOD(grpc_duplex_impl::Result, Write, (const sensor::NetworkConnectionInfoMessage& obj, const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WriteAsync, (const sensor::NetworkConnectionInfoMessage& obj), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilStarted, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(bool, Sleep, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WritesDoneAsync, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WritesDone, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, FinishAsync, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilFinished, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, Finish, (grpc::Status * status, const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc::Status, Finish, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(void, TryCancel, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, Shutdown, (), (override)); -}; - -class MockNetworkConnectionInfoServiceComm : public INetworkConnectionInfoServiceComm { - public: - MOCK_METHOD(void, ResetClientContext, (), (override)); - MOCK_METHOD(bool, WaitForConnectionReady, (const std::function& check_interrupted), (override)); - MOCK_METHOD(void, TryCancel, (), (override)); - MOCK_METHOD(sensor::NetworkConnectionInfoService::StubInterface*, GetStub, (), (override)); - MOCK_METHOD(std::unique_ptr>, PushNetworkConnectionInfoOpenStream, (std::function receive_func), (override)); -}; - -/* gRPC payload objects are not strictly the ones of our internal model. - This helper class converts back NetworkConnectionInfoMessage into internal model in order - to compare them with the expected content. */ -class NetworkConnectionInfoMessageParser { - public: - NetworkConnectionInfoMessageParser(const sensor::NetworkConnectionInfoMessage& msg) { - for (auto conn : msg.info().updated_connections()) { - updated_connections_.emplace( - Connection( - conn.container_id(), - Endpoint(IPNetFromProto(conn.local_address()), conn.local_address().port()), - Endpoint(IPNetFromProto(conn.remote_address()), conn.remote_address().port()), - L4ProtocolFromProto(conn.protocol()), - conn.role() == sensor::ROLE_SERVER), - !conn.has_close_timestamp()); - } - } - - /* our internal model for connection life-cycle events uses the Connection object as the key for a hashmap. - The value in the hashmap describes the kind of event: true stands for a creation, false for a deletion. */ - const std::unordered_map& get_updated_connections() const { return updated_connections_; } - - private: - std::unordered_map updated_connections_; - - static IPNet IPNetFromProto(const sensor::NetworkAddress& na) { - Address addr; - const bool is_net = (na.ip_network().length() != 0); - - const std::string& bytes_stream = is_net ? na.ip_network() : na.address_data(); - - switch (bytes_stream.length()) { - case 0: - return IPNet(); - case 4: - case 5: { - uint32_t ip; - memcpy((void*)&ip, bytes_stream.c_str(), 4); - addr = Address(ip); - break; - } - case 16: - case 17: { - uint32_t ip[4]; - memcpy((void*)ip, bytes_stream.c_str(), 16); - addr = Address(ip); - break; - } - default: - throw std::runtime_error("Invalid address length"); - } - - if (is_net) { - const char* prefix_len = bytes_stream.c_str() + (bytes_stream.length() - 1); - - return IPNet(addr, *prefix_len); - } else { - return IPNet(addr); - } - } - - static L4Proto L4ProtocolFromProto(storage::L4Protocol proto) { - switch (proto) { - case storage::L4_PROTOCOL_TCP: - return L4Proto::TCP; - case storage::L4_PROTOCOL_UDP: - return L4Proto::UDP; - case storage::L4_PROTOCOL_ICMP: - return L4Proto::ICMP; - default: - return L4Proto::UNKNOWN; - } - } -}; - -/* Simple validation that the service starts and sends at least one event */ -TEST(NetworkStatusNotifier, SimpleStartStop) { - bool running = true; - CollectorConfig config_; - std::shared_ptr conn_scraper = std::make_shared(); - auto conn_tracker = std::make_shared(); - auto comm = std::make_shared(); - Semaphore sem(0); // to wait for the service to accomplish its job. - - // the connection is always ready - EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); - // gRPC shuts down the loop, so we will want writer->Sleep to return with false - EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); - - /* This is what NetworkStatusNotifier calls to create streams - receive_func is a callback that we can use to simulate messages coming from the sensor - We return an object that will get called when connections and endpoints are reported */ - EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) - .Times(1) - .WillOnce([&sem, &running](std::function receive_func) -> std::unique_ptr> { - auto duplex_writer = MakeUnique(); - - // the service is sending Sensor a message - EXPECT_CALL(*duplex_writer, Write).WillRepeatedly([&sem, &running](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - for (auto cnx : msg.info().updated_connections()) { - std::cout << cnx.container_id() << std::endl; - } - sem.release(); // notify that the test should end - return Result(Status::OK); - }); - EXPECT_CALL(*duplex_writer, Sleep).WillRepeatedly(ReturnPointee(&running)); - EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); - - return duplex_writer; - }); - - // Connections/Endpoints returned by the scrapper - EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&sem](std::vector* connections, std::vector* listen_endpoints) -> bool { - // this is the data which will trigger NetworkStatusNotifier to create a connection event (purpose of the test) - connections->emplace_back("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); - return true; - }); - - auto net_status_notifier = MakeUnique(conn_scraper, - conn_tracker, - comm, - config_); - - net_status_notifier->Start(); - - EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); - - net_status_notifier->Stop(); -} - -/* This test checks whether deltas are computed appropriately in case the "known network" list is received after a connection - is already reported (and matches one of the networks). - - scrapper initialy reports a connection - - the connection is reported - - we feed a "known network" into the NetworkStatusNotifier - - we check that the former declared connection is deleted and redeclared as part of this network */ -TEST(NetworkStatusNotifier, UpdateIPnoAfterglow) { - bool running = true; - MockCollectorConfig config; - config.DisableAfterglow(); - std::shared_ptr conn_scraper = std::make_shared(); - auto conn_tracker = std::make_shared(); - auto comm = std::make_shared(); - std::function network_flows_callback; - Semaphore sem(0); // to wait for the service to accomplish its job. - - // the connection as scrapped (public) - Connection conn1("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); - // the same server connection normalized - Connection conn2("containerId", Endpoint(Address(), 1024), Endpoint(Address(255, 255, 255, 255), 0), L4Proto::TCP, true); - // the same server connection normalized and grouped in a known subnet - Connection conn3("containerId", Endpoint(Address(), 1024), Endpoint(IPNet(Address(139, 45, 0, 0), 16), 0), L4Proto::TCP, true); - - // the connection is always ready - EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); - // gRPC shuts down the loop, so we will want writer->Sleep to return with false - EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); - - /* This is what NetworkStatusNotifier calls to create streams - receive_func is a callback that we can use to simulate messages coming from the sensor - We return an object that will get called when connections and endpoints are reported */ - EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) - .Times(1) - .WillOnce([&sem, - &running, - &conn2, - &conn3, - &network_flows_callback](std::function receive_func) -> std::unique_ptr> { - auto duplex_writer = MakeUnique(); - network_flows_callback = receive_func; - - // the service is sending Sensor a message - EXPECT_CALL(*duplex_writer, Write) - .WillOnce([&conn2, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - // the connection reported by the scrapper is annouced as generic public - EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn2, true))); - return Result(Status::OK); - }) - .WillOnce([&conn2, &conn3, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - // after the network is declared, the connection switches to the new state - // conn3 appears and conn2 is destroyed - EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn3, true), std::make_pair(conn2, false))); - - // Done - sem.release(); - - return Result(Status::OK); - }) - .WillRepeatedly(Return(Result(Status::OK))); - - EXPECT_CALL(*duplex_writer, Sleep) - .WillOnce(ReturnPointee(&running)) // first time, we let the scrapper do its job - .WillOnce([&running, &network_flows_callback](const gpr_timespec& deadline) { - // The connection is known now, let's declare a "known network" - sensor::NetworkFlowsControlMessage msg; - unsigned char content[] = {139, 45, 0, 0, 16}; // address in network order, plus prefix length - std::string network((char*)content, sizeof(content)); - - auto ip_networks = msg.mutable_ip_networks(); - ip_networks->set_ipv4_networks(network); - - network_flows_callback(&msg); - return running; - }) - .WillRepeatedly(ReturnPointee(&running)); - - EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); - - return duplex_writer; - }); - - // Connections/Endpoints returned by the scrapper (first detection of the connection) - EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&conn1](std::vector* connections, std::vector* listen_endpoints) -> bool { - connections->emplace_back(conn1); - return true; - }); - - auto net_status_notifier = MakeUnique(conn_scraper, - conn_tracker, - comm, - config); - - net_status_notifier->Start(); - - // Wait for the first scrape to occur - EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); - - net_status_notifier->Stop(); -} - -} // namespace - -} // namespace collector From fa37810a7b09faaae7b45f41b41d8ed1a93a4cea Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 6 Mar 2024 21:47:55 -0800 Subject: [PATCH 06/41] More debugging --- integration-tests/suites/common/collector_manager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index d504db4631..fbca97c06d 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -287,6 +287,8 @@ func (c *CollectorManager) RestoreCoreDumpPath() error { // to the local maching func (c *CollectorManager) GetCoreDump(coreDumpFile string) error { fmt.Println("In GetCoreDump") + fmt.Println(c.env["ENABLE_CORE_DUMP"]) + fmt.Println(config.HostInfo().IsLocal()) if c.env["ENABLE_CORE_DUMP"] == "true" && !config.HostInfo().IsLocal() { fmt.Println("Getting core dump") cmd := []string{"sudo", "chmod", "755", coreDumpFile} From c42bec50ae3bb1814a73a45ee148329c42271afc Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Wed, 6 Mar 2024 22:30:31 -0800 Subject: [PATCH 07/41] Not checking if the tests are running locally before modifying the core dump file path --- .../suites/common/collector_manager.go | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index fbca97c06d..fb47d2854b 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -243,28 +243,28 @@ func (c *CollectorManager) stopContainer(name string) error { // The core_pattern file is backed up, because we don't want to permanently change it func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { fmt.Println("In SetCoreDumpPath") - if !config.HostInfo().IsLocal() { - corePatternFile := "/proc/sys/kernel/core_pattern" - corePatternBackupFile := "/tmp/core_pattern_backup" - cmdBackupCorePattern := []string{"sudo", "cp", corePatternFile, corePatternBackupFile} - cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "sudo", "tee", corePatternFile} - - fmt.Println(corePatternFile) - fmt.Println(corePatternBackupFile) - fmt.Println(cmdBackupCorePattern) - fmt.Println(cmdSetCoreDumpPath) - var err error - _, err = c.executor.Exec(cmdBackupCorePattern...) - if err != nil { - fmt.Printf("Error: Unable to backup core_pattern file. %v\n", err) - return err - } - _, err = c.executor.Exec(cmdSetCoreDumpPath...) - if err != nil { - fmt.Printf("Error: Unable to set core dump file path in core_pattern. %v\n", err) - return err - } + // if !config.HostInfo().IsLocal() { + corePatternFile := "/proc/sys/kernel/core_pattern" + corePatternBackupFile := "/tmp/core_pattern_backup" + cmdBackupCorePattern := []string{"sudo", "cp", corePatternFile, corePatternBackupFile} + cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "sudo", "tee", corePatternFile} + + fmt.Println(corePatternFile) + fmt.Println(corePatternBackupFile) + fmt.Println(cmdBackupCorePattern) + fmt.Println(cmdSetCoreDumpPath) + var err error + _, err = c.executor.Exec(cmdBackupCorePattern...) + if err != nil { + fmt.Printf("Error: Unable to backup core_pattern file. %v\n", err) + return err + } + _, err = c.executor.Exec(cmdSetCoreDumpPath...) + if err != nil { + fmt.Printf("Error: Unable to set core dump file path in core_pattern. %v\n", err) + return err } + // } return nil } From 95dfff1caf11ae938741334f5870aa1eabb5f9ab Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 11:50:45 -0800 Subject: [PATCH 08/41] Not using sudo to modify core_pattern file --- integration-tests/suites/common/collector_manager.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index fb47d2854b..a99661753c 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -246,8 +246,10 @@ func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { // if !config.HostInfo().IsLocal() { corePatternFile := "/proc/sys/kernel/core_pattern" corePatternBackupFile := "/tmp/core_pattern_backup" - cmdBackupCorePattern := []string{"sudo", "cp", corePatternFile, corePatternBackupFile} - cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "sudo", "tee", corePatternFile} + cmdBackupCorePattern := []string{"cp", corePatternFile, corePatternBackupFile} + //cmdBackupCorePattern := []string{"sudo", "cp", corePatternFile, corePatternBackupFile} + cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "tee", corePatternFile} + //cmdSetCoreDumpPath := []string{"echo", "'" + coreDumpFile + "'", "|", "sudo", "tee", corePatternFile} fmt.Println(corePatternFile) fmt.Println(corePatternBackupFile) From bad2bf4940aefd7846a29680a0276c3c60db6816 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 15:21:35 -0800 Subject: [PATCH 09/41] More debugging --- integration-tests/suites/common/collector_manager.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index a99661753c..34113c442d 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -267,6 +267,14 @@ func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { return err } // } + + cmdCat := []string{"cat", corePatternFile} + catResult, err := c.executor.Exec(cmdCat...) + if err != nil { + fmt.Print("Unable to read corePatternFile") + } + fmt.Println(catResult) + return nil } From 0d2516cf53f79411be5b170dd4854b6331e33bfb Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 17:01:17 -0800 Subject: [PATCH 10/41] Trying to change core dump file path in the ansible playbook --- ansible/roles/run-test-target/tasks/test-collection-method.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index c2a5835cf4..c2d4f16a10 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -65,6 +65,7 @@ - name: Run integration tests become: "{{ runtime_as_root }}" shell: | + echo '/tmp/core.out' | sudo tee /proc/sys/kernel/core_pattern {{ runtime_command }} run -it --rm \ -v {{ remote_log_mount }}:{{ container_logs_root }} \ -v ~/.docker:/root/.docker \ From a3c8d0b434caef2ee08920f56ed3a687009e3414 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 19:44:15 -0800 Subject: [PATCH 11/41] Checking tmp directory of integration tests --- ansible/roles/run-test-target/tasks/test-collection-method.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index c2d4f16a10..b11e22ada2 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -80,6 +80,8 @@ -e COLLECTOR_LOG_LEVEL \ {{ integration_tests_image }} \ {{ run_args }} -test.v 2>&1 + + ls -lrt /tmp environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" From c9a5c44589c8b4cd382c33348b92703bfafc5b9f Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 20:34:08 -0800 Subject: [PATCH 12/41] Trying to change the permissions of the core dump file --- .../run-test-target/tasks/test-collection-method.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index b11e22ada2..dd651ea7d3 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -82,6 +82,7 @@ {{ run_args }} -test.v 2>&1 ls -lrt /tmp + environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" @@ -93,6 +94,17 @@ # ensure that this is printed tags: [print_action] + - name: Check if the file exists + ansible.builtin.stat: + path: /tmp/core.out + register: file_stat + + - name: Change permissions if the file exists + ansible.builtin.file: + path: /tmp/core.out + mode: "u+r" + when: file_stat.stat.exists + rescue: - name: Set tests as failed set_fact: From 3fda8d2f84c249b3b8e70f7ac82b50c24e0266c4 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Thu, 7 Mar 2024 21:05:40 -0800 Subject: [PATCH 13/41] Core dump permissions are changed in the always section --- .../tasks/test-collection-method.yml | 21 ++++++++++--------- collector/lib/NetworkStatusNotifier.cpp | 4 +++- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index dd651ea7d3..764b7f068f 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -94,16 +94,6 @@ # ensure that this is printed tags: [print_action] - - name: Check if the file exists - ansible.builtin.stat: - path: /tmp/core.out - register: file_stat - - - name: Change permissions if the file exists - ansible.builtin.file: - path: /tmp/core.out - mode: "u+r" - when: file_stat.stat.exists rescue: - name: Set tests as failed @@ -142,6 +132,17 @@ dest: "{{ logs_root }}/integration-test.log" delegate_to: localhost + - name: Check if the file exists + ansible.builtin.stat: + path: /tmp/core.out + register: file_stat + + - name: Change permissions if the file exists + ansible.builtin.file: + path: /tmp/core.out + mode: "u+r" + when: file_stat.stat.exists + - name: Fetch core dump fetch: src: "/tmp/core.out" diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index 35bd09f2e0..2f99e66f4f 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -313,7 +313,9 @@ void NetworkStatusNotifier::RunSingleAfterglow(IDuplexClientWriter Date: Fri, 8 Mar 2024 11:14:45 -0800 Subject: [PATCH 14/41] Trying to check tmp directory --- .../roles/run-test-target/tasks/test-collection-method.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 764b7f068f..6f7e72b1d7 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -143,6 +143,10 @@ mode: "u+r" when: file_stat.stat.exists + - name: Check tmp directory + become: "{{ runtime_as_root }}" + shell: "ls -l /tmp" + - name: Fetch core dump fetch: src: "/tmp/core.out" From 801f645eeb0d0e418931b394130d8e3dae78dd03 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Fri, 8 Mar 2024 16:39:45 -0800 Subject: [PATCH 15/41] Better debugging --- .../roles/run-test-target/tasks/test-collection-method.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 6f7e72b1d7..28fbb4a44f 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -146,6 +146,12 @@ - name: Check tmp directory become: "{{ runtime_as_root }}" shell: "ls -l /tmp" + register: ls_tmp_output + + - name: Print tmp directory contents + debug: + var: ls_tmp_output.stdout_lines + - name: Fetch core dump fetch: From 9264a4e21cb47486f0d48be9cba51ee855fccf68 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Fri, 8 Mar 2024 19:37:32 -0800 Subject: [PATCH 16/41] Using root user to fetch core dump --- ansible/roles/run-test-target/tasks/test-collection-method.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 28fbb4a44f..6356bb7d5d 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -152,12 +152,13 @@ debug: var: ls_tmp_output.stdout_lines - - name: Fetch core dump + become: yes fetch: src: "/tmp/core.out" dest: "{{ logs_root }}/" flat: true + when: file_stat.stat.exists - name: Report community.general.make: From 564ae4e137e8490f1ded217cd2e48d2bb6e71573 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 9 Mar 2024 14:58:55 -0800 Subject: [PATCH 17/41] Getting core dump files of the form /tmp/core.out.* not just /tmp/core.out --- .../tasks/test-collection-method.yml | 4 +- collector/lib/NetworkStatusNotifier.cpp | 6 ++ .../suites/common/collector_manager.go | 55 ------------------- 3 files changed, 8 insertions(+), 57 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 6356bb7d5d..5a5bdbd7f6 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -155,10 +155,10 @@ - name: Fetch core dump become: yes fetch: - src: "/tmp/core.out" + src: "/tmp/core.out.*" dest: "{{ logs_root }}/" flat: true - when: file_stat.stat.exists + #when: file_stat.stat.exists - name: Report community.general.make: diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index 2f99e66f4f..7639c60ade 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -316,6 +316,12 @@ void NetworkStatusNotifier::RunSingleAfterglow(IDuplexClientWriter Date: Sat, 9 Mar 2024 16:04:03 -0800 Subject: [PATCH 18/41] Fixed error so that all core dumps should be fetched --- ansible/roles/run-test-target/tasks/test-collection-method.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 5a5bdbd7f6..5929b598df 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -155,10 +155,9 @@ - name: Fetch core dump become: yes fetch: - src: "/tmp/core.out.*" + src: "/tmp/core.out*" dest: "{{ logs_root }}/" flat: true - #when: file_stat.stat.exists - name: Report community.general.make: From 84e4aa3f25d7af4911d5e7805d45b7de2b022446 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 9 Mar 2024 18:15:05 -0800 Subject: [PATCH 19/41] Fetching core dump files in a loop --- .../tasks/test-collection-method.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 5929b598df..e6d0a68efd 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -152,12 +152,24 @@ debug: var: ls_tmp_output.stdout_lines - - name: Fetch core dump + - name: Get core dump file paths become: yes + shell: "ls /tmp/core.out*" + register: core_files + + - name: Fetch each core dump file fetch: - src: "/tmp/core.out*" + src: "{{ item }}" dest: "{{ logs_root }}/" flat: true + with_items: "{{ core_files.stdout_lines }}" + + #- name: Fetch core dump + # become: yes + # fetch: + # src: "/tmp/core.out*" + # dest: "{{ logs_root }}/" + # flat: true - name: Report community.general.make: From 577afc4740f8b7aa32b96fc0d08ce68db930207d Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sat, 9 Mar 2024 21:10:38 -0800 Subject: [PATCH 20/41] Trying to fix error --- .../run-test-target/tasks/test-collection-method.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index e6d0a68efd..6f1199f90a 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -157,12 +157,21 @@ shell: "ls /tmp/core.out*" register: core_files + # - name: Fetch each core dump file + # fetch: + # src: "{{ item }}" + # dest: "{{ logs_root }}/" + # flat: true + # with_items: "{{ core_files.stdout_lines }}" + - name: Fetch each core dump file fetch: src: "{{ item }}" dest: "{{ logs_root }}/" flat: true with_items: "{{ core_files.stdout_lines }}" + loop_control: + loop_var: core_file #- name: Fetch core dump # become: yes From 56328e23799b7d95a8d9b9af792ab1cfb349478b Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 10:19:17 -0700 Subject: [PATCH 21/41] Using root to fetch core dump files --- .../tasks/test-collection-method.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 6f1199f90a..111beb9a34 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -157,14 +157,8 @@ shell: "ls /tmp/core.out*" register: core_files - # - name: Fetch each core dump file - # fetch: - # src: "{{ item }}" - # dest: "{{ logs_root }}/" - # flat: true - # with_items: "{{ core_files.stdout_lines }}" - - name: Fetch each core dump file + become: yes fetch: src: "{{ item }}" dest: "{{ logs_root }}/" @@ -173,6 +167,14 @@ loop_control: loop_var: core_file + # - name: Fetch each core dump file + # fetch: + # src: "{{ item }}" + # dest: "{{ logs_root }}/" + # flat: true + # with_items: "{{ core_files.stdout_lines }}" + + #- name: Fetch core dump # become: yes # fetch: From 3c11cd46591b87d1b06ed4c2c37877df82655f66 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 11:53:24 -0700 Subject: [PATCH 22/41] Trying to follow pattern for fetching log files --- .../tasks/test-collection-method.yml | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 111beb9a34..f7eabdd5e8 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -152,21 +152,46 @@ debug: var: ls_tmp_output.stdout_lines + #- name: Get core dump file paths + # become: yes + # shell: "ls /tmp/core.out*" + # register: core_files + - name: Get core dump file paths become: yes - shell: "ls /tmp/core.out*" + find: + paths: "/tmp/core.out*" register: core_files - name: Fetch each core dump file become: yes fetch: - src: "{{ item }}" + src: "{{ core_file }}" dest: "{{ logs_root }}/" flat: true - with_items: "{{ core_files.stdout_lines }}" + loop: "{{ core_files.files | map(attribute='path') | list }}" loop_control: loop_var: core_file + - name: Check tmp directory 2 + become: "{{ runtime_as_root }}" + shell: "ls -l /tmp" + register: ls_tmp_output_2 + + - name: Print tmp directory contents 2 + debug: + var: ls_tmp_output_2.stdout_lines + + # - name: Fetch each core dump file + # become: yes + # fetch: + # src: "{{ item }}" + # dest: "{{ logs_root }}/" + # flat: true + # with_items: "{{ core_files.stdout_lines }}" + # loop_control: + # loop_var: core_file + # - name: Fetch each core dump file # fetch: # src: "{{ item }}" From f492202eb3663e9edd14a2685d21d66973ca912d Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 14:50:16 -0700 Subject: [PATCH 23/41] Moving core dump files to their own directory --- .../tasks/test-collection-method.yml | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index f7eabdd5e8..59029658ca 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -81,8 +81,6 @@ {{ integration_tests_image }} \ {{ run_args }} -test.v 2>&1 - ls -lrt /tmp - environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" @@ -132,20 +130,26 @@ dest: "{{ logs_root }}/integration-test.log" delegate_to: localhost + - name: Move core dump files to core dump directory + become: yes + shell: | + mkdir /tmp/core_dumps + mv /tmp/core.out* /tmp/core_dumps || true + - name: Check if the file exists ansible.builtin.stat: path: /tmp/core.out register: file_stat - - name: Change permissions if the file exists - ansible.builtin.file: - path: /tmp/core.out - mode: "u+r" - when: file_stat.stat.exists + #- name: Change permissions if the file exists + # ansible.builtin.file: + # path: /tmp/core.out + # mode: "u+r" + # when: file_stat.stat.exists - name: Check tmp directory become: "{{ runtime_as_root }}" - shell: "ls -l /tmp" + shell: "ls -l /tmp/core_dumps" register: ls_tmp_output - name: Print tmp directory contents @@ -160,7 +164,7 @@ - name: Get core dump file paths become: yes find: - paths: "/tmp/core.out*" + paths: "/tmp/core_dumps/" register: core_files - name: Fetch each core dump file @@ -175,7 +179,7 @@ - name: Check tmp directory 2 become: "{{ runtime_as_root }}" - shell: "ls -l /tmp" + shell: "ls -l /tmp/core_dumps" register: ls_tmp_output_2 - name: Print tmp directory contents 2 From 4d7a7a709ad9d3e407cda996c96b11c1c6cc77dd Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 16:06:43 -0700 Subject: [PATCH 24/41] Clean up --- .../tasks/test-collection-method.yml | 59 ------------------- .../suites/common/collector_manager.go | 30 ---------- 2 files changed, 89 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 59029658ca..1a18e766af 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -136,31 +136,6 @@ mkdir /tmp/core_dumps mv /tmp/core.out* /tmp/core_dumps || true - - name: Check if the file exists - ansible.builtin.stat: - path: /tmp/core.out - register: file_stat - - #- name: Change permissions if the file exists - # ansible.builtin.file: - # path: /tmp/core.out - # mode: "u+r" - # when: file_stat.stat.exists - - - name: Check tmp directory - become: "{{ runtime_as_root }}" - shell: "ls -l /tmp/core_dumps" - register: ls_tmp_output - - - name: Print tmp directory contents - debug: - var: ls_tmp_output.stdout_lines - - #- name: Get core dump file paths - # become: yes - # shell: "ls /tmp/core.out*" - # register: core_files - - name: Get core dump file paths become: yes find: @@ -177,40 +152,6 @@ loop_control: loop_var: core_file - - name: Check tmp directory 2 - become: "{{ runtime_as_root }}" - shell: "ls -l /tmp/core_dumps" - register: ls_tmp_output_2 - - - name: Print tmp directory contents 2 - debug: - var: ls_tmp_output_2.stdout_lines - - # - name: Fetch each core dump file - # become: yes - # fetch: - # src: "{{ item }}" - # dest: "{{ logs_root }}/" - # flat: true - # with_items: "{{ core_files.stdout_lines }}" - # loop_control: - # loop_var: core_file - - # - name: Fetch each core dump file - # fetch: - # src: "{{ item }}" - # dest: "{{ logs_root }}/" - # flat: true - # with_items: "{{ core_files.stdout_lines }}" - - - #- name: Fetch core dump - # become: yes - # fetch: - # src: "/tmp/core.out*" - # dest: "{{ logs_root }}/" - # flat: true - - name: Report community.general.make: chdir: "{{ integration_tests_root }}" diff --git a/integration-tests/suites/common/collector_manager.go b/integration-tests/suites/common/collector_manager.go index 35acdccb68..69d7905430 100644 --- a/integration-tests/suites/common/collector_manager.go +++ b/integration-tests/suites/common/collector_manager.go @@ -27,7 +27,6 @@ type CollectorManager struct { env map[string]string config map[string]any bootstrapOnly bool - coreDumpPath string testName string CollectorOutput string @@ -74,7 +73,6 @@ func NewCollectorManager(e Executor, name string) *CollectorManager { mounts: mounts, config: collectorConfig, testName: name, - coreDumpPath: "/tmp/core.out", } } @@ -105,11 +103,6 @@ func (c *CollectorManager) Launch() error { } func (c *CollectorManager) TearDown() error { - coreDumpErr := c.GetCoreDump(c.coreDumpPath) - if coreDumpErr != nil { - return coreDumpErr - } - isRunning, err := c.IsRunning() if err != nil { fmt.Println("Error: Checking if container running") @@ -156,11 +149,6 @@ func (c *CollectorManager) getAllContainers() (string, error) { } func (c *CollectorManager) launchCollector() error { - coreDumpErr := c.SetCoreDumpPath(c.coreDumpPath) - if coreDumpErr != nil { - return coreDumpErr - } - cmd := []string{RuntimeCommand, "run", "--name", "collector", "--privileged", @@ -238,21 +226,3 @@ func (c *CollectorManager) stopContainer(name string) error { _, err := c.executor.StopContainer(name) return err } - -// Sets the path to where core dumps are saved to. This is specified in the core_pattern file -// The core_pattern file is backed up, because we don't want to permanently change it -func (c *CollectorManager) SetCoreDumpPath(coreDumpFile string) error { - - return nil -} - -// Restores the backed up core_pattern file, which sets the location where core dumps are written to. -func (c *CollectorManager) RestoreCoreDumpPath() error { - return nil -} - -// If the integration test is run on a remote host the core dump needs to be copied from the remote host -// to the local maching -func (c *CollectorManager) GetCoreDump(coreDumpFile string) error { - return nil -} From f683effccf3fb8ebd0ebefd522c6fce1a7f493a6 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 17:09:42 -0700 Subject: [PATCH 25/41] Removed intentional crash. Restored unit tests --- collector/lib/NetworkStatusNotifier.cpp | 9 - collector/test/NetworkStatusNotifierTest.cpp | 339 +++++++++++++++++++ 2 files changed, 339 insertions(+), 9 deletions(-) create mode 100644 collector/test/NetworkStatusNotifierTest.cpp diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index 7639c60ade..d944b30b93 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -313,15 +313,6 @@ void NetworkStatusNotifier::RunSingleAfterglow(IDuplexClientWriter lock(mutex_); + value_++; + cond_.notify_one(); + } + + void acquire() { + std::unique_lock lock(mutex_); + + while (value_ <= 0) + cond_.wait(lock); + value_--; + } + + template + bool try_acquire_for(const std::chrono::duration& rel_time) { + auto deadline = std::chrono::steady_clock::now() + rel_time; + std::unique_lock lock(mutex_); + + while (value_ <= 0) + if (cond_.wait_until(lock, deadline) == std::cv_status::timeout) + return false; + value_--; + return true; + } + + private: + int value_; + std::condition_variable cond_; + std::mutex mutex_; +}; + +class MockCollectorConfig : public collector::CollectorConfig { + public: + MockCollectorConfig() : collector::CollectorConfig() {} + + void DisableAfterglow() { + enable_afterglow_ = false; + } +}; + +class MockConnScraper : public IConnScraper { + public: + MOCK_METHOD(bool, Scrape, (std::vector * connections, std::vector* listen_endpoints), (override)); +}; + +class MockDuplexClientWriter : public IDuplexClientWriter { + public: + MOCK_METHOD(grpc_duplex_impl::Result, Write, (const sensor::NetworkConnectionInfoMessage& obj, const gpr_timespec& deadline), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, WriteAsync, (const sensor::NetworkConnectionInfoMessage& obj), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilStarted, (const gpr_timespec& deadline), (override)); + MOCK_METHOD(bool, Sleep, (const gpr_timespec& deadline), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, WritesDoneAsync, (), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, WritesDone, (const gpr_timespec& deadline), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, FinishAsync, (), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilFinished, (const gpr_timespec& deadline), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, Finish, (grpc::Status * status, const gpr_timespec& deadline), (override)); + MOCK_METHOD(grpc::Status, Finish, (const gpr_timespec& deadline), (override)); + MOCK_METHOD(void, TryCancel, (), (override)); + MOCK_METHOD(grpc_duplex_impl::Result, Shutdown, (), (override)); +}; + +class MockNetworkConnectionInfoServiceComm : public INetworkConnectionInfoServiceComm { + public: + MOCK_METHOD(void, ResetClientContext, (), (override)); + MOCK_METHOD(bool, WaitForConnectionReady, (const std::function& check_interrupted), (override)); + MOCK_METHOD(void, TryCancel, (), (override)); + MOCK_METHOD(sensor::NetworkConnectionInfoService::StubInterface*, GetStub, (), (override)); + MOCK_METHOD(std::unique_ptr>, PushNetworkConnectionInfoOpenStream, (std::function receive_func), (override)); +}; + +/* gRPC payload objects are not strictly the ones of our internal model. + This helper class converts back NetworkConnectionInfoMessage into internal model in order + to compare them with the expected content. */ +class NetworkConnectionInfoMessageParser { + public: + NetworkConnectionInfoMessageParser(const sensor::NetworkConnectionInfoMessage& msg) { + for (auto conn : msg.info().updated_connections()) { + updated_connections_.emplace( + Connection( + conn.container_id(), + Endpoint(IPNetFromProto(conn.local_address()), conn.local_address().port()), + Endpoint(IPNetFromProto(conn.remote_address()), conn.remote_address().port()), + L4ProtocolFromProto(conn.protocol()), + conn.role() == sensor::ROLE_SERVER), + !conn.has_close_timestamp()); + } + } + + /* our internal model for connection life-cycle events uses the Connection object as the key for a hashmap. + The value in the hashmap describes the kind of event: true stands for a creation, false for a deletion. */ + const std::unordered_map& get_updated_connections() const { return updated_connections_; } + + private: + std::unordered_map updated_connections_; + + static IPNet IPNetFromProto(const sensor::NetworkAddress& na) { + Address addr; + const bool is_net = (na.ip_network().length() != 0); + + const std::string& bytes_stream = is_net ? na.ip_network() : na.address_data(); + + switch (bytes_stream.length()) { + case 0: + return IPNet(); + case 4: + case 5: { + uint32_t ip; + memcpy((void*)&ip, bytes_stream.c_str(), 4); + addr = Address(ip); + break; + } + case 16: + case 17: { + uint32_t ip[4]; + memcpy((void*)ip, bytes_stream.c_str(), 16); + addr = Address(ip); + break; + } + default: + throw std::runtime_error("Invalid address length"); + } + + if (is_net) { + const char* prefix_len = bytes_stream.c_str() + (bytes_stream.length() - 1); + + return IPNet(addr, *prefix_len); + } else { + return IPNet(addr); + } + } + + static L4Proto L4ProtocolFromProto(storage::L4Protocol proto) { + switch (proto) { + case storage::L4_PROTOCOL_TCP: + return L4Proto::TCP; + case storage::L4_PROTOCOL_UDP: + return L4Proto::UDP; + case storage::L4_PROTOCOL_ICMP: + return L4Proto::ICMP; + default: + return L4Proto::UNKNOWN; + } + } +}; + +/* Simple validation that the service starts and sends at least one event */ +TEST(NetworkStatusNotifier, SimpleStartStop) { + bool running = true; + CollectorConfig config_; + std::shared_ptr conn_scraper = std::make_shared(); + auto conn_tracker = std::make_shared(); + auto comm = std::make_shared(); + Semaphore sem(0); // to wait for the service to accomplish its job. + + // the connection is always ready + EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); + // gRPC shuts down the loop, so we will want writer->Sleep to return with false + EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); + + /* This is what NetworkStatusNotifier calls to create streams + receive_func is a callback that we can use to simulate messages coming from the sensor + We return an object that will get called when connections and endpoints are reported */ + EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) + .Times(1) + .WillOnce([&sem, &running](std::function receive_func) -> std::unique_ptr> { + auto duplex_writer = MakeUnique(); + + // the service is sending Sensor a message + EXPECT_CALL(*duplex_writer, Write).WillRepeatedly([&sem, &running](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { + for (auto cnx : msg.info().updated_connections()) { + std::cout << cnx.container_id() << std::endl; + } + sem.release(); // notify that the test should end + return Result(Status::OK); + }); + EXPECT_CALL(*duplex_writer, Sleep).WillRepeatedly(ReturnPointee(&running)); + EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); + + return duplex_writer; + }); + + // Connections/Endpoints returned by the scrapper + EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&sem](std::vector* connections, std::vector* listen_endpoints) -> bool { + // this is the data which will trigger NetworkStatusNotifier to create a connection event (purpose of the test) + connections->emplace_back("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); + return true; + }); + + auto net_status_notifier = MakeUnique(conn_scraper, + conn_tracker, + comm, + config_); + + net_status_notifier->Start(); + + EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); + + net_status_notifier->Stop(); +} + +/* This test checks whether deltas are computed appropriately in case the "known network" list is received after a connection + is already reported (and matches one of the networks). + - scrapper initialy reports a connection + - the connection is reported + - we feed a "known network" into the NetworkStatusNotifier + - we check that the former declared connection is deleted and redeclared as part of this network */ +TEST(NetworkStatusNotifier, UpdateIPnoAfterglow) { + bool running = true; + MockCollectorConfig config; + config.DisableAfterglow(); + std::shared_ptr conn_scraper = std::make_shared(); + auto conn_tracker = std::make_shared(); + auto comm = std::make_shared(); + std::function network_flows_callback; + Semaphore sem(0); // to wait for the service to accomplish its job. + + // the connection as scrapped (public) + Connection conn1("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); + // the same server connection normalized + Connection conn2("containerId", Endpoint(Address(), 1024), Endpoint(Address(255, 255, 255, 255), 0), L4Proto::TCP, true); + // the same server connection normalized and grouped in a known subnet + Connection conn3("containerId", Endpoint(Address(), 1024), Endpoint(IPNet(Address(139, 45, 0, 0), 16), 0), L4Proto::TCP, true); + + // the connection is always ready + EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); + // gRPC shuts down the loop, so we will want writer->Sleep to return with false + EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); + + /* This is what NetworkStatusNotifier calls to create streams + receive_func is a callback that we can use to simulate messages coming from the sensor + We return an object that will get called when connections and endpoints are reported */ + EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) + .Times(1) + .WillOnce([&sem, + &running, + &conn2, + &conn3, + &network_flows_callback](std::function receive_func) -> std::unique_ptr> { + auto duplex_writer = MakeUnique(); + network_flows_callback = receive_func; + + // the service is sending Sensor a message + EXPECT_CALL(*duplex_writer, Write) + .WillOnce([&conn2, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { + // the connection reported by the scrapper is annouced as generic public + EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn2, true))); + return Result(Status::OK); + }) + .WillOnce([&conn2, &conn3, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { + // after the network is declared, the connection switches to the new state + // conn3 appears and conn2 is destroyed + EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn3, true), std::make_pair(conn2, false))); + + // Done + sem.release(); + + return Result(Status::OK); + }) + .WillRepeatedly(Return(Result(Status::OK))); + + EXPECT_CALL(*duplex_writer, Sleep) + .WillOnce(ReturnPointee(&running)) // first time, we let the scrapper do its job + .WillOnce([&running, &network_flows_callback](const gpr_timespec& deadline) { + // The connection is known now, let's declare a "known network" + sensor::NetworkFlowsControlMessage msg; + unsigned char content[] = {139, 45, 0, 0, 16}; // address in network order, plus prefix length + std::string network((char*)content, sizeof(content)); + + auto ip_networks = msg.mutable_ip_networks(); + ip_networks->set_ipv4_networks(network); + + network_flows_callback(&msg); + return running; + }) + .WillRepeatedly(ReturnPointee(&running)); + + EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); + + return duplex_writer; + }); + + // Connections/Endpoints returned by the scrapper (first detection of the connection) + EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&conn1](std::vector* connections, std::vector* listen_endpoints) -> bool { + connections->emplace_back(conn1); + return true; + }); + + auto net_status_notifier = MakeUnique(conn_scraper, + conn_tracker, + comm, + config); + + net_status_notifier->Start(); + + // Wait for the first scrape to occur + EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); + + net_status_notifier->Stop(); +} + +} // namespace + +} // namespace collector From f9268d3c186f85e0f40aaffd9962deb2ee4843ea Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 19:15:41 -0700 Subject: [PATCH 26/41] Removed unneeded line --- .github/workflows/integration-tests-vm-type.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests-vm-type.yml b/.github/workflows/integration-tests-vm-type.yml index 70ff9e4795..c7da3db47e 100644 --- a/.github/workflows/integration-tests-vm-type.yml +++ b/.github/workflows/integration-tests-vm-type.yml @@ -117,4 +117,3 @@ jobs: path: | ${{ github.workspace }}/integration-tests/container-logs/**/* ${{ github.workspace }}/integration-tests/performance-logs/**/* - ${{ github.workspace }}/integration-tests/core.out From f645a1d6572ec4b8825476fc599f3de9b77add8b Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Sun, 10 Mar 2024 19:18:31 -0700 Subject: [PATCH 27/41] Removed blank lines --- ansible/roles/run-test-target/tasks/test-collection-method.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 1a18e766af..afb5ef29d4 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -80,7 +80,6 @@ -e COLLECTOR_LOG_LEVEL \ {{ integration_tests_image }} \ {{ run_args }} -test.v 2>&1 - environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" @@ -92,7 +91,6 @@ # ensure that this is printed tags: [print_action] - rescue: - name: Set tests as failed set_fact: From 25eab83a7433e8e714464df36a4497560476e2e1 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 11:01:30 -0700 Subject: [PATCH 28/41] Trying to add test name to core dump path. Adding intentional seg fault back in --- .../tasks/test-collection-method.yml | 25 +- collector/lib/NetworkStatusNotifier.cpp | 2 + collector/test/NetworkStatusNotifierTest.cpp | 339 ------------------ 3 files changed, 23 insertions(+), 343 deletions(-) delete mode 100644 collector/test/NetworkStatusNotifierTest.cpp diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index afb5ef29d4..ebb9c85b2d 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -26,13 +26,27 @@ integration_tests_image: "{{ lookup('env', 'TEST_IMAGE', default='quay.io/rhacs-eng/qa-multi-arch:collector-tests-' + collector_tag_result.stdout) }}" - set_fact: - run_args: -test.run {{ collector_test }} -test.timeout 60m -test.count=1 + tests_to_run: + - collector_test: "TestProcessNetwork" + run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" + - collector_test: "TestImageLabelJSON" + run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" when: collector_test != "ci-integration-tests" - set_fact: - run_args: -test.timeout 120m -test.short + tests_to_run: + - collector_test: "ci-integration-tests" + run_args: "-test.timeout 120m -test.short" when: collector_test == "ci-integration-tests" + #- set_fact: + # run_args: -test.run {{ collector_test }} -test.timeout 60m -test.count=1 + # when: collector_test != "ci-integration-tests" + # + #- set_fact: + # run_args: -test.timeout 120m -test.short + # when: collector_test == "ci-integration-tests" + - name: Make logs directory file: path: "{{ remote_log_mount }}" @@ -65,7 +79,7 @@ - name: Run integration tests become: "{{ runtime_as_root }}" shell: | - echo '/tmp/core.out' | sudo tee /proc/sys/kernel/core_pattern + echo '/tmp/core.{{ collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern {{ runtime_command }} run -it --rm \ -v {{ remote_log_mount }}:{{ container_logs_root }} \ -v ~/.docker:/root/.docker \ @@ -79,7 +93,7 @@ -e COLLECTOR_QA_TAG \ -e COLLECTOR_LOG_LEVEL \ {{ integration_tests_image }} \ - {{ run_args }} -test.v 2>&1 + {{ item.run_args }} -test.v 2>&1 environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" @@ -90,6 +104,9 @@ register: test_result # ensure that this is printed tags: [print_action] + loop: "{{ tests_to_run }}" + loop_control: + loop_var: item rescue: - name: Set tests as failed diff --git a/collector/lib/NetworkStatusNotifier.cpp b/collector/lib/NetworkStatusNotifier.cpp index d944b30b93..fd744cba0a 100644 --- a/collector/lib/NetworkStatusNotifier.cpp +++ b/collector/lib/NetworkStatusNotifier.cpp @@ -313,6 +313,8 @@ void NetworkStatusNotifier::RunSingleAfterglow(IDuplexClientWriter -#include -#include - -#include - -#include "internalapi/sensor/network_connection_iservice.grpc.pb.h" - -#include "CollectorArgs.h" -#include "CollectorConfig.h" -#include "DuplexGRPC.h" -#include "NetworkStatusNotifier.h" -#include "Utility.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -namespace collector { - -namespace { - -using grpc_duplex_impl::Result; -using grpc_duplex_impl::Status; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::ReturnPointee; -using ::testing::UnorderedElementsAre; - -/* Semaphore: A subset of the semaphore feature. - We use it to wait (with a timeout) for the NetworkStatusNotifier service thread to achieve the test scenario. - (C++20 comes with a compatible class (std::counting_semaphore) and we probably want to use this once we - update the C++ version). */ -class Semaphore { - public: - Semaphore(int value = 1) : value_(value) {} - - void release() { - std::unique_lock lock(mutex_); - value_++; - cond_.notify_one(); - } - - void acquire() { - std::unique_lock lock(mutex_); - - while (value_ <= 0) - cond_.wait(lock); - value_--; - } - - template - bool try_acquire_for(const std::chrono::duration& rel_time) { - auto deadline = std::chrono::steady_clock::now() + rel_time; - std::unique_lock lock(mutex_); - - while (value_ <= 0) - if (cond_.wait_until(lock, deadline) == std::cv_status::timeout) - return false; - value_--; - return true; - } - - private: - int value_; - std::condition_variable cond_; - std::mutex mutex_; -}; - -class MockCollectorConfig : public collector::CollectorConfig { - public: - MockCollectorConfig() : collector::CollectorConfig() {} - - void DisableAfterglow() { - enable_afterglow_ = false; - } -}; - -class MockConnScraper : public IConnScraper { - public: - MOCK_METHOD(bool, Scrape, (std::vector * connections, std::vector* listen_endpoints), (override)); -}; - -class MockDuplexClientWriter : public IDuplexClientWriter { - public: - MOCK_METHOD(grpc_duplex_impl::Result, Write, (const sensor::NetworkConnectionInfoMessage& obj, const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WriteAsync, (const sensor::NetworkConnectionInfoMessage& obj), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilStarted, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(bool, Sleep, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WritesDoneAsync, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WritesDone, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, FinishAsync, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, WaitUntilFinished, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, Finish, (grpc::Status * status, const gpr_timespec& deadline), (override)); - MOCK_METHOD(grpc::Status, Finish, (const gpr_timespec& deadline), (override)); - MOCK_METHOD(void, TryCancel, (), (override)); - MOCK_METHOD(grpc_duplex_impl::Result, Shutdown, (), (override)); -}; - -class MockNetworkConnectionInfoServiceComm : public INetworkConnectionInfoServiceComm { - public: - MOCK_METHOD(void, ResetClientContext, (), (override)); - MOCK_METHOD(bool, WaitForConnectionReady, (const std::function& check_interrupted), (override)); - MOCK_METHOD(void, TryCancel, (), (override)); - MOCK_METHOD(sensor::NetworkConnectionInfoService::StubInterface*, GetStub, (), (override)); - MOCK_METHOD(std::unique_ptr>, PushNetworkConnectionInfoOpenStream, (std::function receive_func), (override)); -}; - -/* gRPC payload objects are not strictly the ones of our internal model. - This helper class converts back NetworkConnectionInfoMessage into internal model in order - to compare them with the expected content. */ -class NetworkConnectionInfoMessageParser { - public: - NetworkConnectionInfoMessageParser(const sensor::NetworkConnectionInfoMessage& msg) { - for (auto conn : msg.info().updated_connections()) { - updated_connections_.emplace( - Connection( - conn.container_id(), - Endpoint(IPNetFromProto(conn.local_address()), conn.local_address().port()), - Endpoint(IPNetFromProto(conn.remote_address()), conn.remote_address().port()), - L4ProtocolFromProto(conn.protocol()), - conn.role() == sensor::ROLE_SERVER), - !conn.has_close_timestamp()); - } - } - - /* our internal model for connection life-cycle events uses the Connection object as the key for a hashmap. - The value in the hashmap describes the kind of event: true stands for a creation, false for a deletion. */ - const std::unordered_map& get_updated_connections() const { return updated_connections_; } - - private: - std::unordered_map updated_connections_; - - static IPNet IPNetFromProto(const sensor::NetworkAddress& na) { - Address addr; - const bool is_net = (na.ip_network().length() != 0); - - const std::string& bytes_stream = is_net ? na.ip_network() : na.address_data(); - - switch (bytes_stream.length()) { - case 0: - return IPNet(); - case 4: - case 5: { - uint32_t ip; - memcpy((void*)&ip, bytes_stream.c_str(), 4); - addr = Address(ip); - break; - } - case 16: - case 17: { - uint32_t ip[4]; - memcpy((void*)ip, bytes_stream.c_str(), 16); - addr = Address(ip); - break; - } - default: - throw std::runtime_error("Invalid address length"); - } - - if (is_net) { - const char* prefix_len = bytes_stream.c_str() + (bytes_stream.length() - 1); - - return IPNet(addr, *prefix_len); - } else { - return IPNet(addr); - } - } - - static L4Proto L4ProtocolFromProto(storage::L4Protocol proto) { - switch (proto) { - case storage::L4_PROTOCOL_TCP: - return L4Proto::TCP; - case storage::L4_PROTOCOL_UDP: - return L4Proto::UDP; - case storage::L4_PROTOCOL_ICMP: - return L4Proto::ICMP; - default: - return L4Proto::UNKNOWN; - } - } -}; - -/* Simple validation that the service starts and sends at least one event */ -TEST(NetworkStatusNotifier, SimpleStartStop) { - bool running = true; - CollectorConfig config_; - std::shared_ptr conn_scraper = std::make_shared(); - auto conn_tracker = std::make_shared(); - auto comm = std::make_shared(); - Semaphore sem(0); // to wait for the service to accomplish its job. - - // the connection is always ready - EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); - // gRPC shuts down the loop, so we will want writer->Sleep to return with false - EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); - - /* This is what NetworkStatusNotifier calls to create streams - receive_func is a callback that we can use to simulate messages coming from the sensor - We return an object that will get called when connections and endpoints are reported */ - EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) - .Times(1) - .WillOnce([&sem, &running](std::function receive_func) -> std::unique_ptr> { - auto duplex_writer = MakeUnique(); - - // the service is sending Sensor a message - EXPECT_CALL(*duplex_writer, Write).WillRepeatedly([&sem, &running](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - for (auto cnx : msg.info().updated_connections()) { - std::cout << cnx.container_id() << std::endl; - } - sem.release(); // notify that the test should end - return Result(Status::OK); - }); - EXPECT_CALL(*duplex_writer, Sleep).WillRepeatedly(ReturnPointee(&running)); - EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); - - return duplex_writer; - }); - - // Connections/Endpoints returned by the scrapper - EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&sem](std::vector* connections, std::vector* listen_endpoints) -> bool { - // this is the data which will trigger NetworkStatusNotifier to create a connection event (purpose of the test) - connections->emplace_back("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); - return true; - }); - - auto net_status_notifier = MakeUnique(conn_scraper, - conn_tracker, - comm, - config_); - - net_status_notifier->Start(); - - EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); - - net_status_notifier->Stop(); -} - -/* This test checks whether deltas are computed appropriately in case the "known network" list is received after a connection - is already reported (and matches one of the networks). - - scrapper initialy reports a connection - - the connection is reported - - we feed a "known network" into the NetworkStatusNotifier - - we check that the former declared connection is deleted and redeclared as part of this network */ -TEST(NetworkStatusNotifier, UpdateIPnoAfterglow) { - bool running = true; - MockCollectorConfig config; - config.DisableAfterglow(); - std::shared_ptr conn_scraper = std::make_shared(); - auto conn_tracker = std::make_shared(); - auto comm = std::make_shared(); - std::function network_flows_callback; - Semaphore sem(0); // to wait for the service to accomplish its job. - - // the connection as scrapped (public) - Connection conn1("containerId", Endpoint(Address(10, 0, 1, 32), 1024), Endpoint(Address(139, 45, 27, 4), 999), L4Proto::TCP, true); - // the same server connection normalized - Connection conn2("containerId", Endpoint(Address(), 1024), Endpoint(Address(255, 255, 255, 255), 0), L4Proto::TCP, true); - // the same server connection normalized and grouped in a known subnet - Connection conn3("containerId", Endpoint(Address(), 1024), Endpoint(IPNet(Address(139, 45, 0, 0), 16), 0), L4Proto::TCP, true); - - // the connection is always ready - EXPECT_CALL(*comm, WaitForConnectionReady).WillRepeatedly(Return(true)); - // gRPC shuts down the loop, so we will want writer->Sleep to return with false - EXPECT_CALL(*comm, TryCancel).Times(1).WillOnce([&running] { running = false; }); - - /* This is what NetworkStatusNotifier calls to create streams - receive_func is a callback that we can use to simulate messages coming from the sensor - We return an object that will get called when connections and endpoints are reported */ - EXPECT_CALL(*comm, PushNetworkConnectionInfoOpenStream) - .Times(1) - .WillOnce([&sem, - &running, - &conn2, - &conn3, - &network_flows_callback](std::function receive_func) -> std::unique_ptr> { - auto duplex_writer = MakeUnique(); - network_flows_callback = receive_func; - - // the service is sending Sensor a message - EXPECT_CALL(*duplex_writer, Write) - .WillOnce([&conn2, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - // the connection reported by the scrapper is annouced as generic public - EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn2, true))); - return Result(Status::OK); - }) - .WillOnce([&conn2, &conn3, &sem](const sensor::NetworkConnectionInfoMessage& msg, const gpr_timespec& deadline) -> Result { - // after the network is declared, the connection switches to the new state - // conn3 appears and conn2 is destroyed - EXPECT_THAT(NetworkConnectionInfoMessageParser(msg).get_updated_connections(), UnorderedElementsAre(std::make_pair(conn3, true), std::make_pair(conn2, false))); - - // Done - sem.release(); - - return Result(Status::OK); - }) - .WillRepeatedly(Return(Result(Status::OK))); - - EXPECT_CALL(*duplex_writer, Sleep) - .WillOnce(ReturnPointee(&running)) // first time, we let the scrapper do its job - .WillOnce([&running, &network_flows_callback](const gpr_timespec& deadline) { - // The connection is known now, let's declare a "known network" - sensor::NetworkFlowsControlMessage msg; - unsigned char content[] = {139, 45, 0, 0, 16}; // address in network order, plus prefix length - std::string network((char*)content, sizeof(content)); - - auto ip_networks = msg.mutable_ip_networks(); - ip_networks->set_ipv4_networks(network); - - network_flows_callback(&msg); - return running; - }) - .WillRepeatedly(ReturnPointee(&running)); - - EXPECT_CALL(*duplex_writer, WaitUntilStarted).WillRepeatedly(Return(Result(Status::OK))); - - return duplex_writer; - }); - - // Connections/Endpoints returned by the scrapper (first detection of the connection) - EXPECT_CALL(*conn_scraper, Scrape).WillRepeatedly([&conn1](std::vector* connections, std::vector* listen_endpoints) -> bool { - connections->emplace_back(conn1); - return true; - }); - - auto net_status_notifier = MakeUnique(conn_scraper, - conn_tracker, - comm, - config); - - net_status_notifier->Start(); - - // Wait for the first scrape to occur - EXPECT_TRUE(sem.try_acquire_for(std::chrono::seconds(5))); - - net_status_notifier->Stop(); -} - -} // namespace - -} // namespace collector From 9e7f634d36cd355688634438a7de44f4dd789133 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 11:37:01 -0700 Subject: [PATCH 29/41] Fixed indentation --- .../tasks/test-collection-method.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index ebb9c85b2d..da95446d4e 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -39,13 +39,6 @@ run_args: "-test.timeout 120m -test.short" when: collector_test == "ci-integration-tests" - #- set_fact: - # run_args: -test.run {{ collector_test }} -test.timeout 60m -test.count=1 - # when: collector_test != "ci-integration-tests" - # - #- set_fact: - # run_args: -test.timeout 120m -test.short - # when: collector_test == "ci-integration-tests" - name: Make logs directory file: @@ -104,9 +97,9 @@ register: test_result # ensure that this is printed tags: [print_action] - loop: "{{ tests_to_run }}" - loop_control: - loop_var: item + loop: "{{ tests_to_run }}" + loop_control: + loop_var: item rescue: - name: Set tests as failed From 9144e14b0b4a68dcec0fec85bbab15882c853f73 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 14:14:17 -0700 Subject: [PATCH 30/41] Trying to fix Write integration test log --- .../run-test-target/tasks/test-collection-method.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index da95446d4e..b030e0b444 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -134,10 +134,16 @@ - name: Write integration test log copy: - content: "{{ test_result.stdout }}" + content: "{{ test_result.results | map(attribute='stdout') | join('\n') }}" dest: "{{ logs_root }}/integration-test.log" delegate_to: localhost + # - name: Write integration test log + # copy: + # content: "{{ test_result.stdout }}" + # dest: "{{ logs_root }}/integration-test.log" + # delegate_to: localhost + - name: Move core dump files to core dump directory become: yes shell: | From 4c5c41aaefca0f7c70e8f6fa41f64f26efd08220 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 15:03:20 -0700 Subject: [PATCH 31/41] Not using item for loop_var --- .../roles/run-test-target/tasks/test-collection-method.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index b030e0b444..4c9ed18e4b 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -72,7 +72,7 @@ - name: Run integration tests become: "{{ runtime_as_root }}" shell: | - echo '/tmp/core.{{ collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern + echo '/tmp/core.{{ test_to_run.collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern {{ runtime_command }} run -it --rm \ -v {{ remote_log_mount }}:{{ container_logs_root }} \ -v ~/.docker:/root/.docker \ @@ -86,7 +86,7 @@ -e COLLECTOR_QA_TAG \ -e COLLECTOR_LOG_LEVEL \ {{ integration_tests_image }} \ - {{ item.run_args }} -test.v 2>&1 + {{ test_to_run.run_args }} -test.v 2>&1 environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" @@ -99,7 +99,7 @@ tags: [print_action] loop: "{{ tests_to_run }}" loop_control: - loop_var: item + loop_var: test_to_run rescue: - name: Set tests as failed From 9f1e34539e0bee6b8b4e7f9283ad8a9cb6c56957 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 17:10:24 -0700 Subject: [PATCH 32/41] Fixed Output test log. Core dumps are written directly to /tmp/core_dumps --- .../tasks/test-collection-method.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 4c9ed18e4b..96b578180c 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -57,6 +57,11 @@ creates: ~/.docker/config.json when: runtime_command == "podman" +- name: Make core dumps directory + become: yes + shell: | + mkdir /tmp/core_dumps + - block: # Some considerations for running the test container: @@ -72,7 +77,7 @@ - name: Run integration tests become: "{{ runtime_as_root }}" shell: | - echo '/tmp/core.{{ test_to_run.collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern + echo '/tmp/core_dumps/core-{{ test_to_run.collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern {{ runtime_command }} run -it --rm \ -v {{ remote_log_mount }}:{{ container_logs_root }} \ -v ~/.docker:/root/.docker \ @@ -108,9 +113,14 @@ - name: Output test log debug: - msg: "{{ test_result.stdout }}" + msg: "{{ test_result.results | map(attribute='stdout') | join('\n') }}" tags: [print_action] + #- name: Output test log + # debug: + # msg: "{{ test_result.stdout }}" + # tags: [print_action] + always: - name: Make logs directories file: @@ -144,11 +154,6 @@ # dest: "{{ logs_root }}/integration-test.log" # delegate_to: localhost - - name: Move core dump files to core dump directory - become: yes - shell: | - mkdir /tmp/core_dumps - mv /tmp/core.out* /tmp/core_dumps || true - name: Get core dump file paths become: yes From 2d73cd3282390041ddc5f90c4ca0dc438f881cf0 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 18:37:08 -0700 Subject: [PATCH 33/41] Should run two tests one at a time --- .../tasks/test-collection-method.yml | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 96b578180c..5392b93f3a 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -27,19 +27,18 @@ - set_fact: tests_to_run: - - collector_test: "TestProcessNetwork" - run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" - - collector_test: "TestImageLabelJSON" - run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" + - collector_test: "{{ collector_test }}" + run_args: "-test.run {{ collector_test }} -test.timeout 120m -test.short" when: collector_test != "ci-integration-tests" - set_fact: tests_to_run: - - collector_test: "ci-integration-tests" - run_args: "-test.timeout 120m -test.short" + - collector_test: "TestProcessNetwork" + run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" + - collector_test: "TestImageLabelJSON" + run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" when: collector_test == "ci-integration-tests" - - name: Make logs directory file: path: "{{ remote_log_mount }}" @@ -148,13 +147,6 @@ dest: "{{ logs_root }}/integration-test.log" delegate_to: localhost - # - name: Write integration test log - # copy: - # content: "{{ test_result.stdout }}" - # dest: "{{ logs_root }}/integration-test.log" - # delegate_to: localhost - - - name: Get core dump file paths become: yes find: From 81e83749835782ff75775adc7108c89ef0be6f69 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 19:53:35 -0700 Subject: [PATCH 34/41] Check if directory exists before creating it --- .../roles/run-test-target/tasks/test-collection-method.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 5392b93f3a..a2a35f13b4 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -59,7 +59,9 @@ - name: Make core dumps directory become: yes shell: | - mkdir /tmp/core_dumps + if [[ ! -d "/tmp/core_dumps" ]]; then + mkdir /tmp/core_dumps + fi - block: From 9126289d335629136839f2e4cbe7a9ec9b74af00 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Mon, 11 Mar 2024 22:50:03 -0700 Subject: [PATCH 35/41] Going through all of the integration tests. Not using an if statement to check if core_dumps directory exists --- .../tasks/test-collection-method.yml | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index a2a35f13b4..64f5ea4e45 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -37,6 +37,48 @@ run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" - collector_test: "TestImageLabelJSON" run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" + - collector_test: "TestMissingProcScrape" + run_args: "-test.run TestMissingProcScrape -test.timeout 60m -test.count=1" + - collector_test: "TestRepeatedNetworkFlow" + run_args: "-test.run TestRepeatedNetworkFlow -test.timeout 60m -test.count=1" + - collector_test: "TestProcfsScraper" + run_args: "-test.run TestProcfsScraper -test.timeout 60m -test.count=1" + - collector_test: "TestProcessListeningOnPort" + run_args: "-test.run TestProcessListeningOnPort -test.timeout 60m -test.count=1" + - collector_test: "TestSymbolicLinkProcess" + run_args: "-test.run TestSymbolicLinkProcess -test.timeout 60m -test.count=1" + - collector_test: "TestSocat" + run_args: "-test.run TestSocat -test.timeout 60m -test.count=1" + - collector_test: "TestDuplicateEndpoints" + run_args: "-test.run TestDuplicateEndpoints -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsNormal" + run_args: "-test.run TestConnectionsAndEndpointsNormal -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsHighLowPorts" + run_args: "-test.run TestConnectionsAndEndpointsHighLowPorts -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsServerHigh" + run_args: "-test.run TestConnectionsAndEndpointsServerHigh -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsSourcePort" + run_args: "-test.run TestConnectionsAndEndpointsSourcePort -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsUDPNormal" + run_args: "-test.run TestConnectionsAndEndpointsUDPNormal -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsUDPNoReuseaddr" + run_args: "-test.run TestConnectionsAndEndpointsUDPNoReuseaddr -test.timeout 60m -test.count=1" + - collector_test: "TestConnectionsAndEndpointsUDPNoFork" + run_args: "-test.run TestConnectionsAndEndpointsUDPNoFork -test.timeout 60m -test.count=1" + - collector_test: "TestAsyncConnectionBlocked" + run_args: "-test.run TestAsyncConnectionBlocked -test.timeout 60m -test.count=1" + - collector_test: "TestAsyncConnectionSuccess" + run_args: "-test.run TestAsyncConnectionSuccess -test.timeout 60m -test.count=1" + - collector_test: "TestAsyncConnectionBlockedWithDisableTracking" + run_args: "-test.run TestAsyncConnectionBlockedWithDisableTracking -test.timeout 60m -test.count=1" + - collector_test: "TestAsyncConnectionSuccessWithDisableTracking" + run_args: "-test.run TestAsyncConnectionSuccessWithDisableTracking -test.timeout 60m -test.count=1" + - collector_test: "TestCollectorStartup" + run_args: "-test.run TestCollectorStartup -test.timeout 60m -test.count=1" + - collector_test: "TestPerfEvent" + run_args: "-test.run TestPerfEvent -test.timeout 60m -test.count=1" + - collector_test: "TestGperftools" + run_args: "-test.run TestGperftools -test.timeout 60m -test.count=1" when: collector_test == "ci-integration-tests" - name: Make logs directory @@ -59,9 +101,10 @@ - name: Make core dumps directory become: yes shell: | - if [[ ! -d "/tmp/core_dumps" ]]; then - mkdir /tmp/core_dumps - fi + mkdir /tmp/core_dumps || true + #if [[ ! -d "/tmp/core_dumps" ]]; then + # mkdir /tmp/core_dumps + #fi - block: From 63cebb61357afbc4f9c2a3d4dabbb1e606e0b29f Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 15:46:07 -0700 Subject: [PATCH 36/41] Trying to get a list of tests instead of hard coding them --- .../tasks/test-collection-method.yml | 133 +++++++++++------- 1 file changed, 81 insertions(+), 52 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 64f5ea4e45..9f00316907 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -25,61 +25,90 @@ collector_image: "{{ lookup('env', 'COLLECTOR_IMAGE', default='quay.io/rhacs-eng/collector:' + collector_tag_result.stdout) }}" integration_tests_image: "{{ lookup('env', 'TEST_IMAGE', default='quay.io/rhacs-eng/qa-multi-arch:collector-tests-' + collector_tag_result.stdout) }}" +- name: Get list of tests + shell: | + cd {{ collector_root }}/integration-tests + go test -list Test -test.run=^$ + register: test_list + changed_when: false + + #- set_fact: + # tests_to_run: + # - collector_test: "{{ collector_test }}" + # run_args: "-test.run {{ collector_test }} -test.timeout 120m -test.short" + # when: collector_test != "ci-integration-tests" + - set_fact: tests_to_run: - - collector_test: "{{ collector_test }}" - run_args: "-test.run {{ collector_test }} -test.timeout 120m -test.short" + - "{{ collector_test }}" when: collector_test != "ci-integration-tests" - set_fact: - tests_to_run: - - collector_test: "TestProcessNetwork" - run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" - - collector_test: "TestImageLabelJSON" - run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" - - collector_test: "TestMissingProcScrape" - run_args: "-test.run TestMissingProcScrape -test.timeout 60m -test.count=1" - - collector_test: "TestRepeatedNetworkFlow" - run_args: "-test.run TestRepeatedNetworkFlow -test.timeout 60m -test.count=1" - - collector_test: "TestProcfsScraper" - run_args: "-test.run TestProcfsScraper -test.timeout 60m -test.count=1" - - collector_test: "TestProcessListeningOnPort" - run_args: "-test.run TestProcessListeningOnPort -test.timeout 60m -test.count=1" - - collector_test: "TestSymbolicLinkProcess" - run_args: "-test.run TestSymbolicLinkProcess -test.timeout 60m -test.count=1" - - collector_test: "TestSocat" - run_args: "-test.run TestSocat -test.timeout 60m -test.count=1" - - collector_test: "TestDuplicateEndpoints" - run_args: "-test.run TestDuplicateEndpoints -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsNormal" - run_args: "-test.run TestConnectionsAndEndpointsNormal -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsHighLowPorts" - run_args: "-test.run TestConnectionsAndEndpointsHighLowPorts -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsServerHigh" - run_args: "-test.run TestConnectionsAndEndpointsServerHigh -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsSourcePort" - run_args: "-test.run TestConnectionsAndEndpointsSourcePort -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsUDPNormal" - run_args: "-test.run TestConnectionsAndEndpointsUDPNormal -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsUDPNoReuseaddr" - run_args: "-test.run TestConnectionsAndEndpointsUDPNoReuseaddr -test.timeout 60m -test.count=1" - - collector_test: "TestConnectionsAndEndpointsUDPNoFork" - run_args: "-test.run TestConnectionsAndEndpointsUDPNoFork -test.timeout 60m -test.count=1" - - collector_test: "TestAsyncConnectionBlocked" - run_args: "-test.run TestAsyncConnectionBlocked -test.timeout 60m -test.count=1" - - collector_test: "TestAsyncConnectionSuccess" - run_args: "-test.run TestAsyncConnectionSuccess -test.timeout 60m -test.count=1" - - collector_test: "TestAsyncConnectionBlockedWithDisableTracking" - run_args: "-test.run TestAsyncConnectionBlockedWithDisableTracking -test.timeout 60m -test.count=1" - - collector_test: "TestAsyncConnectionSuccessWithDisableTracking" - run_args: "-test.run TestAsyncConnectionSuccessWithDisableTracking -test.timeout 60m -test.count=1" - - collector_test: "TestCollectorStartup" - run_args: "-test.run TestCollectorStartup -test.timeout 60m -test.count=1" - - collector_test: "TestPerfEvent" - run_args: "-test.run TestPerfEvent -test.timeout 60m -test.count=1" - - collector_test: "TestGperftools" - run_args: "-test.run TestGperftools -test.timeout 60m -test.count=1" - when: collector_test == "ci-integration-tests" + tests_to_run: "{{ test_list.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" + when: collector_test == "ci-integration-tests" + + #- set_fact: + # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | map('regex_replace', '^(Test\\w+)', {'collector_test': '\\1', 'run_args': '-test.run \\1 -test.timeout 60m -test.count=1'}) | list }}" + # when: collector_test == "ci-integration-tests" + + + #- set_fact: + # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" + # when: collector_test == "ci-integration-tests" + + #- set_fact: + # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" + # when: collector_test == "ci-integration-tests" + + #- set_fact: + # tests_to_run: + # - collector_test: "TestProcessNetwork" + # run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" + # - collector_test: "TestImageLabelJSON" + # run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" + # - collector_test: "TestMissingProcScrape" + # run_args: "-test.run TestMissingProcScrape -test.timeout 60m -test.count=1" + # - collector_test: "TestRepeatedNetworkFlow" + # run_args: "-test.run TestRepeatedNetworkFlow -test.timeout 60m -test.count=1" + # - collector_test: "TestProcfsScraper" + # run_args: "-test.run TestProcfsScraper -test.timeout 60m -test.count=1" + # - collector_test: "TestProcessListeningOnPort" + # run_args: "-test.run TestProcessListeningOnPort -test.timeout 60m -test.count=1" + # - collector_test: "TestSymbolicLinkProcess" + # run_args: "-test.run TestSymbolicLinkProcess -test.timeout 60m -test.count=1" + # - collector_test: "TestSocat" + # run_args: "-test.run TestSocat -test.timeout 60m -test.count=1" + # - collector_test: "TestDuplicateEndpoints" + # run_args: "-test.run TestDuplicateEndpoints -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsNormal" + # run_args: "-test.run TestConnectionsAndEndpointsNormal -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsHighLowPorts" + # run_args: "-test.run TestConnectionsAndEndpointsHighLowPorts -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsServerHigh" + # run_args: "-test.run TestConnectionsAndEndpointsServerHigh -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsSourcePort" + # run_args: "-test.run TestConnectionsAndEndpointsSourcePort -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsUDPNormal" + # run_args: "-test.run TestConnectionsAndEndpointsUDPNormal -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsUDPNoReuseaddr" + # run_args: "-test.run TestConnectionsAndEndpointsUDPNoReuseaddr -test.timeout 60m -test.count=1" + # - collector_test: "TestConnectionsAndEndpointsUDPNoFork" + # run_args: "-test.run TestConnectionsAndEndpointsUDPNoFork -test.timeout 60m -test.count=1" + # - collector_test: "TestAsyncConnectionBlocked" + # run_args: "-test.run TestAsyncConnectionBlocked -test.timeout 60m -test.count=1" + # - collector_test: "TestAsyncConnectionSuccess" + # run_args: "-test.run TestAsyncConnectionSuccess -test.timeout 60m -test.count=1" + # - collector_test: "TestAsyncConnectionBlockedWithDisableTracking" + # run_args: "-test.run TestAsyncConnectionBlockedWithDisableTracking -test.timeout 60m -test.count=1" + # - collector_test: "TestAsyncConnectionSuccessWithDisableTracking" + # run_args: "-test.run TestAsyncConnectionSuccessWithDisableTracking -test.timeout 60m -test.count=1" + # - collector_test: "TestCollectorStartup" + # run_args: "-test.run TestCollectorStartup -test.timeout 60m -test.count=1" + # - collector_test: "TestPerfEvent" + # run_args: "-test.run TestPerfEvent -test.timeout 60m -test.count=1" + # - collector_test: "TestGperftools" + # run_args: "-test.run TestGperftools -test.timeout 60m -test.count=1" + # when: collector_test == "ci-integration-tests" - name: Make logs directory file: @@ -121,7 +150,7 @@ - name: Run integration tests become: "{{ runtime_as_root }}" shell: | - echo '/tmp/core_dumps/core-{{ test_to_run.collector_test }}.out' | sudo tee /proc/sys/kernel/core_pattern + echo '/tmp/core_dumps/core-{{ test_to_run }}.out' | sudo tee /proc/sys/kernel/core_pattern {{ runtime_command }} run -it --rm \ -v {{ remote_log_mount }}:{{ container_logs_root }} \ -v ~/.docker:/root/.docker \ @@ -135,7 +164,7 @@ -e COLLECTOR_QA_TAG \ -e COLLECTOR_LOG_LEVEL \ {{ integration_tests_image }} \ - {{ test_to_run.run_args }} -test.v 2>&1 + -test.run {{ test_to_run }} -test.timeout 60m -test.count=1 -test.v 2>&1 environment: COLLECTION_METHOD: "{{ collection_method }}" VM_CONFIG: "{{ vm_config }}" From b3ea43025a96e0a56d7c072ecb970f83b4229071 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 17:18:04 -0700 Subject: [PATCH 37/41] Cleaned up --- .../tasks/test-collection-method.yml | 77 ------------------- 1 file changed, 77 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 9f00316907..21a6b68e9f 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -32,12 +32,6 @@ register: test_list changed_when: false - #- set_fact: - # tests_to_run: - # - collector_test: "{{ collector_test }}" - # run_args: "-test.run {{ collector_test }} -test.timeout 120m -test.short" - # when: collector_test != "ci-integration-tests" - - set_fact: tests_to_run: - "{{ collector_test }}" @@ -47,69 +41,6 @@ tests_to_run: "{{ test_list.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" when: collector_test == "ci-integration-tests" - #- set_fact: - # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | map('regex_replace', '^(Test\\w+)', {'collector_test': '\\1', 'run_args': '-test.run \\1 -test.timeout 60m -test.count=1'}) | list }}" - # when: collector_test == "ci-integration-tests" - - - #- set_fact: - # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" - # when: collector_test == "ci-integration-tests" - - #- set_fact: - # tests_to_run: "{{ test_output.stdout_lines | select('regex', '^(Test\\w+)$') | map('regex_replace', '^(Test\\w+)$', '\\1') | list }}" - # when: collector_test == "ci-integration-tests" - - #- set_fact: - # tests_to_run: - # - collector_test: "TestProcessNetwork" - # run_args: "-test.run TestProcessNetwork -test.timeout 60m -test.count=1" - # - collector_test: "TestImageLabelJSON" - # run_args: "-test.run TestImageLabelJSON -test.timeout 60m -test.count=1" - # - collector_test: "TestMissingProcScrape" - # run_args: "-test.run TestMissingProcScrape -test.timeout 60m -test.count=1" - # - collector_test: "TestRepeatedNetworkFlow" - # run_args: "-test.run TestRepeatedNetworkFlow -test.timeout 60m -test.count=1" - # - collector_test: "TestProcfsScraper" - # run_args: "-test.run TestProcfsScraper -test.timeout 60m -test.count=1" - # - collector_test: "TestProcessListeningOnPort" - # run_args: "-test.run TestProcessListeningOnPort -test.timeout 60m -test.count=1" - # - collector_test: "TestSymbolicLinkProcess" - # run_args: "-test.run TestSymbolicLinkProcess -test.timeout 60m -test.count=1" - # - collector_test: "TestSocat" - # run_args: "-test.run TestSocat -test.timeout 60m -test.count=1" - # - collector_test: "TestDuplicateEndpoints" - # run_args: "-test.run TestDuplicateEndpoints -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsNormal" - # run_args: "-test.run TestConnectionsAndEndpointsNormal -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsHighLowPorts" - # run_args: "-test.run TestConnectionsAndEndpointsHighLowPorts -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsServerHigh" - # run_args: "-test.run TestConnectionsAndEndpointsServerHigh -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsSourcePort" - # run_args: "-test.run TestConnectionsAndEndpointsSourcePort -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsUDPNormal" - # run_args: "-test.run TestConnectionsAndEndpointsUDPNormal -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsUDPNoReuseaddr" - # run_args: "-test.run TestConnectionsAndEndpointsUDPNoReuseaddr -test.timeout 60m -test.count=1" - # - collector_test: "TestConnectionsAndEndpointsUDPNoFork" - # run_args: "-test.run TestConnectionsAndEndpointsUDPNoFork -test.timeout 60m -test.count=1" - # - collector_test: "TestAsyncConnectionBlocked" - # run_args: "-test.run TestAsyncConnectionBlocked -test.timeout 60m -test.count=1" - # - collector_test: "TestAsyncConnectionSuccess" - # run_args: "-test.run TestAsyncConnectionSuccess -test.timeout 60m -test.count=1" - # - collector_test: "TestAsyncConnectionBlockedWithDisableTracking" - # run_args: "-test.run TestAsyncConnectionBlockedWithDisableTracking -test.timeout 60m -test.count=1" - # - collector_test: "TestAsyncConnectionSuccessWithDisableTracking" - # run_args: "-test.run TestAsyncConnectionSuccessWithDisableTracking -test.timeout 60m -test.count=1" - # - collector_test: "TestCollectorStartup" - # run_args: "-test.run TestCollectorStartup -test.timeout 60m -test.count=1" - # - collector_test: "TestPerfEvent" - # run_args: "-test.run TestPerfEvent -test.timeout 60m -test.count=1" - # - collector_test: "TestGperftools" - # run_args: "-test.run TestGperftools -test.timeout 60m -test.count=1" - # when: collector_test == "ci-integration-tests" - - name: Make logs directory file: path: "{{ remote_log_mount }}" @@ -131,9 +62,6 @@ become: yes shell: | mkdir /tmp/core_dumps || true - #if [[ ! -d "/tmp/core_dumps" ]]; then - # mkdir /tmp/core_dumps - #fi - block: @@ -189,11 +117,6 @@ msg: "{{ test_result.results | map(attribute='stdout') | join('\n') }}" tags: [print_action] - #- name: Output test log - # debug: - # msg: "{{ test_result.stdout }}" - # tags: [print_action] - always: - name: Make logs directories file: From 6bb231adfac23117e71043beb4a2068f28b6722c Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 17:23:51 -0700 Subject: [PATCH 38/41] X-Smart-Branch-Parent: jv-ROX-22866-save-core-dumps-to-collector-ci-artifacts From 4111253132a534fe5643af97776036d70865a01e Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 17:58:43 -0700 Subject: [PATCH 39/41] Using grep instead of go to get integration tests --- .../roles/run-test-target/tasks/test-collection-method.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index 21a6b68e9f..a3b5d4112f 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -26,9 +26,11 @@ integration_tests_image: "{{ lookup('env', 'TEST_IMAGE', default='quay.io/rhacs-eng/qa-multi-arch:collector-tests-' + collector_tag_result.stdout) }}" - name: Get list of tests + become: "{{ runtime_as_root }}" shell: | cd {{ collector_root }}/integration-tests - go test -list Test -test.run=^$ + grep func\ Test integration_test.go | awk '{print $2}' | sed 's|(.*||' + #go test -list Test -test.run=^$ register: test_list changed_when: false From 3d211298ba361d4854c2cad87cc55723acd7ff85 Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 20:24:40 -0700 Subject: [PATCH 40/41] Added debugging --- .../roles/run-test-target/tasks/test-collection-method.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index a3b5d4112f..d5af4c0df5 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -34,6 +34,10 @@ register: test_list changed_when: false +- name: Output list of tests + debug: + msg: "{{ test_list.stdout_lines }}" + - set_fact: tests_to_run: - "{{ collector_test }}" From d7376d696d8f511c618b0d2cceda47844f6f618c Mon Sep 17 00:00:00 2001 From: JoukoVirtanen Date: Tue, 12 Mar 2024 21:13:47 -0700 Subject: [PATCH 41/41] More debugging --- .../tasks/test-collection-method.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ansible/roles/run-test-target/tasks/test-collection-method.yml b/ansible/roles/run-test-target/tasks/test-collection-method.yml index d5af4c0df5..9e9e1da516 100644 --- a/ansible/roles/run-test-target/tasks/test-collection-method.yml +++ b/ansible/roles/run-test-target/tasks/test-collection-method.yml @@ -29,15 +29,34 @@ become: "{{ runtime_as_root }}" shell: | cd {{ collector_root }}/integration-tests - grep func\ Test integration_test.go | awk '{print $2}' | sed 's|(.*||' + grep "func Test" integration_test.go | awk '{print $2}' | sed 's|(.*||' + #grep func\ Test integration_test.go | awk '{print $2}' | sed 's|(.*||' #go test -list Test -test.run=^$ register: test_list - changed_when: false + +- name: Get contents of integration_test.go + become: "{{ runtime_as_root }}" + shell: | + cd {{ collector_root }}/integration-tests + cat integration_test.go + register: test_contents + +- name: Output test contents + debug: + msg: "{{ test_contents.stdout_lines }}" + +- name: Output test contents 2 + debug: + msg: "{{ test_contents.stdout }}" - name: Output list of tests debug: msg: "{{ test_list.stdout_lines }}" +- name: Output list of tests 2 + debug: + msg: "{{ test_list.stdout }}" + - set_fact: tests_to_run: - "{{ collector_test }}"