Implement setup recovery to encapsulate critical cleanup

integration-tests/suites/async_connections.go

@@ -30,6 +30,7 @@ type AsyncConnectionTestSuite struct {
  *     a dummy address.
  */
 func (s *AsyncConnectionTestSuite) SetupSuite() {
+	defer s.RecoverSetup("server", "client")
 	s.StartContainerStats()
 
 	collector := s.Collector()

integration-tests/suites/base.go

@@ -27,6 +27,8 @@ const (
 	parentUIDStr             = "ParentUid"
 	parentExecFilePathStr    = "ParentExecFilePath"
 
+	containerStatsName = "container-stats"
+
 	defaultWaitTickSeconds = 30 * time.Second
 )
 
@@ -130,8 +132,21 @@ func (s *IntegrationTestSuiteBase) SetSelinuxPermissiveIfNeeded() error {
 	return nil
 }
 
+// RecoverSetup wraps common recovery functionality that tests can use
+// in cases where SetupSuite may fail (panic). It should be deferred:
+//
+// defer s.RecoverSetup("nginx", "nginx-curl")
+func (s *IntegrationTestSuiteBase) RecoverSetup(containers ...string) {
+	if r := recover(); r != nil {
+		containers = append(containers, containerStatsName)
+		s.cleanupContainer(containers)
+		s.StopCollector()
+		panic(r)
+	}
+}
+
 func (s *IntegrationTestSuiteBase) GetContainerStats() (stats []ContainerStat) {
-	logs, err := s.containerLogs("container-stats")
+	logs, err := s.containerLogs(containerStatsName)
 	if err != nil {
 		assert.FailNow(s.T(), "container-stats failure")
 		return nil
@@ -142,7 +157,7 @@ func (s *IntegrationTestSuiteBase) GetContainerStats() (stats []ContainerStat) {
 		json.Unmarshal([]byte(line), &stat)
 		stats = append(stats, stat)
 	}
-	s.cleanupContainer([]string{"container-stats"})
+	s.cleanupContainer([]string{containerStatsName})
 	return stats
 }
 
@@ -355,9 +370,8 @@ func (s *IntegrationTestSuiteBase) RunImageWithJSONLabels() {
 }
 
 func (s *IntegrationTestSuiteBase) StartContainerStats() {
-	name := "container-stats"
 	image := config.Images().QaImageByKey("performance-stats")
-	args := []string{name, "-v", common.RuntimeSocket + ":/var/run/docker.sock", image}
+	args := []string{containerStatsName, "-v", common.RuntimeSocket + ":/var/run/docker.sock", image}
 
 	err := s.Executor().PullImage(image)
 	s.Require().NoError(err)

integration-tests/suites/benchmark.go

@@ -128,6 +128,7 @@ func (b *BenchmarkTestSuiteBase) StopPerfTools() {
 }
 
 func (s *BenchmarkCollectorTestSuite) SetupSuite() {
+	defer s.RecoverSetup("perf", "bcc", "bpftrace", "init")
 	s.StartContainerStats()
 
 	s.StartPerfTools()

integration-tests/suites/collector_startup.go

@@ -5,6 +5,7 @@ type CollectorStartupTestSuite struct {
 }
 
 func (s *CollectorStartupTestSuite) SetupSuite() {
+	defer s.RecoverSetup()
 	s.StartCollector(false)
 }
 

integration-tests/suites/connections_and_endpoints.go

@@ -29,6 +29,7 @@ type ConnectionsAndEndpointsTestSuite struct {
 }
 
 func (s *ConnectionsAndEndpointsTestSuite) SetupSuite() {
+	defer s.RecoverSetup(s.Server.Name, s.Client.Name)
 	s.StartContainerStats()
 	collector := s.Collector()
 

integration-tests/suites/duplicate_endpoints.go

@@ -31,6 +31,7 @@ func (s *DuplicateEndpointsTestSuite) killSocatProcess(port int) {
 }
 
 func (s *DuplicateEndpointsTestSuite) SetupSuite() {
+	defer s.RecoverSetup("socat")
 	s.StartContainerStats()
 
 	collector := s.Collector()

integration-tests/suites/image_json.go

@@ -5,6 +5,7 @@ type ImageLabelJSONTestSuite struct {
 }
 
 func (s *ImageLabelJSONTestSuite) SetupSuite() {
+	defer s.RecoverSetup()
 	s.StartCollector(false)
 }
 
@@ -14,5 +15,5 @@ func (s *ImageLabelJSONTestSuite) TestRunImageWithJSONLabel() {
 
 func (s *ImageLabelJSONTestSuite) TearDownSuite() {
 	s.StopCollector()
-	s.cleanupContainer([]string{"collector", "grpc-server", "jsonlabel"})
+	s.cleanupContainer([]string{"grpc-server", "jsonlabel"})
 }

integration-tests/suites/listening_ports.go

@@ -16,6 +16,7 @@ type ProcessListeningOnPortTestSuite struct {
 }
 
 func (s *ProcessListeningOnPortTestSuite) SetupSuite() {
+	defer s.RecoverSetup("process-ports")
 	s.StartContainerStats()
 
 	collector := s.Collector()
@@ -56,7 +57,7 @@ func (s *ProcessListeningOnPortTestSuite) SetupSuite() {
 
 func (s *ProcessListeningOnPortTestSuite) TearDownSuite() {
 	s.StopCollector()
-	s.cleanupContainer([]string{"process-ports", "collector"})
+	s.cleanupContainer([]string{"process-ports"})
 	stats := s.GetContainerStats()
 	s.PrintContainerStats(stats)
 	s.WritePerfResults("ProcessListeningOnPort", stats, s.metrics)

integration-tests/suites/missing_proc_scrape.go

@@ -13,6 +13,8 @@ type MissingProcScrapeTestSuite struct {
 const fakeProcDir = "/tmp/fake-proc"
 
 func (s *MissingProcScrapeTestSuite) SetupSuite() {
+	defer s.RecoverSetup()
+
 	_, err := os.Stat(fakeProcDir)
 	if os.IsNotExist(err) {
 		err = os.MkdirAll(fakeProcDir, os.ModePerm)

integration-tests/suites/process_network.go

@@ -23,6 +23,7 @@ type ProcessNetworkTestSuite struct {
 // Launches nginx container
 // Execs into nginx and does a sleep
 func (s *ProcessNetworkTestSuite) SetupSuite() {
+	defer s.RecoverSetup("nginx", "nginx-curl")
 	s.StartContainerStats()
 	s.StartCollector(false)
 

integration-tests/suites/procfs_scraper.go

@@ -23,6 +23,8 @@ type ProcfsScraperTestSuite struct {
 // other tests. The purpose is that we want ProcfsScraper to see the nginx endpoint and we do not want
 // NetworkSignalHandler to see the nginx endpoint.
 func (s *ProcfsScraperTestSuite) SetupSuite() {
+	defer s.RecoverSetup("nginx")
+
 	s.StartContainerStats()
 
 	s.Collector().Env["COLLECTOR_CONFIG"] = `{"logLevel":"debug","turnOffScrape":` + strconv.FormatBool(s.TurnOffScrape) + `,"scrapeInterval":2}`

integration-tests/suites/repeated_network_flow.go

@@ -34,6 +34,7 @@ type RepeatedNetworkFlowTestSuite struct {
 // Launches gRPC server in insecure mode
 // Launches nginx container
 func (s *RepeatedNetworkFlowTestSuite) SetupSuite() {
+	defer s.RecoverSetup("nginx", "nginx-curl")
 	s.StartContainerStats()
 
 	s.Collector().Env["COLLECTOR_CONFIG"] = `{"logLevel":"debug","turnOffScrape":true,"scrapeInterval":` + strconv.Itoa(s.ScrapeInterval) + `}`
@@ -91,7 +92,7 @@ func (s *RepeatedNetworkFlowTestSuite) SetupSuite() {
 
 func (s *RepeatedNetworkFlowTestSuite) TearDownSuite() {
 	s.StopCollector()
-	s.cleanupContainer([]string{"nginx", "nginx-curl", "collector"})
+	s.cleanupContainer([]string{"nginx", "nginx-curl"})
 	stats := s.GetContainerStats()
 	s.PrintContainerStats(stats)
 	s.WritePerfResults("repeated_network_flow", stats, s.metrics)

integration-tests/suites/socat.go

@@ -29,6 +29,7 @@ type SocatTestSuite struct {
 }
 
 func (s *SocatTestSuite) SetupSuite() {
+	defer s.RecoverSetup("socat")
 	s.StartContainerStats()
 
 	s.Collector().Env["COLLECTOR_CONFIG"] = `{"logLevel":"debug","turnOffScrape":false,"scrapeInterval":2}`
@@ -53,7 +54,7 @@ func (s *SocatTestSuite) SetupSuite() {
 
 func (s *SocatTestSuite) TearDownSuite() {
 	s.StopCollector()
-	s.cleanupContainer([]string{"socat", "collector"})
+	s.cleanupContainer([]string{"socat"})
 	stats := s.GetContainerStats()
 	s.PrintContainerStats(stats)
 	s.WritePerfResults("Socat", stats, s.metrics)

integration-tests/suites/symlink_process.go

@@ -15,6 +15,7 @@ type SymbolicLinkProcessTestSuite struct {
 }
 
 func (s *SymbolicLinkProcessTestSuite) SetupSuite() {
+	defer s.RecoverSetup("process-ports")
 	s.StartContainerStats()
 
 	s.Collector().Env["COLLECTOR_CONFIG"] = `{"logLevel":"debug","turnOffScrape":false,"scrapeInterval":2}`
@@ -42,7 +43,7 @@ func (s *SymbolicLinkProcessTestSuite) SetupSuite() {
 
 func (s *SymbolicLinkProcessTestSuite) TearDownSuite() {
 	s.StopCollector()
-	s.cleanupContainer([]string{"process-ports", "collector"})
+	s.cleanupContainer([]string{"process-ports"})
 	stats := s.GetContainerStats()
 	s.PrintContainerStats(stats)
 	s.WritePerfResults("SymbolicLinkProcess", stats, s.metrics)