WiP: Add cli for checking CI test flakiness

.gitignore

@@ -1,4 +1,5 @@
-junit2jira
+/junit2jira
+/flakechecker
 .idea
 # Binaries for programs and plugins
 *.exe

README.md

@@ -4,7 +4,7 @@ Convert test failures to jira issues
 
 ### Build
 ```shell
-go build ./...
+go build  -o . ./...
 ```
 
 ### Test

cmd/flakechecker/bq_client.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"cloud.google.com/go/bigquery"
+	"context"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/iterator"
+	"regexp"
+	"time"
+)
+
+const projectID = "acs-san-stackroxci"
+const queryTimeout = 1 * time.Minute
+
+type biqQueryClient interface {
+	GetRatioForTest(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error)
+}
+
+type biqQueryClientImpl struct {
+	client *bigquery.Client
+}
+
+func getNewBigQueryClient() (biqQueryClient, error) {
+	ctx := context.Background()
+
+	client, err := bigquery.NewClient(ctx, projectID)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating BigQuery client")
+	}
+
+	return &biqQueryClientImpl{client: client}, nil
+}
+
+func getFilteredTestName(name string) string {
+	// Apply the same filtering used in DB to generate test names not influenced with version Z number.
+	regFilterTestName := regexp.MustCompile("(.*)((-v|: )\\d{3}\\.\\d+\\.)(\\d+)(.*)")
+	return regFilterTestName.ReplaceAllString(name, `${1}${2}z${5}`)
+}
+
+func (c *biqQueryClientImpl) GetRatioForTest(flakeTestConfig *flakeCheckerRecord, testName string) (int, int, error) {
+	query := c.client.Query(
+		`SELECT JobName, FilteredName, Classname, TotalAll, FailRatio
+    FROM acs-san-stackroxci.ci_metrics.stackrox_tests_recent_flaky_tests
+    WHERE JobName = @jobName AND FilteredName = @filteredName AND Classname = @classname
+`)
+
+	query.Parameters = []bigquery.QueryParameter{
+		{Name: "jobName", Value: flakeTestConfig.RatioJobName},
+		{Name: "filteredName", Value: getFilteredTestName(testName)},
+		{Name: "classname", Value: flakeTestConfig.Classname},
+	}
+
+	ctx, _ := context.WithTimeout(context.Background(), queryTimeout)
+	resIter, err := query.Read(ctx)
+	if err != nil {
+		return 0, 0, errors.Wrap(err, "query data from BigQuery")
+	}
+
+	// We need only first record. No need to loop over iterator.
+	var record recentFlakyTestsRecord
+	if errNext := resIter.Next(&record); errNext != nil {
+		return 0, 0, errors.Wrap(errNext, "read BigQuery record")
+	}
+
+	if errNext := resIter.Next(&record); !errors.Is(errNext, iterator.Done) {
+		log.Warnf("Expected to find one row in DB, but got more for query params: %s", query.Parameters)
+	}
+
+	return record.TotalAll, record.FailRatio, nil
+}

cmd/flakechecker/flake_config.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"os"
+	"regexp"
+)
+
+// flakeCheckerRecord represents configuration record used by flakechecker to evaluate failed tests
+//
+// It contains the following fields:
+// match_job_name - name of the job that should be evaluated by flakechecker. i.e. (branch should be evaluated, but main not)
+// ratio_job_name - job name that should be used for ratio calculation. i.e. we take main branch test runs as base for evaluation of flake ratio
+// test_name_regex - regex used to match test names. Some test names contain detailed information (i.e. version 4.4.4), but we want to use ratio for all tests in that group (i.e. 4.4.z). Using regex allow us to group tests differently.
+// classname - class name of the test that should be isolated. With this option we can isolate single flake test from suite and isolate only that one from the rest.
+// ratio_threshold - failure percentage that is allowed for this test. This information is usually fetched from historical executions and data collected in DB.
+//
+// This record also contains helper fields where we keep compiled regex.
+type flakeCheckerRecord struct {
+	MatchJobName   string `json:"match_job_name"`
+	RatioJobName   string `json:"ratio_job_name"`
+	TestNameRegex  string `json:"test_name_regex"`
+	Classname      string `json:"classname"`
+	RatioThreshold int    `json:"ratio_threshold"`
+
+	regexMatchJobName  *regexp.Regexp
+	regexTestNameRegex *regexp.Regexp
+}
+
+func (r *flakeCheckerRecord) updateRegex() error {
+	validRegex, err := regexp.Compile(fmt.Sprintf("^%s$", r.MatchJobName))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("invalid flake config match job regex: %v", r))
+	}
+	r.regexMatchJobName = validRegex
+
+	validRegex, err = regexp.Compile(fmt.Sprintf("^%s$", r.TestNameRegex))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("invalid flake config test name regex: %v", r))
+	}
+	r.regexTestNameRegex = validRegex
+
+	return nil
+}
+
+func (r *flakeCheckerRecord) matchJobName(jobName string) (bool, error) {
+	if r.regexMatchJobName == nil {
+		err := r.updateRegex()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return r.regexMatchJobName.MatchString(jobName), nil
+}
+
+func (r *flakeCheckerRecord) matchTestName(testName string) (bool, error) {
+	if r.regexTestNameRegex == nil {
+		err := r.updateRegex()
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return r.regexTestNameRegex.MatchString(testName), nil
+}
+
+func (r *flakeCheckerRecord) matchClassname(classname string) (bool, error) {
+	return classname == r.Classname, nil
+}
+
+func loadFlakeConfigFile(fileName string) ([]*flakeCheckerRecord, error) {
+	jsonConfigFile, err := os.Open(fileName)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("open flake config file: %s", fileName))
+	}
+	defer jsonConfigFile.Close()
+
+	jsonConfigFileData, err := io.ReadAll(jsonConfigFile)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("read flake config file: %s", fileName))
+	}
+
+	flakeConfigs := make([]*flakeCheckerRecord, 0)
+	err = json.Unmarshal(jsonConfigFileData, &flakeConfigs)
+	if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf("parse flake config file: %s", fileName))
+	}
+
+	// Validate regex-es in config and prepare them.
+	for _, flakeConfig := range flakeConfigs {
+		if err := flakeConfig.updateRegex(); err != nil {
+			return nil, err
+		}
+	}
+
+	return flakeConfigs, nil
+}

cmd/flakechecker/flake_config_test.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"regexp"
+	"testing"
+)
+
+func TestLoadFlakeConfigFile(t *testing.T) {
+	samples := []struct {
+		name     string
+		fileName string
+
+		expectError  bool
+		expectConfig []*flakeCheckerRecord
+	}{
+		{
+			name:         "no config file",
+			fileName:     "no_config.json",
+			expectError:  true,
+			expectConfig: nil,
+		},
+		{
+			name:        "valid config file",
+			fileName:    "testdata/flake-config.json",
+			expectError: false,
+			expectConfig: []*flakeCheckerRecord{
+				{
+					MatchJobName:       "pr-.*",
+					RatioJobName:       "main-branch-tests",
+					TestNameRegex:      "TestLoadFlakeConf.*",
+					Classname:          "TestLoadFlakeConfigFile",
+					RatioThreshold:     5,
+					regexMatchJobName:  regexp.MustCompile("^pr-.*$"),
+					regexTestNameRegex: regexp.MustCompile("^TestLoadFlakeConf.*$"),
+				},
+				{
+					MatchJobName:       "pull-request-tests",
+					RatioJobName:       "main-branch-tests",
+					TestNameRegex:      "TestLoadFlakeConfigFile",
+					Classname:          "TestLoadFlakeConfigFile",
+					RatioThreshold:     10,
+					regexMatchJobName:  regexp.MustCompile("^pull-request-tests$"),
+					regexTestNameRegex: regexp.MustCompile("^TestLoadFlakeConfigFile$"),
+				},
+			},
+		},
+	}
+
+	for _, sample := range samples {
+		t.Run(sample.name, func(tt *testing.T) {
+			config, err := loadFlakeConfigFile(sample.fileName)
+
+			assert.Equal(tt, sample.expectError, err != nil)
+			assert.Equal(tt, sample.expectConfig, config)
+		})
+	}
+}