From e64134792162718d1f5564bbc1322005c353ec48 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 12 Nov 2024 10:53:23 -0500 Subject: [PATCH] Add v6 distribution client (#2150) * add distribution client Signed-off-by: Alex Goodman * show message on deprecation or EOL Signed-off-by: Alex Goodman * rename metadata.json to description.json Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- go.mod | 1 + grype/db/internal/schemaver/schema_ver.go | 45 +++ .../db/internal/schemaver/schema_ver_test.go | 95 +++++ grype/db/v6/description.go | 87 ++++ grype/db/v6/description_test.go | 127 ++++++ grype/db/v6/distribution/client.go | 250 ++++++++++++ grype/db/v6/distribution/client_test.go | 382 ++++++++++++++++++ grype/db/v6/distribution/latest.go | 105 +++++ grype/db/v6/distribution/latest_test.go | 234 +++++++++++ grype/db/v6/distribution/status.go | 16 + 10 files changed, 1342 insertions(+) create mode 100644 grype/db/internal/schemaver/schema_ver.go create mode 100644 grype/db/internal/schemaver/schema_ver_test.go create mode 100644 grype/db/v6/description.go create mode 100644 grype/db/v6/description_test.go create mode 100644 grype/db/v6/distribution/client.go create mode 100644 grype/db/v6/distribution/client_test.go create mode 100644 grype/db/v6/distribution/latest.go create mode 100644 grype/db/v6/distribution/latest_test.go create mode 100644 grype/db/v6/distribution/status.go diff --git a/go.mod b/go.mod index bd013906eea..3b741af1b61 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.0 require ( github.com/CycloneDX/cyclonedx-go v0.9.1 github.com/Masterminds/sprig/v3 v3.3.0 + github.com/OneOfOne/xxhash v1.2.8 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/adrg/xdg v0.5.3 github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 diff --git a/grype/db/internal/schemaver/schema_ver.go b/grype/db/internal/schemaver/schema_ver.go new file mode 100644 index 00000000000..f850e9c723f --- /dev/null +++ b/grype/db/internal/schemaver/schema_ver.go @@ -0,0 +1,45 @@ +package schemaver + +import ( + "fmt" + "strconv" + "strings" +) + +type SchemaVer string + +func New(model, revision, addition int) SchemaVer { + return SchemaVer(fmt.Sprintf("%d.%d.%d", model, revision, addition)) +} + +func (s SchemaVer) String() string { + return string(s) +} + +func (s SchemaVer) ModelPart() (int, bool) { + v, ok := parseVersionPart(s, 0) + if v == 0 { + ok = false + } + return v, ok +} + +func (s SchemaVer) RevisionPart() (int, bool) { + return parseVersionPart(s, 1) +} + +func (s SchemaVer) AdditionPart() (int, bool) { + return parseVersionPart(s, 2) +} + +func parseVersionPart(s SchemaVer, index int) (int, bool) { + parts := strings.Split(string(s), ".") + if len(parts) <= index { + return 0, false + } + value, err := strconv.Atoi(parts[index]) + if err != nil { + return 0, false + } + return value, true +} diff --git a/grype/db/internal/schemaver/schema_ver_test.go b/grype/db/internal/schemaver/schema_ver_test.go new file mode 100644 index 00000000000..e4d591cfb17 --- /dev/null +++ b/grype/db/internal/schemaver/schema_ver_test.go @@ -0,0 +1,95 @@ +package schemaver + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSchemaVer_VersionComponents(t *testing.T) { + tests := []struct { + name string + version SchemaVer + expectedModel int + expectedRevision int + expectedAddition int + }{ + { + name: "go case", + version: "1.2.3", + expectedModel: 1, + expectedRevision: 2, + expectedAddition: 3, + }, + { + name: "model only", + version: "1.0.0", + expectedModel: 1, + expectedRevision: 0, + expectedAddition: 0, + }, + { + name: "invalid model", + version: "0.2.3", + expectedModel: -1, + expectedRevision: 2, + expectedAddition: 3, + }, + { + name: "invalid version format", + version: "invalid.version", + expectedModel: -1, + expectedRevision: -1, + expectedAddition: -1, + }, + { + name: "zero version", + version: "0.0.0", + expectedModel: -1, + expectedRevision: 0, + expectedAddition: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + type subject struct { + name string + exp int + fn func() (int, bool) + } + + for _, sub := range []subject{ + { + name: "model", + exp: tt.expectedModel, + fn: tt.version.ModelPart, + }, + { + name: "revision", + exp: tt.expectedRevision, + fn: tt.version.RevisionPart, + }, + { + name: "addition", + exp: tt.expectedAddition, + fn: tt.version.AdditionPart, + }, + } { + t.Run(sub.name, func(t *testing.T) { + act, ok := sub.fn() + + if sub.exp == -1 { + require.False(t, ok, fmt.Sprintf("Expected %s to be invalid", sub.name)) + return + } + require.True(t, ok, fmt.Sprintf("Expected %s to be valid", sub.name)) + assert.Equal(t, sub.exp, act, fmt.Sprintf("Expected %s to be %d, got %d", sub.name, sub.exp, act)) + }) + } + + }) + } +} diff --git a/grype/db/v6/description.go b/grype/db/v6/description.go new file mode 100644 index 00000000000..0b0f36926da --- /dev/null +++ b/grype/db/v6/description.go @@ -0,0 +1,87 @@ +package v6 + +import ( + "fmt" + "path" + "time" + + "github.com/OneOfOne/xxhash" + "github.com/spf13/afero" + + "github.com/anchore/grype/grype/db/internal/schemaver" + "github.com/anchore/grype/internal/file" +) + +const DescriptionFileName = "description.json" + +type Description struct { + // SchemaVersion is the version of the DB schema + SchemaVersion schemaver.SchemaVer `json:"schemaVersion,omitempty"` + + // Built is the timestamp the database was built + Built Time `json:"built"` + + // Checksum is the self-describing digest of the database file + Checksum string `json:"checksum"` +} + +type Time struct { + time.Time +} + +func (t Time) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", t.String())), nil +} + +func (t *Time) UnmarshalJSON(data []byte) error { + str := string(data) + if len(str) < 2 || str[0] != '"' || str[len(str)-1] != '"' { + return fmt.Errorf("invalid time format") + } + str = str[1 : len(str)-1] + + parsedTime, err := time.Parse(time.RFC3339, str) + if err != nil { + return err + } + + t.Time = parsedTime.In(time.UTC) + return nil +} + +func (t Time) String() string { + return t.Time.UTC().Round(time.Second).Format(time.RFC3339) +} + +func NewDescriptionFromDir(fs afero.Fs, dir string) (*Description, error) { + // checksum the DB file + dbFilePath := path.Join(dir, VulnerabilityDBFileName) + digest, err := file.HashFile(fs, dbFilePath, xxhash.New64()) + if err != nil { + return nil, fmt.Errorf("failed to calculate checksum for DB file (%s): %w", dbFilePath, err) + } + namedDigest := fmt.Sprintf("xxh64:%s", digest) + + // access the DB to get the built time and schema version + r, err := NewReader(Config{ + DBDirPath: dir, + }) + if err != nil { + return nil, err + } + + meta, err := r.GetDBMetadata() + if err != nil { + return nil, err + } + + return &Description{ + SchemaVersion: schemaver.New(meta.Model, meta.Revision, meta.Addition), + Built: Time{Time: *meta.BuildTimestamp}, + Checksum: namedDigest, + }, nil +} + +func (m Description) String() string { + return fmt.Sprintf("DB(version=%s built=%s checksum=%s)", m.SchemaVersion, m.Built, m.Checksum) +} diff --git a/grype/db/v6/description_test.go b/grype/db/v6/description_test.go new file mode 100644 index 00000000000..dc4949f8637 --- /dev/null +++ b/grype/db/v6/description_test.go @@ -0,0 +1,127 @@ +package v6 + +import ( + "encoding/json" + "fmt" + "io" + "os" + "path" + "testing" + "time" + + "github.com/OneOfOne/xxhash" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/grype/grype/db/internal/schemaver" +) + +func TestNewDatabaseDescriptionFromDir(t *testing.T) { + tempDir := t.TempDir() + + // make a test DB + s, err := NewWriter(Config{DBDirPath: tempDir}) + require.NoError(t, err) + require.NoError(t, s.SetDBMetadata()) + expected, err := s.GetDBMetadata() + require.NoError(t, err) + require.NoError(t, s.Close()) + + // get the xxhash of the db file + hasher := xxhash.New64() + f, err := os.Open(path.Join(tempDir, VulnerabilityDBFileName)) + require.NoError(t, err) + _, err = io.Copy(hasher, f) + require.NoError(t, err) + require.NoError(t, f.Close()) + expectedHash := fmt.Sprintf("xxh64:%x", hasher.Sum(nil)) + + // run the test subject + description, err := NewDescriptionFromDir(afero.NewOsFs(), tempDir) + require.NoError(t, err) + require.NotNil(t, description) + + // did it work? + assert.Equal(t, Description{ + SchemaVersion: schemaver.New(expected.Model, expected.Revision, expected.Addition), + Built: Time{*expected.BuildTimestamp}, + Checksum: expectedHash, + }, *description) +} + +func TestTime_JSONMarshalling(t *testing.T) { + tests := []struct { + name string + time Time + expected string + }{ + { + name: "go case", + time: Time{time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + expected: `"2023-09-26T12:00:00Z"`, + }, + { + name: "convert to utc", + time: Time{time.Date(2023, 9, 26, 13, 0, 0, 0, time.FixedZone("UTC+1", 3600))}, + expected: `"2023-09-26T12:00:00Z"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jsonData, err := json.Marshal(tt.time) + require.NoError(t, err) + require.Equal(t, tt.expected, string(jsonData)) + }) + } +} + +func TestTime_JSONUnmarshalling(t *testing.T) { + tests := []struct { + name string + jsonData string + expectedTime Time + expectError require.ErrorAssertionFunc + }{ + { + name: "use zulu offset", + jsonData: `"2023-09-26T12:00:00Z"`, + expectedTime: Time{time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + }, + { + name: "use tz offset in another timezone", + jsonData: `"2023-09-26T14:00:00+02:00"`, + expectedTime: Time{time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + }, + { + name: "use tz offset that is utc", + jsonData: `"2023-09-26T12:00:00+00:00"`, + expectedTime: Time{time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + }, + { + name: "invalid format", + jsonData: `"invalid-time-format"`, + expectError: require.Error, + }, + { + name: "invalid json", + jsonData: `invalid`, + expectError: require.Error, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectError == nil { + tt.expectError = require.NoError + } + var parsedTime Time + err := json.Unmarshal([]byte(tt.jsonData), &parsedTime) + tt.expectError(t, err) + if err == nil { + assert.Equal(t, tt.expectedTime.Time, parsedTime.Time) + } + }) + } +} diff --git a/grype/db/v6/distribution/client.go b/grype/db/v6/distribution/client.go new file mode 100644 index 00000000000..1aba427bcb1 --- /dev/null +++ b/grype/db/v6/distribution/client.go @@ -0,0 +1,250 @@ +package distribution + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "net/http" + "net/url" + "os" + "path" + "time" + + "github.com/hashicorp/go-cleanhttp" + "github.com/spf13/afero" + "github.com/wagoodman/go-progress" + + "github.com/anchore/clio" + v6 "github.com/anchore/grype/grype/db/v6" + "github.com/anchore/grype/internal/bus" + "github.com/anchore/grype/internal/file" + "github.com/anchore/grype/internal/log" +) + +type Config struct { + ID clio.Identification + + // check/fetch parameters + LatestURL string + CACert string + + // validations + ValidateByHashOnGet bool + RequireUpdateCheck bool + + // timeouts + CheckTimeout time.Duration + UpdateTimeout time.Duration +} + +type Client interface { + IsUpdateAvailable(current *v6.Description) (*Archive, error) + Download(archive Archive, dest string, downloadProgress *progress.Manual) (string, error) +} + +type client struct { + fs afero.Fs + latestHTTPClient *http.Client + updateDownloader file.Getter + config Config +} + +func DefaultConfig() Config { + return Config{ + LatestURL: "https://grype.anchore.io/databases/latest.json", + ValidateByHashOnGet: true, + RequireUpdateCheck: false, + CheckTimeout: 30 * time.Second, + UpdateTimeout: 300 * time.Second, + } +} + +func NewClient(cfg Config) (Client, error) { + fs := afero.NewOsFs() + latestClient, err := defaultHTTPClient(fs, cfg.CACert, withClientTimeout(cfg.CheckTimeout)) + if err != nil { + return client{}, err + } + + dbClient, err := defaultHTTPClient(fs, cfg.CACert, withClientTimeout(cfg.UpdateTimeout)) + if err != nil { + return client{}, err + } + + return client{ + fs: fs, + latestHTTPClient: latestClient, + updateDownloader: file.NewGetter(cfg.ID, dbClient), + config: cfg, + }, nil +} + +// IsUpdateAvailable indicates if there is a new update available as a boolean, and returns the latest db information +// available for this schema. +func (c client) IsUpdateAvailable(current *v6.Description) (*Archive, error) { + log.Debugf("checking for available database updates") + + latestDoc, err := c.latestFromURL() + if err != nil { + if c.config.RequireUpdateCheck { + return nil, fmt.Errorf("check for vulnerability database update failed: %+v", err) + } + log.Warnf("unable to check for vulnerability database update") + log.Debugf("check for vulnerability update failed: %+v", err) + } + + archive, message := c.isUpdateAvailable(current, latestDoc) + + if message != "" { + log.Warn(message) + bus.Notify(message) + } + + return archive, err +} + +func (c client) isUpdateAvailable(current *v6.Description, candidate *LatestDocument) (*Archive, string) { + if candidate == nil { + return nil, "" + } + + var message string + switch candidate.Status { + case StatusDeprecated: + message = "this version of grype will soon stop receiving vulnerability database updates, please update grype" + case StatusEndOfLife: + message = "this version of grype is no longer receiving vulnerability database updates, please update grype" + } + + // compare created data to current db date + if isSupersededBy(current, candidate.Archive.Description) { + log.Debugf("database update available: %s", candidate.Archive.Description) + return &candidate.Archive, message + } + + log.Debugf("no database update available") + return nil, message +} + +func (c client) Download(archive Archive, dest string, downloadProgress *progress.Manual) (string, error) { + defer downloadProgress.SetCompleted() + + // note: as much as I'd like to use the afero FS abstraction here, the go-getter library does not support it + tempDir, err := os.MkdirTemp(dest, "grype-db-download") + if err != nil { + return "", fmt.Errorf("unable to create db temp dir: %w", err) + } + + // download the db to the temp dir + u, err := url.Parse(c.config.LatestURL) + if err != nil { + removeAllOrLog(afero.NewOsFs(), tempDir) + return "", fmt.Errorf("unable to parse db URL %q: %w", c.config.LatestURL, err) + } + + u.Path = path.Join(path.Dir(u.Path), path.Clean(archive.Path)) + + // from go-getter, adding a checksum as a query string will validate the payload after download + // note: the checksum query parameter is not sent to the server + query := u.Query() + if archive.Checksum != "" { + query.Add("checksum", archive.Checksum) + } + u.RawQuery = query.Encode() + + // go-getter will automatically extract all files within the archive to the temp dir + err = c.updateDownloader.GetToDir(tempDir, u.String(), downloadProgress) + if err != nil { + removeAllOrLog(afero.NewOsFs(), tempDir) + return "", fmt.Errorf("unable to download db: %w", err) + } + + return tempDir, nil +} + +// latestFromURL loads a LatestDocument from a URL. +func (c client) latestFromURL() (*LatestDocument, error) { + resp, err := c.latestHTTPClient.Get(c.config.LatestURL) + if err != nil { + return nil, fmt.Errorf("unable to fetch latest.json: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unable to fetch latest.json: %s", resp.Status) + } + + defer resp.Body.Close() + + return NewLatestFromReader(resp.Body) +} + +func withClientTimeout(timeout time.Duration) func(*http.Client) { + return func(c *http.Client) { + c.Timeout = timeout + } +} + +func defaultHTTPClient(fs afero.Fs, caCertPath string, postProcessor ...func(*http.Client)) (*http.Client, error) { + httpClient := cleanhttp.DefaultClient() + httpClient.Timeout = 30 * time.Second + if caCertPath != "" { + rootCAs := x509.NewCertPool() + + pemBytes, err := afero.ReadFile(fs, caCertPath) + if err != nil { + return nil, fmt.Errorf("unable to configure root CAs for curator: %w", err) + } + rootCAs.AppendCertsFromPEM(pemBytes) + + httpClient.Transport.(*http.Transport).TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, + } + } + + for _, pp := range postProcessor { + pp(httpClient) + } + + return httpClient, nil +} + +func removeAllOrLog(fs afero.Fs, dir string) { + if err := fs.RemoveAll(dir); err != nil { + log.WithFields("error", err).Warnf("failed to remove path %q", dir) + } +} + +func isSupersededBy(m *v6.Description, other v6.Description) bool { + if m == nil { + log.Debug("cannot find existing metadata, using update...") + // any valid update beats no database, use it! + return true + } + + otherModelPart, otherOk := other.SchemaVersion.ModelPart() + currentModelPart, currentOk := m.SchemaVersion.ModelPart() + + if !otherOk { + log.Error("existing database has no schema version, doing nothing...") + return false + } + + if !currentOk { + log.Error("update has no schema version, doing nothing...") + return false + } + + if otherModelPart != currentModelPart { + log.WithFields("want", currentModelPart, "received", otherModelPart).Warn("update is for a different DB schema, skipping...") + return false + } + + if other.Built.After(m.Built.Time) { + log.WithFields("existing", m.Built.String(), "candidate", other.Built.String()).Debug("existing database is older than candidate update, using update...") + // the listing is newer than the existing db, use it! + return true + } + + log.Debugf("existing database is already up to date") + return false +} diff --git a/grype/db/v6/distribution/client_test.go b/grype/db/v6/distribution/client_test.go new file mode 100644 index 00000000000..f79a8cbc1c6 --- /dev/null +++ b/grype/db/v6/distribution/client_test.go @@ -0,0 +1,382 @@ +package distribution + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-progress" + + db "github.com/anchore/grype/grype/db/v6" +) + +func TestClient_LatestFromURL(t *testing.T) { + tests := []struct { + name string + setupServer func() *httptest.Server + expectedDoc *LatestDocument + expectedErr require.ErrorAssertionFunc + }{ + { + name: "go case", + setupServer: func() *httptest.Server { + doc := LatestDocument{ + SchemaVersion: "1.0.0", + Status: "active", + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive", + Checksum: "checksum123", + }, + } + data, _ := json.Marshal(doc) + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write(data) + require.NoError(t, err) + })) + }, + expectedDoc: &LatestDocument{ + SchemaVersion: "1.0.0", + Status: "active", + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive", + Checksum: "checksum123", + }, + }, + }, + { + name: "error response", + setupServer: func() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + }, + expectedDoc: nil, + expectedErr: func(t require.TestingT, err error, _ ...interface{}) { + require.Error(t, err) + require.Contains(t, err.Error(), "500 Internal Server Error") + }, + }, + { + name: "malformed JSON response", + setupServer: func() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("malformed json")) + require.NoError(t, err) + })) + }, + expectedDoc: nil, + expectedErr: func(t require.TestingT, err error, _ ...interface{}) { + require.Error(t, err) + require.Contains(t, err.Error(), "invalid character 'm' looking for beginning of value") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectedErr == nil { + tt.expectedErr = require.NoError + } + + server := tt.setupServer() + defer server.Close() + + c, err := NewClient(Config{ + LatestURL: server.URL, + }) + require.NoError(t, err) + + cl := c.(client) + + doc, err := cl.latestFromURL() + tt.expectedErr(t, err) + if err != nil { + return + } + + require.Equal(t, tt.expectedDoc, doc) + }) + } +} + +type mockGetter struct { + mock.Mock +} + +func (m *mockGetter) GetFile(dst, src string, manuals ...*progress.Manual) error { + args := m.Called(dst, src, manuals) + return args.Error(0) +} + +func (m *mockGetter) GetToDir(dst, src string, manuals ...*progress.Manual) error { + args := m.Called(dst, src, manuals) + return args.Error(0) +} + +func TestClient_Download(t *testing.T) { + destDir := t.TempDir() + archive := &Archive{ + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + } + + setup := func() (Client, *mockGetter) { + mg := new(mockGetter) + + c, err := NewClient(Config{ + LatestURL: "http://localhost:8080/latest.json", + }) + require.NoError(t, err) + + cl := c.(client) + cl.updateDownloader = mg + + return cl, mg + } + + t.Run("successful download", func(t *testing.T) { + c, mg := setup() + mg.On("GetToDir", mock.Anything, "http://localhost:8080/path/to/archive.tar.gz?checksum=checksum123", mock.Anything).Return(nil) + + tempDir, err := c.Download(*archive, destDir, &progress.Manual{}) + require.NoError(t, err) + require.True(t, len(tempDir) > 0) + + mg.AssertExpectations(t) + }) + + t.Run("download error", func(t *testing.T) { + c, mg := setup() + mg.On("GetToDir", mock.Anything, "http://localhost:8080/path/to/archive.tar.gz?checksum=checksum123", mock.Anything).Return(errors.New("download failed")) + + tempDir, err := c.Download(*archive, destDir, &progress.Manual{}) + require.Error(t, err) + require.Empty(t, tempDir) + require.Contains(t, err.Error(), "unable to download db") + + mg.AssertExpectations(t) + }) +} + +func TestClient_IsUpdateAvailable(t *testing.T) { + current := &db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + } + + tests := []struct { + name string + candidate *LatestDocument + archive *Archive + message string + }{ + { + name: "update available", + candidate: &LatestDocument{ + SchemaVersion: "1.0.0", + Status: StatusActive, + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + }, + archive: &Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + }, + { + name: "no update available", + candidate: &LatestDocument{ + SchemaVersion: "1.0.0", + Status: "active", + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + }, + archive: nil, + }, + { + name: "no candidate available", + candidate: nil, + archive: nil, + }, + { + name: "candidate deprecated", + candidate: &LatestDocument{ + SchemaVersion: "1.0.0", + Status: StatusDeprecated, + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + }, + archive: &Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + message: "this version of grype will soon stop receiving vulnerability database updates, please update grype", + }, + { + name: "candidate end of life", + candidate: &LatestDocument{ + SchemaVersion: "1.0.0", + Status: StatusEndOfLife, + Archive: Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + }, + archive: &Archive{ + Description: db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + }, + Path: "path/to/archive.tar.gz", + Checksum: "checksum123", + }, + message: "this version of grype is no longer receiving vulnerability database updates, please update grype", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := NewClient(Config{}) + require.NoError(t, err) + + cl := c.(client) + + archive, message := cl.isUpdateAvailable(current, tt.candidate) + assert.Equal(t, tt.message, message) + assert.Equal(t, tt.archive, archive) + }) + } +} + +func TestDatabaseDescription_IsSupersededBy(t *testing.T) { + t1 := time.Date(2023, 9, 26, 12, 0, 0, 0, time.UTC) + t2 := time.Date(2023, 9, 27, 12, 0, 0, 0, time.UTC) + + currentMetadata := db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: t1}, + } + + newerMetadata := db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: t2}, + } + + olderMetadata := db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: t1}, + } + + differentModelMetadata := db.Description{ + SchemaVersion: "2.0.0", + Built: db.Time{Time: t2}, + } + + tests := []struct { + name string + current *db.Description + other db.Description + expected bool + }{ + { + name: "no current metadata", + current: nil, + other: newerMetadata, + expected: true, + }, + { + name: "newer build", + current: ¤tMetadata, + other: newerMetadata, + expected: true, + }, + { + name: "older build", + current: ¤tMetadata, + other: olderMetadata, + expected: false, + }, + { + name: "different schema version", + current: ¤tMetadata, + other: differentModelMetadata, + expected: false, + }, + { + name: "current metadata has no schema version", + current: &db.Description{Built: db.Time{Time: t1}}, + other: newerMetadata, + expected: false, + }, + { + name: "update has no schema version", + current: ¤tMetadata, + other: db.Description{Built: db.Time{Time: t2}}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := isSupersededBy(tt.current, tt.other) + require.Equal(t, tt.expected, result) + }) + } +} diff --git a/grype/db/v6/distribution/latest.go b/grype/db/v6/distribution/latest.go new file mode 100644 index 00000000000..52ae13d7d60 --- /dev/null +++ b/grype/db/v6/distribution/latest.go @@ -0,0 +1,105 @@ +package distribution + +import ( + "encoding/json" + "fmt" + "io" + "sort" + + "github.com/anchore/grype/grype/db/internal/schemaver" + db "github.com/anchore/grype/grype/db/v6" +) + +const LatestFileName = "latest.json" + +type LatestDocument struct { + // SchemaVersion is the version of the DB schema + SchemaVersion schemaver.SchemaVer `json:"schemaVersion"` + + // Status indicates if the database is actively being maintained and distributed + Status Status `json:"status"` + + // Archive is the most recent database that has been built and distributed, additionally annotated with provider-level information + Archive Archive `json:"archive"` +} + +type Archive struct { + // Description contains details about the database contained within the distribution archive + Description db.Description `json:"database"` + + // Path is the path to a DB archive relative to the listing file hosted location. + // Note: this is NOT the absolute URL to download the database. + Path string `json:"path"` + + // Checksum is the self describing digest of the database archive referenced in path + Checksum string `json:"checksum"` +} + +func NewLatestDocument(entries ...Archive) *LatestDocument { + if len(entries) == 0 { + return nil + } + + // sort from most recent to the least recent + sort.SliceStable(entries, func(i, j int) bool { + return entries[i].Description.Built.After(entries[j].Description.Built.Time) + }) + + return &LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: entries[0], + Status: LifecycleStatus, + } +} + +func NewLatestFromReader(reader io.Reader) (*LatestDocument, error) { + var l LatestDocument + + if err := json.NewDecoder(reader).Decode(&l); err != nil { + return nil, fmt.Errorf("unable to parse DB latest.json: %w", err) + } + + // inflate entry data from parent + if l.Archive.Description.SchemaVersion != "" { + l.Archive.Description.SchemaVersion = l.SchemaVersion + } + + return &l, nil +} + +func (l LatestDocument) Write(writer io.Writer) error { + if l.SchemaVersion == "" { + return fmt.Errorf("missing schema version") + } + + if l.Status == "" { + l.Status = LifecycleStatus + } + + if l.Archive.Path == "" { + return fmt.Errorf("missing archive path") + } + + if l.Archive.Checksum == "" { + return fmt.Errorf("missing archive checksum") + } + + if l.Archive.Description.Built.Time.IsZero() { + return fmt.Errorf("missing built time") + } + + if l.Archive.Description.Checksum == "" { + return fmt.Errorf("missing database checksum") + } + + // we don't need to store duplicate information from the archive section in the doc + l.Archive.Description.SchemaVersion = "" + + contents, err := json.MarshalIndent(&l, "", " ") + if err != nil { + return fmt.Errorf("failed to encode listing file: %w", err) + } + + _, err = writer.Write(contents) + return err +} diff --git a/grype/db/v6/distribution/latest_test.go b/grype/db/v6/distribution/latest_test.go new file mode 100644 index 00000000000..b13210e48c6 --- /dev/null +++ b/grype/db/v6/distribution/latest_test.go @@ -0,0 +1,234 @@ +package distribution + +import ( + "bytes" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/grype/grype/db/internal/schemaver" + db "github.com/anchore/grype/grype/db/v6" +) + +func TestNewLatestDocument(t *testing.T) { + t.Run("valid entries", func(t *testing.T) { + archive1 := Archive{ + Description: db.Description{ + Built: db.Time{Time: time.Now()}, + }, + } + archive2 := Archive{ + Description: db.Description{ + Built: db.Time{Time: time.Now().Add(-1 * time.Hour)}, + }, + } + + latestDoc := NewLatestDocument(archive1, archive2) + require.NotNil(t, latestDoc) + require.Equal(t, latestDoc.Archive, archive1) // most recent archive + actual, ok := latestDoc.SchemaVersion.ModelPart() + require.True(t, ok) + require.Equal(t, actual, db.ModelVersion) + }) + + t.Run("no entries", func(t *testing.T) { + latestDoc := NewLatestDocument() + require.Nil(t, latestDoc) + }) +} + +func TestNewLatestFromReader(t *testing.T) { + t.Run("valid JSON", func(t *testing.T) { + latestDoc := LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: db.Time{Time: time.Now().Truncate(time.Second).UTC()}, + }, + }, + Status: "active", + } + + var buf bytes.Buffer + require.NoError(t, json.NewEncoder(&buf).Encode(latestDoc)) + + result, err := NewLatestFromReader(&buf) + require.NoError(t, err) + require.Equal(t, latestDoc.SchemaVersion, result.SchemaVersion) + require.Equal(t, latestDoc.Archive.Description.Built.Time, result.Archive.Description.Built.Time) + }) + + t.Run("invalid JSON", func(t *testing.T) { + invalidJSON := []byte("invalid json") + _, err := NewLatestFromReader(bytes.NewReader(invalidJSON)) + require.Error(t, err) + require.Contains(t, err.Error(), "unable to parse DB latest.json") + }) +} + +func TestLatestDocument_Write(t *testing.T) { + + errContains := func(text string) require.ErrorAssertionFunc { + return func(t require.TestingT, err error, msgAndArgs ...interface{}) { + require.ErrorContains(t, err, text, msgAndArgs...) + } + } + + now := db.Time{Time: time.Now().Truncate(time.Second).UTC()} + + tests := []struct { + name string + latestDoc LatestDocument + expectedError require.ErrorAssertionFunc + }{ + { + name: "valid document", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "xxh64:validchecksum", + }, + // note: status not supplied, should assume to be active + }, + expectedError: require.NoError, + }, + { + name: "explicit status", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "xxh64:validchecksum", + }, + Status: StatusDeprecated, + }, + expectedError: require.NoError, + }, + { + name: "missing schema version", + latestDoc: LatestDocument{ + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "xxh64:validchecksum", + }, + Status: "active", + }, + expectedError: errContains("missing schema version"), + }, + { + name: "missing archive path", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "", // this! + Checksum: "xxh64:validchecksum", + }, + Status: "active", + }, + expectedError: errContains("missing archive path"), + }, + { + name: "missing archive checksum", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "", // this! + }, + Status: "active", + }, + expectedError: errContains("missing archive checksum"), + }, + { + name: "missing built time", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: db.Time{}, // this! + Checksum: "xxh64:validchecksum", + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "xxh64:validchecksum", + }, + Status: "active", + }, + expectedError: errContains("missing built time"), + }, + { + name: "missing database checksum", + latestDoc: LatestDocument{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Archive: Archive{ + Description: db.Description{ + Built: now, + Checksum: "", // this! + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + }, + Path: "valid/path/to/archive", + Checksum: "xxh64:validchecksum", + }, + Status: "active", + }, + expectedError: errContains("missing database checksum"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.expectedError == nil { + tt.expectedError = require.NoError + } + var buf bytes.Buffer + err := tt.latestDoc.Write(&buf) + tt.expectedError(t, err) + if err != nil { + return + } + + var result LatestDocument + assert.NoError(t, json.Unmarshal(buf.Bytes(), &result)) + assert.Equal(t, tt.latestDoc.SchemaVersion, result.SchemaVersion, "schema version mismatch") + assert.Empty(t, result.Archive.Description.SchemaVersion, "nested schema version should be empty") + assert.Equal(t, tt.latestDoc.Archive.Checksum, result.Archive.Checksum, "archive checksum mismatch") + assert.Equal(t, tt.latestDoc.Archive.Description.Built.Time, result.Archive.Description.Built.Time, "built time mismatch") + assert.Equal(t, tt.latestDoc.Archive.Description.Checksum, result.Archive.Description.Checksum, "database checksum mismatch") + assert.Equal(t, tt.latestDoc.Archive.Path, result.Archive.Path, "path mismatch") + if tt.latestDoc.Status == "" { + assert.Equal(t, StatusActive, result.Status, "status mismatch") + } else { + assert.Equal(t, tt.latestDoc.Status, result.Status, "status mismatch") + } + }) + } +} diff --git a/grype/db/v6/distribution/status.go b/grype/db/v6/distribution/status.go new file mode 100644 index 00000000000..2f866d951b7 --- /dev/null +++ b/grype/db/v6/distribution/status.go @@ -0,0 +1,16 @@ +package distribution + +type Status string + +const LifecycleStatus = StatusActive + +const ( + // StatusActive indicates the database is actively being maintained and distributed + StatusActive Status = "active" + + // StatusDeprecated indicates the database is still being distributed but is approaching end of life. Upgrade grype to avoid future disruptions. + StatusDeprecated Status = "deprecated" + + // StatusEndOfLife indicates the database is no longer being distributed. Users must build their own databases or upgrade grype. + StatusEndOfLife Status = "eol" +)