From cabf8bcd5ae2054237568f484fa1346ce155b906 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 4 Oct 2024 16:12:30 -0400 Subject: [PATCH] Add v6 DB metadata store (#2146) * add db metadata store Signed-off-by: Alex Goodman * add functional options to gorm adapter Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- grype/db/internal/gormadapter/open.go | 79 +++++++++---- grype/db/internal/gormadapter/open_test.go | 127 +++++++++++++++++++++ grype/db/v1/store/store.go | 2 +- grype/db/v2/store/store.go | 2 +- grype/db/v3/store/store.go | 2 +- grype/db/v4/store/store.go | 2 +- grype/db/v5/store/store.go | 2 +- grype/db/v6/db.go | 52 +++++++++ grype/db/v6/db_metadata_store.go | 59 ++++++++++ grype/db/v6/db_metadata_store_test.go | 51 +++++++++ grype/db/v6/models.go | 19 +++ grype/db/v6/store.go | 51 +++++++++ 12 files changed, 423 insertions(+), 25 deletions(-) create mode 100644 grype/db/internal/gormadapter/open_test.go create mode 100644 grype/db/v6/db.go create mode 100644 grype/db/v6/db_metadata_store.go create mode 100644 grype/db/v6/db_metadata_store_test.go create mode 100644 grype/db/v6/models.go create mode 100644 grype/db/v6/store.go diff --git a/grype/db/internal/gormadapter/open.go b/grype/db/internal/gormadapter/open.go index 6448dada25a..1b99f05ee15 100644 --- a/grype/db/internal/gormadapter/open.go +++ b/grype/db/internal/gormadapter/open.go @@ -13,6 +13,8 @@ var writerStatements = []string{ // on my box it reduces the time to write from 10 minutes to 10 seconds (with ~1GB memory utilization spikes) `PRAGMA synchronous = OFF`, `PRAGMA journal_mode = MEMORY`, + `PRAGMA cache_size = 100000`, + `PRAGMA mmap_size = 268435456`, // 256 MB } var readOptions = []string{ @@ -21,31 +23,73 @@ var readOptions = []string{ "mode=ro", } -// Open a new connection to a sqlite3 database file -func Open(path string, write bool) (*gorm.DB, error) { - if write { - // the file may or may not exist, so we ignore the error explicitly - _ = os.Remove(path) +type config struct { + path string + write bool + memory bool +} + +type Option func(*config) + +func WithTruncate(truncate bool) Option { + return func(c *config) { + c.write = truncate } +} - connStr, err := connectionString(path) - if err != nil { - return nil, err +func newConfig(path string, opts []Option) config { + c := config{} + c.apply(path, opts) + return c +} + +func (c *config) apply(path string, opts []Option) { + for _, o := range opts { + o(c) + } + c.memory = len(path) == 0 + c.path = path +} + +func (c config) shouldTruncate() bool { + return c.write && !c.memory +} + +func (c config) connectionString() string { + var conn string + if c.path == "" { + conn = ":memory:" + } else { + conn = fmt.Sprintf("file:%s?cache=shared", c.path) } - if !write { + if !c.write && !c.memory { // &immutable=1&cache=shared&mode=ro for _, o := range readOptions { - connStr += fmt.Sprintf("&%s", o) + conn += fmt.Sprintf("&%s", o) + } + } + return conn +} + +// Open a new connection to a sqlite3 database file +func Open(path string, options ...Option) (*gorm.DB, error) { + cfg := newConfig(path, options) + + if cfg.shouldTruncate() { + if _, err := os.Stat(path); err == nil { + if err := os.Remove(path); err != nil { + return nil, fmt.Errorf("unable to remove existing DB file: %w", err) + } } } - dbObj, err := gorm.Open(sqlite.Open(connStr), &gorm.Config{Logger: newLogger()}) + dbObj, err := gorm.Open(sqlite.Open(cfg.connectionString()), &gorm.Config{Logger: newLogger()}) if err != nil { return nil, fmt.Errorf("unable to connect to DB: %w", err) } - if write { + if cfg.write { for _, sqlStmt := range writerStatements { dbObj.Exec(sqlStmt) if dbObj.Error != nil { @@ -54,13 +98,8 @@ func Open(path string, write bool) (*gorm.DB, error) { } } - return dbObj, nil -} + // needed for v6+ + dbObj.Exec("PRAGMA foreign_keys = ON") -// ConnectionString creates a connection string for sqlite3 -func connectionString(path string) (string, error) { - if path == "" { - return "", fmt.Errorf("no db filepath given") - } - return fmt.Sprintf("file:%s?cache=shared", path), nil + return dbObj, nil } diff --git a/grype/db/internal/gormadapter/open_test.go b/grype/db/internal/gormadapter/open_test.go new file mode 100644 index 00000000000..fbdd3e34d90 --- /dev/null +++ b/grype/db/internal/gormadapter/open_test.go @@ -0,0 +1,127 @@ +package gormadapter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConfigApply(t *testing.T) { + tests := []struct { + name string + path string + options []Option + expectedPath string + expectedMemory bool + }{ + { + name: "apply with path", + path: "test.db", + options: []Option{}, + expectedPath: "test.db", + expectedMemory: false, + }, + { + name: "apply with empty path (memory)", + path: "", + options: []Option{}, + expectedPath: "", + expectedMemory: true, + }, + { + name: "apply with truncate option", + path: "test.db", + options: []Option{WithTruncate(true)}, + expectedPath: "test.db", + expectedMemory: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newConfig(tt.path, tt.options) + + require.Equal(t, tt.expectedPath, c.path) + require.Equal(t, tt.expectedMemory, c.memory) + }) + } +} + +func TestConfigShouldTruncate(t *testing.T) { + tests := []struct { + name string + write bool + memory bool + expectedTruncate bool + }{ + { + name: "should truncate when write is true and not memory", + write: true, + memory: false, + expectedTruncate: true, + }, + { + name: "should not truncate when write is false", + write: false, + memory: false, + expectedTruncate: false, + }, + { + name: "should not truncate when using in-memory DB", + write: true, + memory: true, + expectedTruncate: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := config{ + write: tt.write, + memory: tt.memory, + } + require.Equal(t, tt.expectedTruncate, c.shouldTruncate()) + }) + } +} + +func TestConfigConnectionString(t *testing.T) { + tests := []struct { + name string + path string + write bool + memory bool + expectedConnStr string + }{ + { + name: "writable path", + path: "test.db", + write: true, + expectedConnStr: "file:test.db?cache=shared", + }, + { + name: "read-only path", + path: "test.db", + write: false, + expectedConnStr: "file:test.db?cache=shared&immutable=1&cache=shared&mode=ro", + }, + { + name: "in-memory mode", + path: "", + write: false, + memory: true, + expectedConnStr: ":memory:", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := config{ + path: tt.path, + write: tt.write, + memory: tt.memory, + } + require.Equal(t, tt.expectedConnStr, c.connectionString()) + }) + } +} diff --git a/grype/db/v1/store/store.go b/grype/db/v1/store/store.go index 9b22bc89331..949f60b6c4f 100644 --- a/grype/db/v1/store/store.go +++ b/grype/db/v1/store/store.go @@ -21,7 +21,7 @@ type store struct { // New creates a new instance of the store. func New(dbFilePath string, overwrite bool) (v1.Store, error) { - db, err := gormadapter.Open(dbFilePath, overwrite) + db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite)) if err != nil { return nil, err } diff --git a/grype/db/v2/store/store.go b/grype/db/v2/store/store.go index ee5a313d296..818f100de44 100644 --- a/grype/db/v2/store/store.go +++ b/grype/db/v2/store/store.go @@ -21,7 +21,7 @@ type store struct { // New creates a new instance of the store. func New(dbFilePath string, overwrite bool) (v2.Store, error) { - db, err := gormadapter.Open(dbFilePath, overwrite) + db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite)) if err != nil { return nil, err } diff --git a/grype/db/v3/store/store.go b/grype/db/v3/store/store.go index 8643ac4162c..bcf9f6895cf 100644 --- a/grype/db/v3/store/store.go +++ b/grype/db/v3/store/store.go @@ -21,7 +21,7 @@ type store struct { // New creates a new instance of the store. func New(dbFilePath string, overwrite bool) (v3.Store, error) { - db, err := gormadapter.Open(dbFilePath, overwrite) + db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite)) if err != nil { return nil, err } diff --git a/grype/db/v4/store/store.go b/grype/db/v4/store/store.go index bcdb1acada7..1a8727d519a 100644 --- a/grype/db/v4/store/store.go +++ b/grype/db/v4/store/store.go @@ -21,7 +21,7 @@ type store struct { // New creates a new instance of the store. func New(dbFilePath string, overwrite bool) (v4.Store, error) { - db, err := gormadapter.Open(dbFilePath, overwrite) + db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite)) if err != nil { return nil, err } diff --git a/grype/db/v5/store/store.go b/grype/db/v5/store/store.go index 3c7696adf35..8638d47dfc6 100644 --- a/grype/db/v5/store/store.go +++ b/grype/db/v5/store/store.go @@ -21,7 +21,7 @@ type store struct { // New creates a new instance of the store. func New(dbFilePath string, overwrite bool) (v5.Store, error) { - db, err := gormadapter.Open(dbFilePath, overwrite) + db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite)) if err != nil { return nil, err } diff --git a/grype/db/v6/db.go b/grype/db/v6/db.go new file mode 100644 index 00000000000..ca1e3cc8242 --- /dev/null +++ b/grype/db/v6/db.go @@ -0,0 +1,52 @@ +package v6 + +import ( + "io" + "path/filepath" +) + +const ( + VulnerabilityDBFileName = "vulnerability.db" + + // We follow SchemaVer semantics (see https://snowplow.io/blog/introducing-schemaver-for-semantic-versioning-of-schemas) + + // ModelVersion indicates how many breaking schema changes there have been (which will prevent interaction with any historical data) + // note: this must ALWAYS be "6" in the context of this package. + ModelVersion = 6 + + // Revision indicates how many changes have been introduced which **may** prevent interaction with some historical data + Revision = 0 + + // Addition indicates how many changes have been introduced that are compatible with all historical data + Addition = 0 +) + +type ReadWriter interface { + Reader + Writer +} + +type Reader interface { + DBMetadataStoreReader +} + +type Writer interface { + DBMetadataStoreWriter + io.Closer +} + +type Config struct { + DBDirPath string +} + +func (c *Config) DBFilePath() string { + return filepath.Join(c.DBDirPath, VulnerabilityDBFileName) +} + +func NewReader(cfg Config) (Reader, error) { + return newStore(cfg, false) +} + +func NewWriter(cfg Config) (ReadWriter, error) { + return newStore(cfg, true) +} diff --git a/grype/db/v6/db_metadata_store.go b/grype/db/v6/db_metadata_store.go new file mode 100644 index 00000000000..cbcc9e7ab0d --- /dev/null +++ b/grype/db/v6/db_metadata_store.go @@ -0,0 +1,59 @@ +package v6 + +import ( + "fmt" + "time" + + "gorm.io/gorm" + + "github.com/anchore/grype/internal/log" +) + +type DBMetadataStoreWriter interface { + SetDBMetadata() error +} + +type DBMetadataStoreReader interface { + GetDBMetadata() (*DBMetadata, error) +} + +type dbMetadataStore struct { + db *gorm.DB +} + +func newDBMetadataStore(db *gorm.DB) *dbMetadataStore { + return &dbMetadataStore{ + db: db, + } +} + +func (s *dbMetadataStore) GetDBMetadata() (*DBMetadata, error) { + log.Trace("fetching DB metadata record") + + var model DBMetadata + + result := s.db.First(&model) + return &model, result.Error +} + +func (s *dbMetadataStore) SetDBMetadata() error { + log.Trace("writing DB metadata record") + + if err := s.db.Unscoped().Where("true").Delete(&DBMetadata{}).Error; err != nil { + return fmt.Errorf("failed to delete existing DB metadata record: %w", err) + } + + ts := time.Now().UTC() + instance := &DBMetadata{ + BuildTimestamp: &ts, + Model: ModelVersion, + Revision: Revision, + Addition: Addition, + } + + if err := s.db.Create(instance).Error; err != nil { + return fmt.Errorf("failed to create DB metadata record: %w", err) + } + + return nil +} diff --git a/grype/db/v6/db_metadata_store_test.go b/grype/db/v6/db_metadata_store_test.go new file mode 100644 index 00000000000..bc4da93324c --- /dev/null +++ b/grype/db/v6/db_metadata_store_test.go @@ -0,0 +1,51 @@ +package v6 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +func TestDbMetadataStore_empty(t *testing.T) { + s := newDBMetadataStore(setupTestDB(t)) + + // attempt to fetch a non-existent record + actualMetadata, err := s.GetDBMetadata() + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + require.NotNil(t, actualMetadata) +} + +func TestDbMetadataStore(t *testing.T) { + s := newDBMetadataStore(setupTestDB(t)) + + require.NoError(t, s.SetDBMetadata()) + + // fetch the record + actualMetadata, err := s.GetDBMetadata() + require.NoError(t, err) + require.NotNil(t, actualMetadata) + + assert.NotZero(t, *actualMetadata.BuildTimestamp) // a timestamp was set + name, _ := actualMetadata.BuildTimestamp.Zone() + assert.Equal(t, "UTC", name) // the timestamp is in UTC + + actualMetadata.BuildTimestamp = nil // value not under test + + assert.Equal(t, DBMetadata{ + BuildTimestamp: nil, + // expect the correct version info + Model: ModelVersion, + Revision: Revision, + Addition: Addition, + }, *actualMetadata) +} + +func setupTestDB(t *testing.T) *gorm.DB { + // note: empty path means in-memory db + s, err := newStore(Config{}, true) + require.NoError(t, err) + + return s.db +} diff --git a/grype/db/v6/models.go b/grype/db/v6/models.go new file mode 100644 index 00000000000..098dbc2fceb --- /dev/null +++ b/grype/db/v6/models.go @@ -0,0 +1,19 @@ +package v6 + +import "time" + +func models() []any { + return []any{ + // non-domain info + &DBMetadata{}, + } +} + +// non-domain info ////////////////////////////////////////////////////// + +type DBMetadata struct { + BuildTimestamp *time.Time `gorm:"column:build_timestamp;not null"` + Model int `gorm:"column:model;not null"` + Revision int `gorm:"column:revision;not null"` + Addition int `gorm:"column:addition;not null"` +} diff --git a/grype/db/v6/store.go b/grype/db/v6/store.go new file mode 100644 index 00000000000..a95ffe9bf21 --- /dev/null +++ b/grype/db/v6/store.go @@ -0,0 +1,51 @@ +package v6 + +import ( + "fmt" + + "gorm.io/gorm" + + "github.com/anchore/grype/grype/db/internal/gormadapter" + "github.com/anchore/grype/internal/log" +) + +type store struct { + *dbMetadataStore + db *gorm.DB + config Config + write bool +} + +func newStore(cfg Config, write bool) (*store, error) { + db, err := gormadapter.Open(cfg.DBFilePath(), gormadapter.WithTruncate(write)) + if err != nil { + return nil, err + } + + if write { + if err := db.AutoMigrate(models()...); err != nil { + return nil, fmt.Errorf("unable to create tables: %w", err) + } + } + + return &store{ + dbMetadataStore: newDBMetadataStore(db), + db: db, + config: cfg, + write: write, + }, nil +} + +func (s *store) Close() error { + log.Debug("closing store") + if !s.write { + return nil + } + + err := s.db.Exec("VACUUM").Error + if err != nil { + return fmt.Errorf("failed to vacuum: %w", err) + } + + return nil +}