Add v6 DB metadata store (#2146)

* add db metadata store

Signed-off-by: Alex Goodman <[email protected]>

* add functional options to gorm adapter

Signed-off-by: Alex Goodman <[email protected]>

---------

Signed-off-by: Alex Goodman <[email protected]>

grype/db/internal/gormadapter/open.go

@@ -13,6 +13,8 @@ var writerStatements = []string{
 	// on my box it reduces the time to write from 10 minutes to 10 seconds (with ~1GB memory utilization spikes)
 	`PRAGMA synchronous = OFF`,
 	`PRAGMA journal_mode = MEMORY`,
+	`PRAGMA cache_size = 100000`,
+	`PRAGMA mmap_size = 268435456`, // 256 MB
 }
 
 var readOptions = []string{
@@ -21,31 +23,73 @@ var readOptions = []string{
 	"mode=ro",
 }
 
-// Open a new connection to a sqlite3 database file
-func Open(path string, write bool) (*gorm.DB, error) {
-	if write {
-		// the file may or may not exist, so we ignore the error explicitly
-		_ = os.Remove(path)
+type config struct {
+	path   string
+	write  bool
+	memory bool
+}
+
+type Option func(*config)
+
+func WithTruncate(truncate bool) Option {
+	return func(c *config) {
+		c.write = truncate
 	}
+}
 
-	connStr, err := connectionString(path)
-	if err != nil {
-		return nil, err
+func newConfig(path string, opts []Option) config {
+	c := config{}
+	c.apply(path, opts)
+	return c
+}
+
+func (c *config) apply(path string, opts []Option) {
+	for _, o := range opts {
+		o(c)
+	}
+	c.memory = len(path) == 0
+	c.path = path
+}
+
+func (c config) shouldTruncate() bool {
+	return c.write && !c.memory
+}
+
+func (c config) connectionString() string {
+	var conn string
+	if c.path == "" {
+		conn = ":memory:"
+	} else {
+		conn = fmt.Sprintf("file:%s?cache=shared", c.path)
 	}
 
-	if !write {
+	if !c.write && !c.memory {
 		// &immutable=1&cache=shared&mode=ro
 		for _, o := range readOptions {
-			connStr += fmt.Sprintf("&%s", o)
+			conn += fmt.Sprintf("&%s", o)
+		}
+	}
+	return conn
+}
+
+// Open a new connection to a sqlite3 database file
+func Open(path string, options ...Option) (*gorm.DB, error) {
+	cfg := newConfig(path, options)
+
+	if cfg.shouldTruncate() {
+		if _, err := os.Stat(path); err == nil {
+			if err := os.Remove(path); err != nil {
+				return nil, fmt.Errorf("unable to remove existing DB file: %w", err)
+			}
 		}
 	}
 
-	dbObj, err := gorm.Open(sqlite.Open(connStr), &gorm.Config{Logger: newLogger()})
+	dbObj, err := gorm.Open(sqlite.Open(cfg.connectionString()), &gorm.Config{Logger: newLogger()})
 	if err != nil {
 		return nil, fmt.Errorf("unable to connect to DB: %w", err)
 	}
 
-	if write {
+	if cfg.write {
 		for _, sqlStmt := range writerStatements {
 			dbObj.Exec(sqlStmt)
 			if dbObj.Error != nil {
@@ -54,13 +98,8 @@ func Open(path string, write bool) (*gorm.DB, error) {
 		}
 	}
 
-	return dbObj, nil
-}
+	// needed for v6+
+	dbObj.Exec("PRAGMA foreign_keys = ON")
 
-// ConnectionString creates a connection string for sqlite3
-func connectionString(path string) (string, error) {
-	if path == "" {
-		return "", fmt.Errorf("no db filepath given")
-	}
-	return fmt.Sprintf("file:%s?cache=shared", path), nil
+	return dbObj, nil
 }

grype/db/internal/gormadapter/open_test.go

@@ -0,0 +1,127 @@
+package gormadapter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfigApply(t *testing.T) {
+	tests := []struct {
+		name           string
+		path           string
+		options        []Option
+		expectedPath   string
+		expectedMemory bool
+	}{
+		{
+			name:           "apply with path",
+			path:           "test.db",
+			options:        []Option{},
+			expectedPath:   "test.db",
+			expectedMemory: false,
+		},
+		{
+			name:           "apply with empty path (memory)",
+			path:           "",
+			options:        []Option{},
+			expectedPath:   "",
+			expectedMemory: true,
+		},
+		{
+			name:           "apply with truncate option",
+			path:           "test.db",
+			options:        []Option{WithTruncate(true)},
+			expectedPath:   "test.db",
+			expectedMemory: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := newConfig(tt.path, tt.options)
+
+			require.Equal(t, tt.expectedPath, c.path)
+			require.Equal(t, tt.expectedMemory, c.memory)
+		})
+	}
+}
+
+func TestConfigShouldTruncate(t *testing.T) {
+	tests := []struct {
+		name             string
+		write            bool
+		memory           bool
+		expectedTruncate bool
+	}{
+		{
+			name:             "should truncate when write is true and not memory",
+			write:            true,
+			memory:           false,
+			expectedTruncate: true,
+		},
+		{
+			name:             "should not truncate when write is false",
+			write:            false,
+			memory:           false,
+			expectedTruncate: false,
+		},
+		{
+			name:             "should not truncate when using in-memory DB",
+			write:            true,
+			memory:           true,
+			expectedTruncate: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := config{
+				write:  tt.write,
+				memory: tt.memory,
+			}
+			require.Equal(t, tt.expectedTruncate, c.shouldTruncate())
+		})
+	}
+}
+
+func TestConfigConnectionString(t *testing.T) {
+	tests := []struct {
+		name            string
+		path            string
+		write           bool
+		memory          bool
+		expectedConnStr string
+	}{
+		{
+			name:            "writable path",
+			path:            "test.db",
+			write:           true,
+			expectedConnStr: "file:test.db?cache=shared",
+		},
+		{
+			name:            "read-only path",
+			path:            "test.db",
+			write:           false,
+			expectedConnStr: "file:test.db?cache=shared&immutable=1&cache=shared&mode=ro",
+		},
+		{
+			name:            "in-memory mode",
+			path:            "",
+			write:           false,
+			memory:          true,
+			expectedConnStr: ":memory:",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := config{
+				path:   tt.path,
+				write:  tt.write,
+				memory: tt.memory,
+			}
+			require.Equal(t, tt.expectedConnStr, c.connectionString())
+		})
+	}
+}

grype/db/v1/store/store.go

@@ -21,7 +21,7 @@ type store struct {
 
 // New creates a new instance of the store.
 func New(dbFilePath string, overwrite bool) (v1.Store, error) {
-	db, err := gormadapter.Open(dbFilePath, overwrite)
+	db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite))
 	if err != nil {
 		return nil, err
 	}

grype/db/v2/store/store.go

@@ -21,7 +21,7 @@ type store struct {
 
 // New creates a new instance of the store.
 func New(dbFilePath string, overwrite bool) (v2.Store, error) {
-	db, err := gormadapter.Open(dbFilePath, overwrite)
+	db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite))
 	if err != nil {
 		return nil, err
 	}

grype/db/v3/store/store.go

@@ -21,7 +21,7 @@ type store struct {
 
 // New creates a new instance of the store.
 func New(dbFilePath string, overwrite bool) (v3.Store, error) {
-	db, err := gormadapter.Open(dbFilePath, overwrite)
+	db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite))
 	if err != nil {
 		return nil, err
 	}

grype/db/v4/store/store.go

@@ -21,7 +21,7 @@ type store struct {
 
 // New creates a new instance of the store.
 func New(dbFilePath string, overwrite bool) (v4.Store, error) {
-	db, err := gormadapter.Open(dbFilePath, overwrite)
+	db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite))
 	if err != nil {
 		return nil, err
 	}

grype/db/v5/store/store.go

@@ -21,7 +21,7 @@ type store struct {
 
 // New creates a new instance of the store.
 func New(dbFilePath string, overwrite bool) (v5.Store, error) {
-	db, err := gormadapter.Open(dbFilePath, overwrite)
+	db, err := gormadapter.Open(dbFilePath, gormadapter.WithTruncate(overwrite))
 	if err != nil {
 		return nil, err
 	}

grype/db/v6/db.go

@@ -0,0 +1,52 @@
+package v6
+
+import (
+	"io"
+	"path/filepath"
+)
+
+const (
+	VulnerabilityDBFileName = "vulnerability.db"
+
+	// We follow SchemaVer semantics (see https://snowplow.io/blog/introducing-schemaver-for-semantic-versioning-of-schemas)
+
+	// ModelVersion indicates how many breaking schema changes there have been (which will prevent interaction with any historical data)
+	// note: this must ALWAYS be "6" in the context of this package.
+	ModelVersion = 6
+
+	// Revision indicates how many changes have been introduced which **may** prevent interaction with some historical data
+	Revision = 0
+
+	// Addition indicates how many changes have been introduced that are compatible with all historical data
+	Addition = 0
+)
+
+type ReadWriter interface {
+	Reader
+	Writer
+}
+
+type Reader interface {
+	DBMetadataStoreReader
+}
+
+type Writer interface {
+	DBMetadataStoreWriter
+	io.Closer
+}
+
+type Config struct {
+	DBDirPath string
+}
+
+func (c *Config) DBFilePath() string {
+	return filepath.Join(c.DBDirPath, VulnerabilityDBFileName)
+}
+
+func NewReader(cfg Config) (Reader, error) {
+	return newStore(cfg, false)
+}
+
+func NewWriter(cfg Config) (ReadWriter, error) {
+	return newStore(cfg, true)
+}

grype/db/v6/db_metadata_store.go

@@ -0,0 +1,59 @@
+package v6
+
+import (
+	"fmt"
+	"time"
+
+	"gorm.io/gorm"
+
+	"github.com/anchore/grype/internal/log"
+)
+
+type DBMetadataStoreWriter interface {
+	SetDBMetadata() error
+}
+
+type DBMetadataStoreReader interface {
+	GetDBMetadata() (*DBMetadata, error)
+}
+
+type dbMetadataStore struct {
+	db *gorm.DB
+}
+
+func newDBMetadataStore(db *gorm.DB) *dbMetadataStore {
+	return &dbMetadataStore{
+		db: db,
+	}
+}
+
+func (s *dbMetadataStore) GetDBMetadata() (*DBMetadata, error) {
+	log.Trace("fetching DB metadata record")
+
+	var model DBMetadata
+
+	result := s.db.First(&model)
+	return &model, result.Error
+}
+
+func (s *dbMetadataStore) SetDBMetadata() error {
+	log.Trace("writing DB metadata record")
+
+	if err := s.db.Unscoped().Where("true").Delete(&DBMetadata{}).Error; err != nil {
+		return fmt.Errorf("failed to delete existing DB metadata record: %w", err)
+	}
+
+	ts := time.Now().UTC()
+	instance := &DBMetadata{
+		BuildTimestamp: &ts,
+		Model:          ModelVersion,
+		Revision:       Revision,
+		Addition:       Addition,
+	}
+
+	if err := s.db.Create(instance).Error; err != nil {
+		return fmt.Errorf("failed to create DB metadata record: %w", err)
+	}
+
+	return nil
+}