all: account for language package overwrites

gobin/ecosystem.go

@@ -4,17 +4,18 @@ import (
 	"context"
 
 	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/language"
 )
 
 // NewEcosystem provides the ecosystem for handling go binaries.
-func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
+func NewEcosystem(_ context.Context) *indexer.Ecosystem {
 	return &indexer.Ecosystem{
 		Name: "gobin",
 		PackageScanners: func(context.Context) ([]indexer.PackageScanner, error) {
 			return []indexer.PackageScanner{Detector{}}, nil
 		},
 		DistributionScanners: func(context.Context) ([]indexer.DistributionScanner, error) { return nil, nil },
 		RepositoryScanners:   func(context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil },
-		Coalescer:            func(context.Context) (indexer.Coalescer, error) { return &coalescer{}, nil },
+		Coalescer:            language.NewCoalescer,
 	}
 }

gobin/gobin.go

@@ -35,7 +35,7 @@ type Detector struct{}
 
 const (
 	detectorName    = `gobin`
-	detectorVersion = `5`
+	detectorVersion = `6`
 	detectorKind    = `package`
 )
 

java/ecosystem.go

@@ -4,10 +4,11 @@ import (
 	"context"
 
 	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/language"
 )
 
 // NewEcosystem provides the set of scanners for the java ecosystem.
-func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
+func NewEcosystem(_ context.Context) *indexer.Ecosystem {
 	return &indexer.Ecosystem{
 		PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
 			return []indexer.PackageScanner{&Scanner{}}, nil
@@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
 		RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) {
 			return nil, nil
 		},
-		Coalescer: func(_ context.Context) (indexer.Coalescer, error) {
-			return (*coalescer)(nil), nil
-		},
+		Coalescer: language.NewCoalescer,
 	}
 }

java/packagescanner.go

@@ -71,7 +71,7 @@ type Scanner struct {
 func (*Scanner) Name() string { return "java" }
 
 // Version implements scanner.VersionedScanner.
-func (*Scanner) Version() string { return "6" }
+func (*Scanner) Version() string { return "7" }
 
 // Kind implements scanner.VersionedScanner.
 func (*Scanner) Kind() string { return "package" }

language/coalescer.go

@@ -0,0 +1,74 @@
+// Package language implements structs and functions common between
+// programming language indexing implementations.
+package language
+
+import (
+	"context"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/indexer"
+)
+
+var _ indexer.Coalescer = (*coalescer)(nil)
+
+type coalescer struct{}
+
+// NewCoalescer returns a new common programming language coalescer.
+func NewCoalescer(_ context.Context) (indexer.Coalescer, error) {
+	return &coalescer{}, nil
+}
+
+// Coalesce implements [indexer.Coalescer].
+//
+// Image builders may opt to update language-packages instead of deleting and recreating them
+// (as in, there may or may not be a whiteout file to make it clear the package was updated).
+// This function ensures both scenarios are supported.
+func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) {
+	ir := &claircore.IndexReport{
+		Environments: map[string][]*claircore.Environment{},
+		Packages:     map[string]*claircore.Package{},
+		Repositories: map[string]*claircore.Repository{},
+	}
+	// Similar to ir.Packages, except instead of mapping
+	// id -> package, it maps packageDB -> package.
+	// For language packages, it is possible the
+	// packageDB is overwritten between subsequent layers.
+	packages := make(map[string]*claircore.Package)
+	for i := len(ls) - 1; i >= 0; i-- {
+		l := ls[i]
+		// If we didn't find at least one repo in this layer
+		// no point searching for packages.
+		if len(l.Repos) == 0 {
+			continue
+		}
+		rs := make([]string, len(l.Repos))
+		for i, r := range l.Repos {
+			rs[i] = r.ID
+			ir.Repositories[r.ID] = r
+		}
+		for _, pkg := range l.Pkgs {
+			if seen, exists := packages[pkg.PackageDB]; exists {
+				// If the package was renamed or has a different version in a higher (previously seen) layer,
+				// then this is considered a different package.
+				// In that case, ignore the original package in the lower (this) layer.
+				if pkg.Name != seen.Name || pkg.Version != seen.Version {
+					continue
+				}
+				// The name and version are the same, so delete the entry related to the higher (previously seen)
+				// layer, as this package was likely introduced in the lower (this) layer.
+				delete(ir.Packages, seen.ID)
+				delete(ir.Environments, seen.ID)
+			}
+			packages[pkg.PackageDB] = pkg
+			ir.Packages[pkg.ID] = pkg
+			ir.Environments[pkg.ID] = []*claircore.Environment{
+				{
+					PackageDB:     pkg.PackageDB,
+					IntroducedIn:  l.Hash,
+					RepositoryIDs: rs,
+				},
+			}
+		}
+	}
+	return ir, nil
+}

language/coalescer_test.go

@@ -0,0 +1,155 @@
+package language
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/test"
+)
+
+func TestCoalescer(t *testing.T) {
+	t.Parallel()
+	ctx := zlog.Test(context.Background(), t)
+	coalescer := &coalescer{}
+	pkgs := test.GenUniquePackages(6)
+	repo := []*claircore.Repository{{
+		Name: "npm",
+		URI:  "https://www.npmjs.com/",
+	}}
+	layerArtifacts := []*indexer.LayerArtifacts{
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:1],
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:2],
+		},
+		{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  pkgs[:3],
+			Repos: repo,
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs[:4],
+		},
+		{
+			Hash:  test.RandomSHA256Digest(t),
+			Pkgs:  pkgs[:5],
+			Repos: repo,
+		},
+		{
+			Hash: test.RandomSHA256Digest(t),
+			Pkgs: pkgs,
+		},
+	}
+	ir, err := coalescer.Coalesce(ctx, layerArtifacts)
+	if err != nil {
+		t.Fatalf("received error from coalesce method: %v", err)
+	}
+	// Expect 0-5 to have gotten associated with the repository.
+	for i := range pkgs {
+		es, ok := ir.Environments[strconv.Itoa(i)]
+		if !ok && i == 5 {
+			// Left out the last package.
+			continue
+		}
+		e := es[0]
+		if len(e.RepositoryIDs) == 0 {
+			t.Error("expected some repositories")
+		}
+		for _, id := range e.RepositoryIDs {
+			r := ir.Repositories[id]
+			if got, want := r.Name, "npm"; got != want {
+				t.Errorf("got: %q, want: %q", got, want)
+			}
+		}
+	}
+}
+
+func TestCoalescerPackageOverwrite(t *testing.T) {
+	t.Parallel()
+	ctx := zlog.Test(context.Background(), t)
+	coalescer := &coalescer{}
+	repo := []*claircore.Repository{{
+		Name: "npm",
+		URI:  "https://www.npmjs.com/",
+	}}
+	hashes := []claircore.Digest{
+		test.RandomSHA256Digest(t),
+		test.RandomSHA256Digest(t),
+		test.RandomSHA256Digest(t),
+		test.RandomSHA256Digest(t),
+	}
+	layerArtifacts := []*indexer.LayerArtifacts{
+		{
+			Hash: hashes[0],
+			Pkgs: []*claircore.Package{
+				{
+					ID:        "0",
+					Name:      "semver",
+					Version:   "7.3.8",
+					PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
+				},
+			},
+			Repos: repo,
+		},
+		{
+			Hash: hashes[1],
+		},
+		{
+			Hash: hashes[2],
+			Pkgs: []*claircore.Package{
+				{
+					ID:        "1",
+					Name:      "semver",
+					Version:   "7.5.2",
+					PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
+				},
+			},
+			Repos: repo,
+		},
+		{
+			Hash: hashes[3],
+			Pkgs: []*claircore.Package{
+				{
+					ID:        "2",
+					Name:      "semver",
+					Version:   "7.5.2",
+					PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
+				},
+			},
+			Repos: repo,
+		},
+	}
+	ir, err := coalescer.Coalesce(ctx, layerArtifacts)
+	if err != nil {
+		t.Fatalf("received error from coalesce method: %v", err)
+	}
+	if len(ir.Packages) != 1 {
+		t.Fatalf("unexpected number of packages: %d != %d", len(ir.Packages), 1)
+	}
+	pkg, exists := ir.Packages["1"]
+	if !exists {
+		t.Fatal("expected package does not exist")
+	}
+	if pkg.Version != "7.5.2" {
+		t.Fatalf("unexpected version: %s != %s", pkg.Version, "7.5.2")
+	}
+	envs, exists := ir.Environments["1"]
+	if !exists {
+		t.Fatal("expected environments do not exist")
+	}
+	if len(envs) != 1 {
+		t.Fatalf("unexpected number of envionments: %d != %d", len(envs), 1)
+	}
+	if envs[0].IntroducedIn.String() != hashes[2].String() {
+		t.Fatalf("unexpected introducedIn: %s != %s", envs[0].IntroducedIn.String(), hashes[2].String())
+	}
+}