From bb02ef8dc7d12a19de9ccde8d909098bfac1e2dd Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Fri, 19 Jul 2024 17:01:13 +0000 Subject: [PATCH] feat(oci): support specifying the manifest digest when pushing it See https://github.com/opencontainers/distribution-spec/issues/494 and related work Signed-off-by: Andrei Aaron --- pkg/api/controller_test.go | 39 +--- pkg/api/routes.go | 56 +++++- pkg/api/routes_test.go | 20 +- pkg/extensions/search/search_test.go | 16 +- pkg/extensions/sync/destination.go | 10 +- pkg/extensions/sync/references/cosign.go | 2 +- pkg/extensions/sync/references/oci.go | 2 +- .../sync/references/referrers_tag.go | 6 +- pkg/extensions/sync/sync_internal_test.go | 6 +- pkg/meta/hooks.go | 2 +- pkg/storage/common/common.go | 60 ++++-- pkg/storage/common/common_test.go | 171 +++++++++++++++++- pkg/storage/gc/gc_internal_test.go | 6 +- pkg/storage/imagestore/imagestore.go | 12 +- pkg/storage/local/local_test.go | 24 +-- pkg/storage/s3/s3_test.go | 50 ++--- pkg/storage/scrub_test.go | 2 +- pkg/storage/storage_test.go | 97 +++++----- pkg/storage/types/types.go | 3 +- pkg/test/image-utils/upload.go | 3 + pkg/test/image-utils/write.go | 6 +- pkg/test/image-utils/write_test.go | 2 +- pkg/test/mocks/image_store_mock.go | 7 +- swagger/docs.go | 6 + swagger/swagger.json | 6 + swagger/swagger.yaml | 4 + 26 files changed, 417 insertions(+), 201 deletions(-) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index c8d84710d2..13b28e266d 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -11198,13 +11198,7 @@ func TestSupportedDigestAlgorithms(t *testing.T) { client := resty.New() - // The server picks canonical digests when tags are pushed - // See https://github.com/opencontainers/distribution-spec/issues/494 - // It would be nice to be able to push tags with other digest algorithms and verify those are returned - // but there is no way to specify a client preference - // so all we can do is verify the correct algorithm is returned - - expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String() + expectedDigestStr := image.DigestForAlgorithm(godigest.SHA512).String() verifyReturnedManifestDigest(t, client, baseURL, name, tag, expectedDigestStr) verifyReturnedManifestDigest(t, client, baseURL, name, expectedDigestStr, expectedDigestStr) @@ -11238,13 +11232,7 @@ func TestSupportedDigestAlgorithms(t *testing.T) { client := resty.New() - // The server picks canonical digests when tags are pushed - // See https://github.com/opencontainers/distribution-spec/issues/494 - // It would be nice to be able to push tags with other digest algorithms and verify those are returned - // but there is no way to specify a client preference - // so all we can do is verify the correct algorithm is returned - - expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String() + expectedDigestStr := image.DigestForAlgorithm(godigest.SHA384).String() verifyReturnedManifestDigest(t, client, baseURL, name, tag, expectedDigestStr) verifyReturnedManifestDigest(t, client, baseURL, name, expectedDigestStr, expectedDigestStr) @@ -11266,18 +11254,10 @@ func TestSupportedDigestAlgorithms(t *testing.T) { client := resty.New() - // The server picks canonical digests when tags are pushed - // See https://github.com/opencontainers/distribution-spec/issues/494 - // It would be nice to be able to push tags with other digest algorithms and verify those are returned - // but there is no way to specify a client preference - // so all we can do is verify the correct algorithm is returned - expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String() + expectedDigestStr := multiarch.DigestForAlgorithm(godigest.SHA512).String() verifyReturnedManifestDigest(t, client, baseURL, name, tag, expectedDigestStr) verifyReturnedManifestDigest(t, client, baseURL, name, expectedDigestStr, expectedDigestStr) - - // While the expected multiarch manifest digest is always using the canonical algorithm - // the sub-imgage manifest digest can use any algorith verifyReturnedManifestDigest(t, client, baseURL, name, subImage1.ManifestDescriptor.Digest.String(), subImage1.ManifestDescriptor.Digest.String()) verifyReturnedManifestDigest(t, client, baseURL, name, @@ -11303,9 +11283,6 @@ func TestSupportedDigestAlgorithms(t *testing.T) { expectedDigestStr := multiarch.DigestForAlgorithm(godigest.SHA512).String() verifyReturnedManifestDigest(t, client, baseURL, name, expectedDigestStr, expectedDigestStr) - - // While the expected multiarch manifest digest is always using the canonical algorithm - // the sub-imgage manifest digest can use any algorith verifyReturnedManifestDigest(t, client, baseURL, name, subImage1.ManifestDescriptor.Digest.String(), subImage1.ManifestDescriptor.Digest.String()) verifyReturnedManifestDigest(t, client, baseURL, name, @@ -11328,18 +11305,10 @@ func TestSupportedDigestAlgorithms(t *testing.T) { client := resty.New() - // The server picks canonical digests when tags are pushed - // See https://github.com/opencontainers/distribution-spec/issues/494 - // It would be nice to be able to push tags with other digest algorithms and verify those are returned - // but there is no way to specify a client preference - // so all we can do is verify the correct algorithm is returned - expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String() + expectedDigestStr := multiarch.DigestForAlgorithm(godigest.SHA384).String() verifyReturnedManifestDigest(t, client, baseURL, name, tag, expectedDigestStr) verifyReturnedManifestDigest(t, client, baseURL, name, expectedDigestStr, expectedDigestStr) - - // While the expected multiarch manifest digest is always using the canonical algorithm - // the sub-imgage manifest digest can use any algorith verifyReturnedManifestDigest(t, client, baseURL, name, subImage1.ManifestDescriptor.Digest.String(), subImage1.ManifestDescriptor.Digest.String()) verifyReturnedManifestDigest(t, client, baseURL, name, diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 1314f5f574..d221122a39 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -610,7 +610,9 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http digest, err := godigest.Parse(digestStr) if !ok || digestStr == "" || err != nil { - response.WriteHeader(http.StatusBadRequest) + details := map[string]string{"digest": digestStr} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } @@ -660,6 +662,7 @@ func (rh *RouteHandler) GetReferrers(response http.ResponseWriter, request *http // @Produce json // @Param name path string true "repository name" // @Param reference path string true "image reference or digest" +// @Param digest query string false "manifest digest" // @Header 201 {object} constants.DistContentDigestKey // @Success 201 {string} string "created" // @Failure 400 {string} string "bad request" @@ -686,6 +689,29 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht return } + var expectedDigest godigest.Digest + + // if a digest is present in the query parameters use it + digests, ok := request.URL.Query()["digest"] + if ok { + if len(digests) != 1 { + details := map[string]string{"digest": fmt.Sprintf("%v", digests)} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) + + return + } + + expectedDigest = godigest.Digest(digests[0]) + if err := expectedDigest.Validate(); err != nil { + details := map[string]string{"digest": digests[0]} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) + + return + } + } + mediaType := request.Header.Get("Content-Type") if !storageCommon.IsSupportedMediaType(mediaType) { err := apiErr.NewError(apiErr.MANIFEST_INVALID).AddDetail(map[string]string{"mediaType": mediaType}) @@ -704,7 +730,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht return } - digest, subjectDigest, err := imgStore.PutImageManifest(name, reference, mediaType, body) + digest, subjectDigest, err := imgStore.PutImageManifest(name, reference, mediaType, body, expectedDigest) if err != nil { details := zerr.GetDetails(err) if errors.Is(err, zerr.ErrRepoNotFound) { //nolint:gocritic // errorslint conflicts with gocritic:IfElseChain @@ -927,7 +953,9 @@ func (rh *RouteHandler) CheckBlob(response http.ResponseWriter, request *http.Re digestStr, ok := vars["digest"] if !ok || digestStr == "" { - response.WriteHeader(http.StatusNotFound) + details := map[string]string{"digest": digestStr} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } @@ -1068,7 +1096,9 @@ func (rh *RouteHandler) GetBlob(response http.ResponseWriter, request *http.Requ digestStr, ok := vars["digest"] if !ok || digestStr == "" { - response.WriteHeader(http.StatusNotFound) + details := map[string]string{"digest": digestStr} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } @@ -1176,7 +1206,9 @@ func (rh *RouteHandler) DeleteBlob(response http.ResponseWriter, request *http.R digest, err := godigest.Parse(digestStr) if !ok || digestStr == "" || err != nil { - response.WriteHeader(http.StatusNotFound) + details := map[string]string{"digest": digestStr} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } @@ -1309,7 +1341,9 @@ func (rh *RouteHandler) CreateBlobUpload(response http.ResponseWriter, request * digests, ok := request.URL.Query()["digest"] if ok { if len(digests) != 1 { - response.WriteHeader(http.StatusBadRequest) + details := map[string]string{"digest": fmt.Sprintf("%v", digests)} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } @@ -1555,7 +1589,7 @@ func (rh *RouteHandler) PatchBlobUpload(response http.ResponseWriter, request *h // @Param session_id path string true "upload session_id" // @Param digest query string true "blob/layer digest" // @Success 201 {string} string "created" -// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{digest}" +// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{session_id}" // @Header 200 {object} constants.DistContentDigestKey // @Failure 404 {string} string "not found" // @Failure 500 {string} string "internal server error" @@ -1581,14 +1615,18 @@ func (rh *RouteHandler) UpdateBlobUpload(response http.ResponseWriter, request * digests, ok := request.URL.Query()["digest"] if !ok || len(digests) != 1 { - response.WriteHeader(http.StatusBadRequest) + details := map[string]string{"digest": fmt.Sprintf("%v", digests)} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } digest, err := godigest.Parse(digests[0]) if err != nil { - response.WriteHeader(http.StatusBadRequest) + details := map[string]string{"digest": digests[0]} + e := apiErr.NewError(apiErr.DIGEST_INVALID).AddDetail(details) + zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) return } diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go index a4dd62c395..a3a4ebed2d 100644 --- a/pkg/api/routes_test.go +++ b/pkg/api/routes_test.go @@ -233,8 +233,8 @@ func TestRoutes(t *testing.T) { "reference": "reference", }, &mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", zerr.ErrRepoNotFound }, @@ -248,8 +248,8 @@ func TestRoutes(t *testing.T) { }, &mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", zerr.ErrManifestNotFound }, @@ -262,8 +262,8 @@ func TestRoutes(t *testing.T) { "reference": "reference", }, &mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", zerr.ErrBadManifest }, @@ -276,8 +276,8 @@ func TestRoutes(t *testing.T) { "reference": "reference", }, &mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", zerr.ErrBlobNotFound }, @@ -291,8 +291,8 @@ func TestRoutes(t *testing.T) { "reference": "reference", }, &mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", zerr.ErrRepoBadVersion }, diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 088ea83940..06691ca1dd 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -4552,8 +4552,8 @@ func TestMetaDBWhenSigningImages(t *testing.T) { Convey("imageIsSignature fails", func() { // make image store ignore the wrong format of the input ctlr.StoreController.DefaultStore = mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", nil }, @@ -5632,8 +5632,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { Convey("imageIsSignature fails", func() { ctlr.StoreController.DefaultStore = mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", nil }, @@ -5658,8 +5658,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { return configBlob, nil }, - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", nil }, @@ -5688,8 +5688,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { return configBlob, nil }, - PutImageManifestFn: func(repo, reference, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error, ) { return "", "", ErrTestError }, diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 400a73a6bf..24198a45c6 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -113,7 +113,7 @@ func (registry *DestinationRegistry) CommitImage(imageReference types.ImageRefer // is image manifest switch mediaType { case ispec.MediaTypeImageManifest: - if err := registry.copyManifest(repo, manifestBlob, reference, tempImageStore); err != nil { + if err := registry.copyManifest(repo, manifestBlob, reference, manifestDigest, tempImageStore); err != nil { if errors.Is(err, zerr.ErrImageLintAnnotations) { registry.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload manifest because of missing annotations") @@ -148,7 +148,7 @@ func (registry *DestinationRegistry) CommitImage(imageReference types.ImageRefer return err } - if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), + if err := registry.copyManifest(repo, manifestBuf, manifest.Digest.String(), manifest.Digest, tempImageStore); err != nil { if errors.Is(err, zerr.ErrImageLintAnnotations) { registry.log.Error().Str("errorType", common.TypeOf(err)). @@ -161,7 +161,7 @@ func (registry *DestinationRegistry) CommitImage(imageReference types.ImageRefer } } - _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob) + _, _, err = imageStore.PutImageManifest(repo, reference, mediaType, manifestBlob, manifestDigest) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference). Err(err).Msg("couldn't upload manifest") @@ -193,7 +193,7 @@ func (registry *DestinationRegistry) CleanupImage(imageReference types.ImageRefe } func (registry *DestinationRegistry) copyManifest(repo string, manifestContent []byte, reference string, - tempImageStore storageTypes.ImageStore, + manifestDigest digest.Digest, tempImageStore storageTypes.ImageStore, ) error { imageStore := registry.storeController.GetImageStore(repo) @@ -226,7 +226,7 @@ func (registry *DestinationRegistry) copyManifest(repo string, manifestContent [ } digest, _, err := imageStore.PutImageManifest(repo, reference, - ispec.MediaTypeImageManifest, manifestContent) + ispec.MediaTypeImageManifest, manifestContent, manifestDigest) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload manifest") diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index bb62ad7f5d..15156d900d 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -135,7 +135,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote // push manifest referenceDigest, _, err := imageStore.PutImageManifest(localRepo, cosignTag, - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, digest) if err != nil { ref.log.Error().Str("errorType", common.TypeOf(err)). Str("repository", localRepo).Str("subject", subjectDigestStr). diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 16c78f4d33..4e1c5c5fc9 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -224,7 +224,7 @@ func syncManifest(ctx context.Context, client *client.Client, imageStore storage } refDigest, _, err = imageStore.PutImageManifest(localRepo, desc.Digest.String(), - desc.MediaType, OCIRefBuf) + desc.MediaType, OCIRefBuf, desc.Digest) if err != nil { log.Error().Str("errorType", common.TypeOf(err)). Str("repository", localRepo).Str("subject", subjectDigestStr). diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go index 21ffd27551..567f81afc5 100644 --- a/pkg/extensions/sync/references/referrers_tag.go +++ b/pkg/extensions/sync/references/referrers_tag.go @@ -85,7 +85,9 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe return refsDigests, err } - skipTagRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, string(godigest.FromBytes(indexContent))) + indexDigest := godigest.FromBytes(indexContent) + + skipTagRefs, err := ref.canSkipReferences(localRepo, subjectDigestStr, indexDigest.String()) if err != nil { ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("couldn't check if the upstream index with referrers tag for image can be skipped") @@ -131,7 +133,7 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe referrersTag := getReferrersTagFromSubjectDigest(subjectDigestStr) - _, _, err = imageStore.PutImageManifest(localRepo, referrersTag, index.MediaType, indexContent) + _, _, err = imageStore.PutImageManifest(localRepo, referrersTag, index.MediaType, indexContent, indexDigest) if err != nil { ref.log.Error().Str("errorType", common.TypeOf(err)). Str("repository", localRepo).Str("subject", subjectDigestStr). diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index ed79e59e71..c3f77ef136 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -286,7 +286,7 @@ func TestDestinationRegistry(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -302,7 +302,7 @@ func TestDestinationRegistry(t *testing.T) { indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent, "") So(err, ShouldBeNil) Convey("sync index image", func() { @@ -462,7 +462,7 @@ func TestDestinationRegistry(t *testing.T) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) Convey("sync image", func() { diff --git a/pkg/meta/hooks.go b/pkg/meta/hooks.go index 74fe3e0dcc..d403a94e1c 100644 --- a/pkg/meta/hooks.go +++ b/pkg/meta/hooks.go @@ -80,7 +80,7 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, log.Info().Str("component", "metadb").Msg("restoring image store") // restore image store - _, _, err := imgStore.PutImageManifest(repo, reference, mediaType, manifestBlob) + _, _, err := imgStore.PutImageManifest(repo, reference, mediaType, manifestBlob, digest) if err != nil { log.Error().Err(err).Str("component", "metadb"). Msg("failed to restore manifest to image store, database is not consistent") diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 3565e857f4..7aa6db7f4f 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -153,29 +153,59 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy return nil } -// Returns the canonical digest or the digest provided by the reference if any -// Per spec, the canonical digest would always be returned to the client in -// request headers, but that does not make sense if the client requested a different digest algorithm -// See https://github.com/opencontainers/distribution-spec/issues/494 -func GetAndValidateRequestDigest(body []byte, reference string, log zlog.Logger) ( - godigest.Digest, error, +// Returns the digest for a specific byte slice. +// By default, if the expectedDigest is empty and the reference is a tag, the canonical algorithm is used. +// If the expectedDigest is not empty, that algorithm will be used and the value +// would be compared with the computed value. +// If the reference is a digest, that algorithm will be used and the value would be compared with the computed value. +// If both expectedDigest and reference are sent as digests, the values would be compared, +// and then compared with the computed value. +func GetAndValidateRequestDigest(body []byte, reference string, expectedDigest godigest.Digest, log zlog.Logger) ( + godigest.Digest, bool, error, ) { - expectedDigest, err := godigest.Parse(reference) - if err != nil { - // This is a non-digest reference - return godigest.Canonical.FromBytes(body), err + refIsDigest := false + + referenceDigest, err := godigest.Parse(reference) + if err == nil { + refIsDigest = true + } + + // compare provided digest values against one another before spending time computing the digest of the body + if refIsDigest && expectedDigest != "" { + if referenceDigest.String() != expectedDigest.String() { + log.Error().Str("reference digest", referenceDigest.String()).Str("query digest", expectedDigest.String()). + Msg("the digest provided in the image reference is different from the one provided as a query parameter") + + return expectedDigest, refIsDigest, zerr.ErrBadManifest + } } - actualDigest := expectedDigest.Algorithm().FromBytes(body) + var digestAlgorithm godigest.Algorithm + if expectedDigest != "" { //nolint:gocritic // code is more clear this way + digestAlgorithm = expectedDigest.Algorithm() + } else if refIsDigest { + digestAlgorithm = referenceDigest.Algorithm() + } else { + digestAlgorithm = godigest.Canonical + } + + actualDigest := digestAlgorithm.FromBytes(body) - if expectedDigest.String() != actualDigest.String() { + if expectedDigest != "" && expectedDigest.String() != actualDigest.String() { log.Error().Str("actual", actualDigest.String()).Str("expected", expectedDigest.String()). - Msg("failed to validate manifest digest") + Msg("the actual digest is different from the one provided as a query parameter") + + return actualDigest, refIsDigest, zerr.ErrBadManifest + } + + if refIsDigest && referenceDigest.String() != actualDigest.String() { + log.Error().Str("actual", actualDigest.String()).Str("reference", referenceDigest.String()). + Msg("the actual digest is different from the one provided in the image reference") - return actualDigest, zerr.ErrBadManifest + return actualDigest, refIsDigest, zerr.ErrBadManifest } - return actualDigest, nil + return actualDigest, refIsDigest, nil } /* diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index af340084b1..483b150d4b 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -26,6 +26,73 @@ import ( var ErrTestError = errors.New("TestError") +func TestValidateDigestComputation(t *testing.T) { + body := []byte{10, 20, 30, 40, 50} + sha256Digest := godigest.SHA256.FromBytes(body) + sha384Digest := godigest.SHA384.FromBytes(body) + sha512Digest := godigest.SHA512.FromBytes(body) + log := log.Logger{Logger: zerolog.New(os.Stdout)} + + Convey("Compute digest using default canonical algorithm", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, "sometag", "", log) + So(err, ShouldBeNil) + So(refIsDigest, ShouldBeFalse) + So(resultedDigest, ShouldEqual, sha256Digest) + }) + + Convey("Verify expected digest with no reference - positive", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, "sometag", sha512Digest, log) + So(err, ShouldBeNil) + So(refIsDigest, ShouldBeFalse) + So(resultedDigest, ShouldEqual, sha512Digest) + }) + + Convey("Verify expected digest with no reference - negative", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest([]byte{10, 20, 30, 40}, + "sometag", sha512Digest, log) + So(err, ShouldNotBeNil) + So(refIsDigest, ShouldBeFalse) + So(resultedDigest, ShouldEqual, godigest.SHA512.FromBytes([]byte{10, 20, 30, 40})) + }) + + Convey("Verify reference digest with no expected - positive", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, sha512Digest.String(), "", log) + So(err, ShouldBeNil) + So(refIsDigest, ShouldBeTrue) + So(resultedDigest, ShouldEqual, sha512Digest) + }) + + Convey("Verify reference digest with no expected - negative", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest([]byte{10, 20, 30, 40}, + sha512Digest.String(), "", log) + So(err, ShouldNotBeNil) + So(refIsDigest, ShouldBeTrue) + So(resultedDigest, ShouldEqual, godigest.SHA512.FromBytes([]byte{10, 20, 30, 40})) + }) + + Convey("Verify reference digest and expected digests - positive", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, sha512Digest.String(), sha512Digest, log) + So(err, ShouldBeNil) + So(refIsDigest, ShouldBeTrue) + So(resultedDigest, ShouldEqual, sha512Digest) + }) + + Convey("Verify reference digest and expected digests - mismatch between them", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, sha384Digest.String(), sha512Digest, log) + So(err, ShouldNotBeNil) + So(refIsDigest, ShouldBeTrue) + So(resultedDigest, ShouldEqual, sha512Digest) + }) + + Convey("Verify reference digest and expected digests - negative with actual", t, func(c C) { + resultedDigest, refIsDigest, err := common.GetAndValidateRequestDigest([]byte{10, 20, 30, 40}, + sha512Digest.String(), sha512Digest, log) + So(err, ShouldNotBeNil) + So(refIsDigest, ShouldBeTrue) + So(resultedDigest, ShouldEqual, godigest.SHA512.FromBytes([]byte{10, 20, 30, 40})) + }) +} + func TestValidateManifest(t *testing.T) { Convey("Make manifest", t, func(c C) { dir := t.TempDir() @@ -58,19 +125,19 @@ func TestValidateManifest(t *testing.T) { body, err := json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageConfig, body) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageConfig, body, "") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrBadManifest) }) Convey("empty manifest with bad media type", func() { - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageConfig, []byte("")) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageConfig, []byte(""), "") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrBadManifest) }) Convey("empty manifest with correct media type", func() { - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, []byte("")) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, []byte(""), "") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrBadManifest) }) @@ -96,7 +163,7 @@ func TestValidateManifest(t *testing.T) { body, err := json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body, "") So(err, ShouldNotBeNil) var internalErr *zerr.Error So(errors.As(err, &internalErr), ShouldBeTrue) @@ -131,7 +198,7 @@ func TestValidateManifest(t *testing.T) { So(err, ShouldBeNil) // this was actually an umoci error on config blob - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body, "") So(err, ShouldBeNil) }) @@ -160,8 +227,100 @@ func TestValidateManifest(t *testing.T) { body, err := json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body, "") + So(err, ShouldBeNil) + }) + + Convey("good manifest with bad expected digest", func() { + content := []byte("this blob doesn't exist") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + digest, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, + body, godigest.Canonical.FromString("123")) + So(err, ShouldNotBeNil) + So(digest, ShouldEqual, godigest.Canonical.FromBytes(body)) + }) + + Convey("good manifest with good expected digest", func() { + content := []byte("this blob doesn't exist") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + digest, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, + body, godigest.SHA512.FromBytes(body)) + So(err, ShouldBeNil) + So(digest, ShouldEqual, godigest.SHA512.FromBytes(body)) + }) + + Convey("good manifest with no expected digest", func() { + content := []byte("this blob doesn't exist") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + digest, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body, "") So(err, ShouldBeNil) + So(digest, ShouldEqual, godigest.Canonical.FromBytes(body)) }) }) } diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index ba535f1696..8f3b936399 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -121,7 +121,7 @@ func TestGarbageCollectManifestErrors(t *testing.T) { manifestDigest := godigest.FromBytes(body) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body, "") So(err, ShouldBeNil) Convey("trigger GetIndex error in GetReferencedBlobs", func() { @@ -236,7 +236,7 @@ func TestGarbageCollectIndexErrors(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -252,7 +252,7 @@ func TestGarbageCollectIndexErrors(t *testing.T) { indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent, "") So(err, ShouldBeNil) index, err = common.GetIndex(imgStore, repoName, log) diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 86ea2e761d..ca7194d6fd 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -453,7 +453,7 @@ func (is *ImageStore) GetImageManifest(repo, reference string) ([]byte, godigest // PutImageManifest adds an image manifest to the repository. func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, + body []byte, expectedDigest godigest.Digest, ) (godigest.Digest, godigest.Digest, error) { if err := is.InitRepo(repo); err != nil { is.log.Debug().Err(err).Msg("init repo") @@ -478,15 +478,9 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli } }() - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) + mDigest, refIsDigest, err := common.GetAndValidateRequestDigest(body, reference, expectedDigest, is.log) if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false + return mDigest, "", err } err = common.ValidateManifest(is, repo, reference, mediaType, body, is.log) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 1c5ae89b63..c388964f75 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -140,7 +140,7 @@ func TestStorageFSAPIs(t *testing.T) { panic(err) } - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldNotBeNil) err = os.Chmod(path.Join(imgStore.RootDir(), repoName, "index.json"), 0o755) @@ -148,7 +148,7 @@ func TestStorageFSAPIs(t *testing.T) { panic(err) } - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) manifestPath := path.Join(imgStore.RootDir(), repoName, "blobs", digest.Algorithm().String(), digest.Encoded()) @@ -174,7 +174,7 @@ func TestStorageFSAPIs(t *testing.T) { panic(err) } - _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldNotBeNil) err = os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755) if err != nil { @@ -356,7 +356,7 @@ func FuzzTestPutGetImageManifest(f *testing.F) { t.Errorf("Error %v occurred while marshaling manifest", err) } mdigest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repoName, mdigest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, mdigest.String(), ispec.MediaTypeImageManifest, manifestBuf, "") if err != nil && errors.Is(err, zerr.ErrBadManifest) { t.Errorf("the error that occurred is %v \n", err) } @@ -409,7 +409,7 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { t.Errorf("Error %v occurred while marshaling manifest", err) } mdigest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repoName, mdigest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, mdigest.String(), ispec.MediaTypeImageManifest, manifestBuf, "") if err != nil && errors.Is(err, zerr.ErrBadManifest) { t.Errorf("the error that occurred is %v \n", err) } @@ -1127,7 +1127,7 @@ func TestDedupeLinks(t *testing.T) { So(err, ShouldBeNil) manifestDigest := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String()) @@ -1186,7 +1186,7 @@ func TestDedupeLinks(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) @@ -2318,7 +2318,7 @@ func TestGarbageCollectErrors(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -2334,7 +2334,7 @@ func TestGarbageCollectErrors(t *testing.T) { indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent, "") So(err, ShouldBeNil) err = os.Chmod(imgStore.BlobPath(repoName, indexDigest), 0o000) @@ -2384,7 +2384,7 @@ func TestGarbageCollectErrors(t *testing.T) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) // trigger GetBlobContent error @@ -2442,10 +2442,10 @@ func TestGarbageCollectErrors(t *testing.T) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) // upload again same manifest so that we trigger manifest conflict - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) time.Sleep(500 * time.Millisecond) diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index a0ec1eb60d..25959990f9 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -511,7 +511,7 @@ func TestGetOCIReferrers(t *testing.T) { mbuflen := mbuf.Len() mdigest := godigest.FromBytes(mblob) - d, _, err := imgStore.PutImageManifest(repo, "1.0", ispec.MediaTypeImageManifest, mbuf.Bytes()) + d, _, err := imgStore.PutImageManifest(repo, "1.0", ispec.MediaTypeImageManifest, mbuf.Bytes(), "") So(d, ShouldEqual, mdigest) So(err, ShouldBeNil) @@ -565,7 +565,7 @@ func TestGetOCIReferrers(t *testing.T) { manBufLen := len(manBuf) manDigest := godigest.FromBytes(manBuf) - _, _, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeImageManifest, manBuf) + _, _, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeImageManifest, manBuf, "") So(err, ShouldBeNil) index, err := imgStore.GetReferrers(repo, mdigest, []string{artifactType}) @@ -835,7 +835,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { err = imgStore.DeleteImageManifest(testImage, "1.0", false) So(err, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(testImage, "1.0", "application/json", []byte{}) + _, _, err = imgStore.PutImageManifest(testImage, "1.0", "application/json", []byte{}, "") So(err, ShouldNotBeNil) _, err = imgStore.PutBlobChunkStreamed(testImage, upload, bytes.NewBufferString(testImage)) @@ -1209,7 +1209,7 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) manifestDigest := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String()) @@ -1283,7 +1283,7 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, - manifestBuf) + manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) @@ -1442,7 +1442,7 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, - manifestBuf) + manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String()) @@ -1614,7 +1614,7 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) manifestDigest := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String()) @@ -1680,7 +1680,7 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, - manifestBuf) + manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) @@ -1886,7 +1886,7 @@ func TestRebuildDedupeIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe1", digest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String()) @@ -1919,7 +1919,7 @@ func TestRebuildDedupeIndex(t *testing.T) { digest = godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", digest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) @@ -2950,7 +2950,7 @@ func TestS3ManifestImageIndex(t *testing.T) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) m1content := content - _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) // create another manifest but upload using its sha256 reference @@ -2993,7 +2993,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(digest, ShouldNotBeNil) m2dgst := digest m2size := len(content) - _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) Convey("Image index", func() { @@ -3033,7 +3033,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) var index ispec.Index @@ -3056,7 +3056,7 @@ func TestS3ManifestImageIndex(t *testing.T) { digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) index1dgst := digest - _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", "test:index1") So(err, ShouldBeNil) @@ -3099,7 +3099,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(digest, ShouldNotBeNil) m4dgst := digest m4size := len(content) - _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.SchemaVersion = 2 @@ -3120,7 +3120,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", "test:index2", ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", "test:index2", ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", "test:index2") So(err, ShouldBeNil) @@ -3149,7 +3149,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", "test:index3", ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", "test:index3", ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", "test:index3") So(err, ShouldBeNil) @@ -3170,7 +3170,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", digest.String()) So(err, ShouldBeNil) @@ -3248,7 +3248,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", digest.String()) So(err, ShouldBeNil) @@ -3266,7 +3266,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("index", "test:index1") So(err, ShouldBeNil) @@ -3319,11 +3319,11 @@ func TestS3ManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageIndex, content) + _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageIndex, content, "") So(err, ShouldNotBeNil) // previously an image index, try writing a manifest - _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageManifest, m1content) + _, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageManifest, m1content, "") So(err, ShouldNotBeNil) }) }) @@ -3389,7 +3389,7 @@ func TestS3ManifestImageIndex(t *testing.T) { So(m1digest, ShouldNotBeNil) m1size := len(content) - _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) // second config @@ -3422,7 +3422,7 @@ func TestS3ManifestImageIndex(t *testing.T) { m2digest := godigest.FromBytes(content) So(m2digest, ShouldNotBeNil) m2size := len(content) - _, _, err = imgStore.PutImageManifest("index", m2digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest("index", m2digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) Convey("Put image index with valid subject", func() { @@ -3448,7 +3448,7 @@ func TestS3ManifestImageIndex(t *testing.T) { idigest := godigest.FromBytes(content) So(idigest, ShouldNotBeNil) - digest1, digest2, err := imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content) + digest1, digest2, err := imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content, "") So(err, ShouldBeNil) So(digest1.String(), ShouldEqual, idigest.String()) So(digest2.String(), ShouldEqual, m1digest.String()) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 11c9f3922f..d3cb1d35f5 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -340,7 +340,7 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper indexBlob, err := json.Marshal(index) So(err, ShouldBeNil) - indexDigest, _, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob) + indexDigest, _, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob, "") So(err, ShouldBeNil) buff := bytes.NewBufferString("") diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 12aee9ac62..2d2fe64aa0 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -455,19 +455,19 @@ func TestStorageAPIs(t *testing.T) { Convey("Bad image manifest", func() { _, _, err = imgStore.PutImageManifest("test", digest.String(), "application/json", - manifestBuf) + manifestBuf, "") So(err, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, - []byte{}) + []byte{}, "") So(err, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, - []byte(`{"test":true}`)) + []byte(`{"test":true}`), "") So(err, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, - manifestBuf) + manifestBuf, "") So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetImageManifest("test", digest.String()) @@ -514,20 +514,20 @@ func TestStorageAPIs(t *testing.T) { badMb, err := json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, badMb) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, badMb, "") So(err, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) // same manifest for coverage - _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "2.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "2.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, err = imgStore.GetImageTags("inexistent") @@ -683,11 +683,11 @@ func TestStorageAPIs(t *testing.T) { Convey("Bad image manifest", func() { _, _, err = imgStore.PutImageManifest("test", digest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest("test", digest.String(), - ispec.MediaTypeImageManifest, []byte("bad json")) + ispec.MediaTypeImageManifest, []byte("bad json"), "") So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetImageManifest("test", digest.String()) @@ -722,12 +722,12 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) digest := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("test", digest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) // same manifest for coverage _, _, err = imgStore.PutImageManifest("test", digest.String(), - ispec.MediaTypeImageManifest, manifestBuf) + ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("test", digest.String()) @@ -819,7 +819,7 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("replace", digest.String()) @@ -870,7 +870,7 @@ func TestStorageAPIs(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) _ = godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("replace", "1.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) }) @@ -987,7 +987,7 @@ func TestMandatoryAnnotations(t *testing.T) { So(err, ShouldBeNil) Convey("Missing mandatory annotations", func() { - _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldNotBeNil) }) @@ -1017,7 +1017,7 @@ func TestMandatoryAnnotations(t *testing.T) { }, driver, cacheDriver) } - _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldNotBeNil) }) }) @@ -1174,7 +1174,7 @@ func TestDeleteBlobsInUse(t *testing.T) { manifestBuf, err := json.Marshal(manifest) So(err, ShouldBeNil) - manifestDigest, _, err := imgStore.PutImageManifest("repo", tag, ispec.MediaTypeImageManifest, manifestBuf) + manifestDigest, _, err := imgStore.PutImageManifest("repo", tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) Convey("Try to delete blob currently in use", func() { @@ -1296,7 +1296,7 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -1312,7 +1312,8 @@ func TestDeleteBlobsInUse(t *testing.T) { indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, indexContent) + indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, + indexContent, "") So(err, ShouldBeNil) Convey("Try to delete manifest being referenced by image index", func() { @@ -1718,7 +1719,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { So(err, ShouldBeNil) digest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) err = gc.CleanRepo(ctx, repoName) @@ -1762,7 +1763,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { // push artifact manifest _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) err = gc.CleanRepo(ctx, repoName) @@ -1908,7 +1909,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { So(err, ShouldBeNil) digest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) // put artifact referencing above image @@ -1949,7 +1950,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { // push artifact manifest _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) // push artifact manifest pointing to artifact above @@ -1965,7 +1966,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) // push orphan artifact (missing subject) @@ -1983,7 +1984,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { // push orphan artifact manifest _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) err = gc.CleanRepo(ctx, repoName) @@ -2040,7 +2041,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { Convey("Garbage collect - don't gc manifests/blobs which are referenced by another image", func() { // upload same image with another tag - _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) err = imgStore.DeleteImageManifest(repoName, tag, false) @@ -2166,7 +2167,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { manifestBuf, err := json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) // sleep so past GC timeout @@ -2229,7 +2230,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) @@ -2286,7 +2287,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { So(err, ShouldBeNil) digest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf, "") So(err, ShouldBeNil) err = gc.CleanRepo(ctx, repo2Name) @@ -2395,7 +2396,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { // push artifact manifest referencing index image _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) artifactManifest.Subject = &ispec.Descriptor{ @@ -2411,7 +2412,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { // push artifact manifest referencing a manifest from index image _, _, err = imgStore.PutImageManifest(repoName, artifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) err = gc.CleanRepo(ctx, repoName) @@ -2550,7 +2551,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { // push artifact manifest _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) artifactManifest.Subject = &ispec.Descriptor{ @@ -2567,7 +2568,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { // push artifact manifest referencing a manifest from index image _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + ispec.MediaTypeImageManifest, artifactManifestIndexBuf, "") So(err, ShouldBeNil) // push artifact manifest pointing to artifact above @@ -2583,7 +2584,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) // push orphan artifact (missing subject) @@ -2601,7 +2602,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { // push orphan artifact manifest _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) @@ -2858,7 +2859,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -2894,7 +2895,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { // push artifact manifest _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) } @@ -2940,7 +2941,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { So(err, ShouldBeNil) digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) innerIndex.Manifests = append(innerIndex.Manifests, ispec.Descriptor{ @@ -2957,7 +2958,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { So(innerIndexDigest, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest(repoName, innerIndexDigest.String(), - ispec.MediaTypeImageIndex, innerIndexContent) + ispec.MediaTypeImageIndex, innerIndexContent, "") So(err, ShouldBeNil) // add inner index into root index @@ -2974,7 +2975,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent, "") So(err, ShouldBeNil) artifactManifest := ispec.Manifest{ @@ -3003,7 +3004,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { // push artifact manifest _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) artifactManifest.Subject = &ispec.Descriptor{ @@ -3020,7 +3021,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { // push artifact manifest referencing a manifest from index image _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + ispec.MediaTypeImageManifest, artifactManifestIndexBuf, "") So(err, ShouldBeNil) artifactManifest.Subject = &ispec.Descriptor{ @@ -3037,7 +3038,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { // push artifact manifest referencing a manifest from index image _, _, err = imgStore.PutImageManifest(repoName, artifactManifestInnerIndexDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestInnerIndexBuf) + ispec.MediaTypeImageManifest, artifactManifestInnerIndexBuf, "") So(err, ShouldBeNil) // push artifact manifest pointing to artifact above @@ -3054,7 +3055,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) // push orphan artifact (missing subject) @@ -3072,7 +3073,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { // push orphan artifact manifest _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), - ispec.MediaTypeImageManifest, artifactManifestBuf) + ispec.MediaTypeImageManifest, artifactManifestBuf, "") So(err, ShouldBeNil) hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) @@ -3243,7 +3244,7 @@ func pushRandomImageIndex(imgStore storageTypes.ImageStore, repoName string, So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content, "") So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ @@ -3259,7 +3260,7 @@ func pushRandomImageIndex(imgStore storageTypes.ImageStore, repoName string, indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) - _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent, "") So(err, ShouldBeNil) return bdgst, digest, indexDigest, int64(len(indexContent)) diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index b4e2448826..2de428053a 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -32,7 +32,8 @@ type ImageStore interface { //nolint:interfacebloat GetNextRepository(repo string) (string, error) GetImageTags(repo string) ([]string, error) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) - PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) + PutImageManifest(repo, reference, mediaType string, body []byte, expectedDigest godigest.Digest, + ) (godigest.Digest, godigest.Digest, error) DeleteImageManifest(repo, reference string, detectCollision bool) error BlobUploadPath(repo, uuid string) string NewBlobUpload(repo string) (string, error) diff --git a/pkg/test/image-utils/upload.go b/pkg/test/image-utils/upload.go index dcf1a7d626..3bbb148aa7 100644 --- a/pkg/test/image-utils/upload.go +++ b/pkg/test/image-utils/upload.go @@ -117,6 +117,7 @@ func UploadImage(img Image, baseURL, repo, ref string) error { resp, err = resty.R(). SetHeader("Content-type", ispec.MediaTypeImageManifest). SetBody(manifestBlob). + SetQueryParam("digest", img.DigestStr()). Put(baseURL + "/v2/" + repo + "/manifests/" + ref) if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated { @@ -217,6 +218,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, ref, user, password stri SetBasicAuth(user, password). SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). SetBody(manifestBlob). + SetQueryParam("digest", img.DigestStr()). Put(baseURL + "/v2/" + repo + "/manifests/" + ref) return err @@ -245,6 +247,7 @@ func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo, ref s resp, err := resty.R(). SetHeader("Content-type", ispec.MediaTypeImageIndex). SetBody(indexBlob). + SetQueryParam("digest", multiImage.DigestStr()). Put(baseURL + "/v2/" + repo + "/manifests/" + ref) if resp.StatusCode() != http.StatusCreated { diff --git a/pkg/test/image-utils/write.go b/pkg/test/image-utils/write.go index 2bf083e1f8..9235b57081 100644 --- a/pkg/test/image-utils/write.go +++ b/pkg/test/image-utils/write.go @@ -52,7 +52,8 @@ func WriteImageToFileSystem(image Image, repoName, ref string, storeController s return err } - _, _, err = store.PutImageManifest(repoName, ref, ispec.MediaTypeImageManifest, manifestBlob) + // we should revisit the posibility of supporting a non-canonical manifest digest in this function + _, _, err = store.PutImageManifest(repoName, ref, ispec.MediaTypeImageManifest, manifestBlob, "") if err != nil { return err } @@ -82,8 +83,9 @@ func WriteMultiArchImageToFileSystem(multiarchImage MultiarchImage, repoName, re return err } + // we should revisit the posibility of supporting a non-canonical manifest digest in this function _, _, err = store.PutImageManifest(repoName, ref, ispec.MediaTypeImageIndex, - indexBlob) + indexBlob, "") return err } diff --git a/pkg/test/image-utils/write_test.go b/pkg/test/image-utils/write_test.go index f8df7813ca..db5473ed8a 100644 --- a/pkg/test/image-utils/write_test.go +++ b/pkg/test/image-utils/write_test.go @@ -67,7 +67,7 @@ func TestWriteImageToFileSystem(t *testing.T) { "tag", storage.StoreController{ DefaultStore: mocks.MockedImageStore{ - PutImageManifestFn: func(repo, reference, mediaType string, body []byte, + PutImageManifestFn: func(repo, reference, mediaType string, body []byte, expectedDigest godigest.Digest, ) (godigest.Digest, godigest.Digest, error) { return "", "", ErrTestError }, diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index ff18afab32..db111b3048 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -21,8 +21,8 @@ type MockedImageStore struct { GetNextRepositoryFn func(repo string) (string, error) GetImageTagsFn func(repo string) ([]string, error) GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error) - PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest, - godigest.Digest, error) + PutImageManifestFn func(repo string, reference string, mediaType string, body []byte, + expectedDigest godigest.Digest) (godigest.Digest, godigest.Digest, error) DeleteImageManifestFn func(repo string, reference string, detectCollision bool) error BlobUploadPathFn func(repo string, uuid string) string NewBlobUploadFn func(repo string) (string, error) @@ -149,9 +149,10 @@ func (is MockedImageStore) PutImageManifest( reference string, mediaType string, body []byte, + expectedDigest godigest.Digest, ) (godigest.Digest, godigest.Digest, error) { if is.PutImageManifestFn != nil { - return is.PutImageManifestFn(repo, reference, mediaType, body) + return is.PutImageManifestFn(repo, reference, mediaType, body, expectedDigest) } return "", "", nil diff --git a/swagger/docs.go b/swagger/docs.go index 95e6089c79..95ce9f4cb1 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -734,6 +734,12 @@ const docTemplate = `{ "name": "reference", "in": "path", "required": true + }, + { + "type": "string", + "description": "manifest digest", + "name": "digest", + "in": "query" } ], "responses": { diff --git a/swagger/swagger.json b/swagger/swagger.json index dee0001522..6eeaf37177 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -726,6 +726,12 @@ "name": "reference", "in": "path", "required": true + }, + { + "type": "string", + "description": "manifest digest", + "name": "digest", + "in": "query" } ], "responses": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 15e46c1953..f52d626fd6 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -780,6 +780,10 @@ paths: name: reference required: true type: string + - description: manifest digest + in: query + name: digest + type: string produces: - application/json responses: