Add SSE and refactor encryption API (#942)

* refactor encryption API

This commit does the following changes:

 - Remove all client-side-encryption code
 - Add unified server-side-encryption interface
 - Implement SSE-C and SSE-S3
   (SSE-KMS needs a bit more investigation to finally decide how to implement)
 - Remove special encryption API calls like Get/PutEncryptedObject
   (They are not really usefull after the encryption refactoring)
 - Update functional tests. The functional tests use the new API now.
 - Update API doc and increment version number to 5.0.0
 - Update S3 encryption examples

The client-side-encryption code is removed because it was hardly used,
poorly designed/specified by AWS and has no advantages compared to SSE
for allmost all uses-cases we encounter.

The SSE implementation is located at the `encrypt` package. It exposes
and abstract and easy to use API. Currently the `encrypt` package
implements SSE-C and SSE-S3. SSE-KMS is not yet implemented because some
more investigation is needed to be aware of all details/requirements/implications
of the encryption context which can be passed to SSE-KMS.

All encryption-specific client APIs - like Get/PutEncryptedObject - are
removed because there is no real benefit in keeping them. SSE can simply
be specified as part of Get/Put/Stat ObjectOptions.

The functional tests are updated.

The API documentations is updated.

The S3 encryption examples are updated.
The new examples use the new encryption API.

Fixes #882

README.md

@@ -156,10 +156,6 @@ The full API Reference is available here.
 * [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects)
 * [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload)
 
-### API Reference: Encrypted Object Operations
-* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject)
-* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject)
-
 ### API Reference : Presigned Operations
 * [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject)
 * [`PresignedPutObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedPutObject)

api-compose-object.go

@@ -19,66 +19,22 @@ package minio
 
 import (
 	"context"
-	"encoding/base64"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
 )
 
-// SSEInfo - represents Server-Side-Encryption parameters specified by
-// a user.
-type SSEInfo struct {
-	key  []byte
-	algo string
-}
-
-// NewSSEInfo - specifies (binary or un-encoded) encryption key and
-// algorithm name. If algo is empty, it defaults to "AES256". Ref:
-// https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
-func NewSSEInfo(key []byte, algo string) SSEInfo {
-	if algo == "" {
-		algo = "AES256"
-	}
-	return SSEInfo{key, algo}
-}
-
-// internal method that computes SSE-C headers
-func (s *SSEInfo) getSSEHeaders(isCopySource bool) map[string]string {
-	if s == nil {
-		return nil
-	}
-
-	cs := ""
-	if isCopySource {
-		cs = "copy-source-"
-	}
-	return map[string]string{
-		"x-amz-" + cs + "server-side-encryption-customer-algorithm": s.algo,
-		"x-amz-" + cs + "server-side-encryption-customer-key":       base64.StdEncoding.EncodeToString(s.key),
-		"x-amz-" + cs + "server-side-encryption-customer-key-MD5":   sumMD5Base64(s.key),
-	}
-}
-
-// GetSSEHeaders - computes and returns headers for SSE-C as key-value
-// pairs. They can be set as metadata in PutObject* requests (for
-// encryption) or be set as request headers in `Core.GetObject` (for
-// decryption).
-func (s *SSEInfo) GetSSEHeaders() map[string]string {
-	return s.getSSEHeaders(false)
-}
-
 // DestinationInfo - type with information about the object to be
 // created via server-side copy requests, using the Compose API.
 type DestinationInfo struct {
 	bucket, object string
-
-	// key for encrypting destination
-	encryption *SSEInfo
+	encryption     encrypt.ServerSide
 
 	// if no user-metadata is provided, it is copied from source
 	// (when there is only once source object in the compose
@@ -97,9 +53,7 @@ type DestinationInfo struct {
 // if needed. If nil is passed, and if only a single source (of any
 // size) is provided in the ComposeObject call, then metadata from the
 // source is copied to the destination.
-func NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo,
-	userMeta map[string]string) (d DestinationInfo, err error) {
-
+func NewDestinationInfo(bucket, object string, sse encrypt.ServerSide, userMeta map[string]string) (d DestinationInfo, err error) {
 	// Input validation.
 	if err = s3utils.CheckValidBucketName(bucket); err != nil {
 		return d, err
@@ -125,7 +79,7 @@ func NewDestinationInfo(bucket, object string, encryptSSEC *SSEInfo,
 	return DestinationInfo{
 		bucket:       bucket,
 		object:       object,
-		encryption:   encryptSSEC,
+		encryption:   sse,
 		userMetadata: m,
 	}, nil
 }
@@ -154,10 +108,8 @@ func (d *DestinationInfo) getUserMetaHeadersMap(withCopyDirectiveHeader bool) ma
 // server-side copying APIs.
 type SourceInfo struct {
 	bucket, object string
-
-	start, end int64
-
-	decryptKey *SSEInfo
+	start, end     int64
+	encryption     encrypt.ServerSide
 	// Headers to send with the upload-part-copy request involving
 	// this source object.
 	Headers http.Header
@@ -169,21 +121,21 @@ type SourceInfo struct {
 // `decryptSSEC` is the decryption key using server-side-encryption
 // with customer provided key. It may be nil if the source is not
 // encrypted.
-func NewSourceInfo(bucket, object string, decryptSSEC *SSEInfo) SourceInfo {
+func NewSourceInfo(bucket, object string, sse encrypt.ServerSide) SourceInfo {
 	r := SourceInfo{
 		bucket:     bucket,
 		object:     object,
 		start:      -1, // range is unspecified by default
-		decryptKey: decryptSSEC,
+		encryption: sse,
 		Headers:    make(http.Header),
 	}
 
 	// Set the source header
 	r.Headers.Set("x-amz-copy-source", s3utils.EncodePath(bucket+"/"+object))
 
 	// Assemble decryption headers for upload-part-copy request
-	for k, v := range decryptSSEC.getSSEHeaders(true) {
-		r.Headers.Set(k, v)
+	if r.encryption != nil {
+		encrypt.SSECopy(r.encryption).Marshal(r.Headers)
 	}
 
 	return r
@@ -245,10 +197,7 @@ func (s *SourceInfo) getProps(c Client) (size int64, etag string, userMeta map[s
 	// Get object info - need size and etag here. Also, decryption
 	// headers are added to the stat request if given.
 	var objInfo ObjectInfo
-	opts := StatObjectOptions{}
-	for k, v := range s.decryptKey.getSSEHeaders(false) {
-		opts.Set(k, v)
-	}
+	opts := StatObjectOptions{GetObjectOptions{ServerSideEncryption: s.encryption}}
 	objInfo, err = c.statObject(context.Background(), s.bucket, s.object, opts)
 	if err != nil {
 		err = ErrInvalidArgument(fmt.Sprintf("Could not stat object - %s/%s: %v", s.bucket, s.object, err))
@@ -480,8 +429,8 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
 		h := srcs[0].Headers
 		// Add destination encryption headers
-		for k, v := range dst.encryption.getSSEHeaders(false) {
-			h.Set(k, v)
+		if dst.encryption != nil {
+			dst.encryption.Marshal(h)
 		}
 
 		// If no user metadata is specified (and so, the
@@ -528,12 +477,7 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 		metaHeaders[k] = v
 	}
 
-	// Add destination encryption headers
-	for k, v := range dst.encryption.getSSEHeaders(false) {
-		metaHeaders[k] = v
-	}
-
-	uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{UserMetadata: metaHeaders})
+	uploadID, err := c.newUploadID(ctx, dst.bucket, dst.object, PutObjectOptions{ServerSideEncryption: dst.encryption, UserMetadata: metaHeaders})
 	if err != nil {
 		return err
 	}
@@ -544,8 +488,8 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	for i, src := range srcs {
 		h := src.Headers
 		// Add destination encryption headers
-		for k, v := range dst.encryption.getSSEHeaders(false) {
-			h.Set(k, v)
+		if dst.encryption != nil {
+			dst.encryption.Marshal(h)
 		}
 
 		// calculate start/end indices of parts after

api-get-object-file.go

@@ -18,14 +18,11 @@
 package minio
 
 import (
+	"context"
 	"io"
 	"os"
 	"path/filepath"
 
-	"github.com/minio/minio-go/pkg/encrypt"
-
-	"context"
-
 	"github.com/minio/minio-go/pkg/s3utils"
 )
 
@@ -40,14 +37,6 @@ func (c Client) FGetObject(bucketName, objectName, filePath string, opts GetObje
 	return c.fGetObjectWithContext(context.Background(), bucketName, objectName, filePath, opts)
 }
 
-// FGetEncryptedObject - Decrypt and store an object at filePath.
-func (c Client) FGetEncryptedObject(bucketName, objectName, filePath string, materials encrypt.Materials) error {
-	if materials == nil {
-		return ErrInvalidArgument("Unable to recognize empty encryption properties")
-	}
-	return c.FGetObject(bucketName, objectName, filePath, GetObjectOptions{Materials: materials})
-}
-
 // fGetObjectWithContext - fgetObject wrapper function with context
 func (c Client) fGetObjectWithContext(ctx context.Context, bucketName, objectName, filePath string, opts GetObjectOptions) error {
 	// Input validation.

api-get-object.go

@@ -27,20 +27,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
 )
 
-// GetEncryptedObject deciphers and streams data stored in the server after applying a specified encryption materials,
-// returned stream should be closed by the caller.
-func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.ReadCloser, error) {
-	if encryptMaterials == nil {
-		return nil, ErrInvalidArgument("Unable to recognize empty encryption properties")
-	}
-
-	return c.GetObject(bucketName, objectName, GetObjectOptions{Materials: encryptMaterials})
-}
-
 // GetObject - returns an seekable, readable object.
 func (c Client) GetObject(bucketName, objectName string, opts GetObjectOptions) (*Object, error) {
 	return c.getObjectWithContext(context.Background(), bucketName, objectName, opts)
@@ -665,15 +654,6 @@ func (c Client) getObject(ctx context.Context, bucketName, objectName string, op
 		Metadata: extractObjMetadata(resp.Header),
 	}
 
-	reader := resp.Body
-	if opts.Materials != nil {
-		err = opts.Materials.SetupDecryptMode(reader, objectStat.Metadata.Get(amzHeaderIV), objectStat.Metadata.Get(amzHeaderKey))
-		if err != nil {
-			return nil, ObjectInfo{}, err
-		}
-		reader = opts.Materials
-	}
-
 	// do not close body here, caller will close
-	return reader, objectStat, nil
+	return resp.Body, objectStat, nil
 }

api-get-options.go

@@ -28,9 +28,8 @@ import (
 // GetObjectOptions are used to specify additional headers or options
 // during GET requests.
 type GetObjectOptions struct {
-	headers map[string]string
-
-	Materials encrypt.Materials
+	headers              map[string]string
+	ServerSideEncryption encrypt.ServerSide
 }
 
 // StatObjectOptions are used to specify additional headers or options
@@ -45,6 +44,9 @@ func (o GetObjectOptions) Header() http.Header {
 	for k, v := range o.headers {
 		headers.Set(k, v)
 	}
+	if o.ServerSideEncryption != nil {
+		o.ServerSideEncryption.Marshal(headers)
+	}
 	return headers
 }
 

api-put-object-context.go

@@ -29,11 +29,5 @@ func (c Client) PutObjectWithContext(ctx context.Context, bucketName, objectName
 	if err != nil {
 		return 0, err
 	}
-	if opts.EncryptMaterials != nil {
-		if err = opts.EncryptMaterials.SetupEncryptMode(reader); err != nil {
-			return 0, err
-		}
-		return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, opts.EncryptMaterials, opts)
-	}
 	return c.putObjectCommon(ctx, bucketName, objectName, reader, objectSize, opts)
 }

api-put-object-multipart.go

@@ -33,6 +33,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
 )
 
@@ -138,7 +139,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
 		// Proceed to upload the part.
 		var objPart ObjectPart
 		objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber,
-			md5Base64, sha256Hex, int64(length), opts.UserMetadata)
+			md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption)
 		if err != nil {
 			return totalUploadedSize, err
 		}
@@ -226,11 +227,9 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN
 	return initiateMultipartUploadResult, nil
 }
 
-const serverEncryptionKeyPrefix = "x-amz-server-side-encryption"
-
 // uploadPart - Uploads a part in a multipart upload.
 func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader,
-	partNumber int, md5Base64, sha256Hex string, size int64, metadata map[string]string) (ObjectPart, error) {
+	partNumber int, md5Base64, sha256Hex string, size int64, sse encrypt.ServerSide) (ObjectPart, error) {
 	// Input validation.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		return ObjectPart{}, err
@@ -260,12 +259,8 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
 
 	// Set encryption headers, if any.
 	customHeader := make(http.Header)
-	for k, v := range metadata {
-		if len(v) > 0 {
-			if strings.HasPrefix(strings.ToLower(k), serverEncryptionKeyPrefix) {
-				customHeader.Set(k, v)
-			}
-		}
+	if sse != nil {
+		sse.Marshal(customHeader)
 	}
 
 	reqMetadata := requestMetadata{

api-put-object-streaming.go

@@ -167,7 +167,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
 				var objPart ObjectPart
 				objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID,
 					sectionReader, uploadReq.PartNum,
-					"", "", partSize, opts.UserMetadata)
+					"", "", partSize, opts.ServerSideEncryption)
 				if err != nil {
 					uploadedPartsCh <- uploadedPartRes{
 						Size:  0,
@@ -280,7 +280,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa
 		var objPart ObjectPart
 		objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID,
 			io.LimitReader(hookReader, partSize),
-			partNumber, "", "", partSize, opts.UserMetadata)
+			partNumber, "", "", partSize, opts.ServerSideEncryption)
 		if err != nil {
 			return totalUploadedSize, err
 		}