Enable ingestion of GCS SAs from files outside of auth config

lib/backend/gcsbackend/config.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//	   http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,8 +15,10 @@ package gcsbackend
 
 import (
 	"github.com/c2h5oh/datasize"
+	"os"
 
 	"github.com/uber/kraken/lib/backend"
+	"gopkg.in/yaml.v2"
 )
 
 // Config defines gcs connection specific
@@ -51,6 +53,11 @@ type AuthConfig struct {
 	} `yaml:"gcs"`
 }
 
+type ServiceAccount struct {
+	Name string `yaml:"name"`
+	Path string `yaml:"path"`
+}
+
 func (c *Config) applyDefaults() {
 	if c.UploadChunkSize == 0 {
 		c.UploadChunkSize = backend.DefaultPartSize
@@ -62,3 +69,30 @@ func (c *Config) applyDefaults() {
 		c.ListMaxKeys = backend.DefaultListMaxKeys
 	}
 }
+
+func BuildUserAuthConfig(sas []ServiceAccount) (UserAuthConfig, error) {
+	gcsUserAuthConfig := UserAuthConfig{}
+
+	for _, sa := range sas {
+		var dataBytes []byte
+		dataBytes, err := os.ReadFile(sa.Path)
+		if err != nil {
+			return nil, err
+		}
+
+		var data string
+		err = yaml.Unmarshal(dataBytes, data)
+		if err != nil {
+			return nil, err
+		}
+
+		gcsUserAuthConfig[sa.Name] = AuthConfig{
+			GCS: struct {
+				AccessBlob string `yaml:"access_blob"`
+			}{
+				AccessBlob: data,
+			},
+		}
+	}
+	return gcsUserAuthConfig, nil
+}

origin/cmd/cmd.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,6 +22,7 @@ import (
 
 	"github.com/uber/kraken/core"
 	"github.com/uber/kraken/lib/backend"
+	"github.com/uber/kraken/lib/backend/gcsbackend"
 	"github.com/uber/kraken/lib/blobrefresh"
 	"github.com/uber/kraken/lib/hashring"
 	"github.com/uber/kraken/lib/healthcheck"
@@ -136,6 +137,14 @@ func Run(flags *Flags, opts ...Option) {
 		}
 	}
 
+	if len(config.GCSServiceAccounts) != 0 {
+		gcsUserAuthConfig, err := gcsbackend.BuildUserAuthConfig(config.GCSServiceAccounts)
+		if err != nil {
+			panic(err)
+		}
+		config.Auth["gcs"] = gcsUserAuthConfig
+	}
+
 	if overrides.logger != nil {
 		log.SetGlobalLogger(overrides.logger.Sugar())
 	} else {

origin/cmd/config.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,6 +16,7 @@ package cmd
 import (
 	"github.com/uber/kraken/core"
 	"github.com/uber/kraken/lib/backend"
+	"github.com/uber/kraken/lib/backend/gcsbackend"
 	"github.com/uber/kraken/lib/blobrefresh"
 	"github.com/uber/kraken/lib/hashring"
 	"github.com/uber/kraken/lib/healthcheck"
@@ -37,23 +38,24 @@ import (
 // Config defines origin server configuration.
 // TODO(evelynl94): consolidate cluster and hashring.
 type Config struct {
-	Verbose       bool
-	ZapLogging    zap.Config               `yaml:"zap"`
-	Cluster       hostlist.Config          `yaml:"cluster"`
-	HashRing      hashring.Config          `yaml:"hashring"`
-	HealthCheck   healthcheck.FilterConfig `yaml:"healthcheck"`
-	BlobServer    blobserver.Config        `yaml:"blobserver"`
-	CAStore       store.CAStoreConfig      `yaml:"castore"`
-	Scheduler     scheduler.Config         `yaml:"scheduler"`
-	NetworkEvent  networkevent.Config      `yaml:"network_event"`
-	PeerIDFactory core.PeerIDFactory       `yaml:"peer_id_factory"`
-	Metrics       metrics.Config           `yaml:"metrics"`
-	MetaInfoGen   metainfogen.Config       `yaml:"metainfogen"`
-	Backends      []backend.Config         `yaml:"backends"`
-	Auth          backend.AuthConfig       `yaml:"auth"`
-	BlobRefresh   blobrefresh.Config       `yaml:"blobrefresh"`
-	LocalDB       localdb.Config           `yaml:"localdb"`
-	WriteBack     persistedretry.Config    `yaml:"writeback"`
-	Nginx         nginx.Config             `yaml:"nginx"`
-	TLS           httputil.TLSConfig       `yaml:"tls"`
+	Verbose            bool
+	ZapLogging         zap.Config                  `yaml:"zap"`
+	Cluster            hostlist.Config             `yaml:"cluster"`
+	HashRing           hashring.Config             `yaml:"hashring"`
+	HealthCheck        healthcheck.FilterConfig    `yaml:"healthcheck"`
+	BlobServer         blobserver.Config           `yaml:"blobserver"`
+	CAStore            store.CAStoreConfig         `yaml:"castore"`
+	Scheduler          scheduler.Config            `yaml:"scheduler"`
+	NetworkEvent       networkevent.Config         `yaml:"network_event"`
+	PeerIDFactory      core.PeerIDFactory          `yaml:"peer_id_factory"`
+	Metrics            metrics.Config              `yaml:"metrics"`
+	MetaInfoGen        metainfogen.Config          `yaml:"metainfogen"`
+	Backends           []backend.Config            `yaml:"backends"`
+	Auth               backend.AuthConfig          `yaml:"auth"`
+	BlobRefresh        blobrefresh.Config          `yaml:"blobrefresh"`
+	LocalDB            localdb.Config              `yaml:"localdb"`
+	WriteBack          persistedretry.Config       `yaml:"writeback"`
+	Nginx              nginx.Config                `yaml:"nginx"`
+	TLS                httputil.TLSConfig          `yaml:"tls"`
+	GCSServiceAccounts []gcsbackend.ServiceAccount `yaml:"gcs_service_accounts"`
 }