Add build index readiness endpoint and client (#377)

As described in #371, all services should implement a "readiness" endpoint, which checks whether the service is ready to serve traffic, i.e. whether it can reach all of its dependencies.

- Add readiness endpoint for build index, which checks whether all its backends are reachable and the origin's readiness.
- Add a client for build-index's readiness endpoint. The client will be used by agent -- agent's readiness endpoint will check build index's readiness, as build index is a dependency for agent.

build-index/tagclient/client.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -38,6 +38,7 @@ var (
 
 // Client wraps tagserver endpoints.
 type Client interface {
+	CheckReadiness() error
 	Put(tag string, d core.Digest) error
 	PutAndReplicate(tag string, d core.Digest) error
 	Get(tag string) (core.Digest, error)
@@ -70,6 +71,14 @@ func NewSingleClient(addr string, config *tls.Config) Client {
 	return &singleClient{addr, config}
 }
 
+func (c *singleClient) CheckReadiness() error {
+	_, err := httputil.Get(
+		fmt.Sprintf("http://%s/readiness", c.addr),
+		httputil.SendTimeout(5*time.Second),
+		httputil.SendTLS(c.tls))
+	return err
+}
+
 func (c *singleClient) Put(tag string, d core.Digest) error {
 	_, err := httputil.Put(
 		fmt.Sprintf("http://%s/tags/%s/digest/%s", c.addr, url.PathEscape(tag), d.String()),
@@ -311,6 +320,33 @@ func (cc *clusterClient) do(request func(c Client) error) error {
 	return err
 }
 
+// doOnce tries the request on only one randomly chosen client without any retries if it fails.
+func (cc *clusterClient) doOnce(request func(c Client) error) error {
+	addrs := cc.hosts.Resolve().Sample(1)
+	if len(addrs) == 0 {
+		return errors.New("cluster client: no hosts could be resolved")
+	}
+	// read the only sampled addr
+	var addr string
+	for addr = range addrs {
+	}
+	err := request(NewSingleClient(addr, cc.tls))
+	if httputil.IsNetworkError(err) {
+		cc.hosts.Failed(addr)
+	}
+	return err
+}
+
+func (cc *clusterClient) CheckReadiness() error {
+	return cc.doOnce(func(c Client) error {
+		err := c.CheckReadiness()
+		if err != nil {
+			return fmt.Errorf("build index not ready: %v", err)
+		}
+		return nil
+	})
+}
+
 func (cc *clusterClient) Put(tag string, d core.Digest) error {
 	return cc.do(func(c Client) error { return c.Put(tag, d) })
 }

build-index/tagserver/server.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -108,6 +108,7 @@ func (s *Server) Handler() http.Handler {
 	r.Use(middleware.LatencyTimer(s.stats))
 
 	r.Get("/health", handler.Wrap(s.healthHandler))
+	r.Get("/readiness", handler.Wrap(s.readinessCheckHandler))
 
 	r.Put("/tags/{tag}/digest/{digest}", handler.Wrap(s.putTagHandler))
 	r.Head("/tags/{tag}", handler.Wrap(s.hasTagHandler))
@@ -145,6 +146,19 @@ func (s *Server) healthHandler(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
+func (s *Server) readinessCheckHandler(w http.ResponseWriter, r *http.Request) error {
+	err := s.backends.CheckReadiness()
+	if err != nil {
+		return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable)
+	}
+	err = s.localOriginClient.CheckReadiness()
+	if err != nil {
+		return handler.Errorf("not ready to serve traffic: %s", err).Status(http.StatusServiceUnavailable)
+	}
+	fmt.Fprintln(w, "OK")
+	return nil
+}
+
 func (s *Server) putTagHandler(w http.ResponseWriter, r *http.Request) error {
 	tag, err := httputil.ParseParam(r, "tag")
 	if err != nil {

build-index/tagserver/server_test.go

@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     http://www.apache.org/licenses/LICENSE-2.0
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,10 +14,12 @@
 package tagserver
 
 import (
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strconv"
 	"testing"
 	"time"
@@ -149,6 +151,69 @@ func TestHealth(t *testing.T) {
 	require.Equal("OK\n", string(b))
 }
 
+func TestCheckReadiness(t *testing.T) {
+	for _, tc := range []struct {
+		name                  string
+		mockStatErr           error
+		mockOriginErr         error
+		expectedErrMsgPattern string
+	}{
+		{
+			name:                  "success",
+			mockStatErr:           nil,
+			mockOriginErr:         nil,
+			expectedErrMsgPattern: "",
+		},
+		{
+			name:                  "failure, 503 (only Stat fails)",
+			mockStatErr:           errors.New("backend storage error"),
+			mockOriginErr:         nil,
+			expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: backend for namespace 'foo-bar/\*' not ready: backend storage error`),
+		},
+		{
+			name:                  "failure, 503 (only origin fails)",
+			mockStatErr:           nil,
+			mockOriginErr:         errors.New("origin error"),
+			expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: origin error`),
+		},
+		{
+			name:                  "failure, 503 (both fail)",
+			mockStatErr:           errors.New("backend storage error"),
+			mockOriginErr:         errors.New("origin error"),
+			expectedErrMsgPattern: fmt.Sprintf(`build index not ready: GET http://127\.0\.0\.1:\d+/readiness 503: not ready to serve traffic: backend for namespace 'foo-bar/\*' not ready: backend storage error`),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			require := require.New(t)
+
+			mocks, cleanup := newServerMocks(t)
+			defer cleanup()
+
+			addr, stop := testutil.StartServer(mocks.handler())
+			defer stop()
+
+			client := newClusterClient(addr)
+			backendClient := mockbackend.NewMockClient(mocks.ctrl)
+			require.NoError(mocks.backends.Register("foo-bar/*", backendClient, true))
+
+			mockStat := &core.BlobInfo{}
+			if tc.mockStatErr != nil {
+				mockStat = nil
+			}
+			backendClient.EXPECT().Stat(backend.ReadinessCheckNamespace, backend.ReadinessCheckName).Return(mockStat, tc.mockStatErr)
+			mocks.originClient.EXPECT().CheckReadiness().Return(tc.mockOriginErr).AnyTimes()
+
+			err := client.CheckReadiness()
+			if tc.expectedErrMsgPattern == "" {
+				require.Nil(err)
+			} else {
+				r, _ := regexp.Compile(tc.expectedErrMsgPattern)
+				require.True(r.MatchString(err.Error()))
+			}
+		})
+	}
+}
+
 func TestPut(t *testing.T) {
 	require := require.New(t)