put cap of 1 second to NewPeerClient connect

gubernator.go

@@ -610,6 +610,7 @@ func (s *V1Instance) SetPeers(peerInfo []PeerInfo) {
 				})
 				if err != nil {
 					s.log.Error("error connecting to peer: %s", err)
+					return
 				}
 			}
 			regionPicker.Add(peer)
@@ -628,6 +629,7 @@ func (s *V1Instance) SetPeers(peerInfo []PeerInfo) {
 			})
 			if err != nil {
 				s.log.Error("error connecting to peer: %s", err)
+				return
 			}
 		}
 		localPicker.Add(peer)

peer_client.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/mailgun/holster/v4/clock"
 	"github.com/mailgun/holster/v4/collections"
@@ -79,7 +80,7 @@ type PeerConfig struct {
 	TraceGRPC bool
 }
 
-// NewPeerClient establishes a connection to a peer in a blocking fashion.
+// NewPeerClient tries to establish a connection to a peer in a blocking fashion with a 1 second timeout.
 // If batching is enabled, it also starts a goroutine where batches will be processed.
 func NewPeerClient(conf PeerConfig) (*PeerClient, error) {
 	peerClient := &PeerClient{
@@ -101,8 +102,11 @@ func NewPeerClient(conf PeerConfig) (*PeerClient, error) {
 		opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+
 	var err error
-	peerClient.conn, err = grpc.Dial(conf.Info.GRPCAddress, opts...)
+	peerClient.conn, err = grpc.DialContext(ctx, conf.Info.GRPCAddress, opts...)
 	if err != nil {
 		return nil, err
 	}