[server/kit] adding NewLogger to use for background logging + updated…

… docs (#170)

* adding NewLogger to use for background logging + upped kit docs

* doc cleanup

* removing tab from docs

* p->P

README.md

@@ -29,16 +29,15 @@ It offers 2 server implementations:
 
 #### [`server/kit`](https://godoc.org/github.com/NYTimes/gizmo/server/kit)
 
-This is an experimental package in Gizmo!
-
-* The rationale behind this package:
-    * A more opinionated server with fewer choices
-    * go-kit is used for serving HTTP/JSON & gRPC is used for serving HTTP2/RPC
-    * Monitoring and metrics are handled by a sidecar (ie. Cloud Endpoints)
-    * Logs always go to stdout/stderr
-    * Using Go's 1.8 graceful HTTP shutdown
-    * Services using this package are meant for deploy to GCP with GKE and Cloud Endpoints.
-
+The `server/kit` package embodies Gizmo's goals to combine with go-kit.
+
+* In this package you'll find:
+    * A more opinionated server with fewer choices.
+    * go-kit used for serving HTTP/JSON & gRPC used for serving HTTP2/RPC
+    * Monitoring and metrics are automatically registered if running within App Engine.
+    * Logs go to stdout locally or directly to Stackdriver when in GCP.
+    * Using Go's 1.8 graceful HTTP shutdown.
+    * Services using this package are expected to deploy to GCP.
 
 #### [`auth`](https://godoc.org/github.com/NYTimes/gizmo/auth)
 

server/kit/README.md

@@ -1,10 +1,17 @@
-# This is an experimental package in Gizmo!
+# Welcome to the 2nd generation of Gizmo servers 🚀!
 
-* The rationale behind this package:
-    * A more opinionated server with fewer choices.
-    * go-kit is used for serving HTTP/JSON & gRPC is used for serving HTTP2/RPC
-    * Logs always go to stdout/stderr by default, but if running on App Engine, trace enabled Stackdriver logging will be used instead.
-    * Using Go's 1.8 graceful HTTP shutdown
-    * Services using this package are meant for deploy to GCP.
+Gizmo's intentions from the beginning were to eventually join forces with the wonders of the [go-kit toolkit](https://github.com/go-kit/kit). This package is meant to embody that goal.
 
-* If you experience any issues please create an issue and/or reach out on the #gizmo channel with what you've found.
+The `kit` server is composed of multiple [kit/transport/http.Servers](https://godoc.org/github.com/go-kit/kit/transport/http#Server) that are tied together with a common HTTP mux, HTTP options and middlewares. By default all HTTP endpoints will be encoded as JSON, but developers may override each [HTTPEndpoint](https://godoc.org/github.com/NYTimes/gizmo/server/kit#HTTPEndpoint) to use whatever encoding they need. If users need to use gRPC, they can can register the same endpoints to serve both HTTP and gRPC requests on two different ports.
+
+This server expects to be configured via environment variables. The available variables can be found by inspecting the [Config struct](https://godoc.org/github.com/NYTimes/gizmo/server/kit#Config) within this package. If no health check or [warm up](https://cloud.google.com/appengine/docs/standard/go111/how-instances-are-managed#warmup_requests) endpoints are defined, this server will automatically register basic endpoints there to return a simple "200 OK" response.
+
+Since NYT uses Google Cloud, deploying this server to that environment provides additional perks:
+
+* If running in the [App Engine 2nd Generation runtime (Go >=1.11)](https://cloud.google.com/appengine/docs/standard/go111/), servers will:
+  * Automatically catch any panics and send them to Stackdriver Error reporting
+  * Automatically use Stackdriver logging and, if `kit.LogXXX` functions are used, logs will be trace enabled and will be combined with their parent access log in the Stackdriver logging console.
+  * Automatically register Stackdriver exporters for Open Census trace and monitoring. Most Google Cloud clients (like [Cloud Spanner](https://godoc.org/cloud.google.com/go/spanner)) will detect this and emit the traces. Users can also add their own trace and monitoring spans via [the Open Census clients](https://godoc.org/go.opencensus.io/trace#example-StartSpan).
+
+
+For an example of how to build a server that utilizes this package, see the [Reading List example](https://github.com/NYTimes/gizmo/tree/master/examples/servers/reading-list#the-reading-list-example).

server/kit/gae_log.go

@@ -5,6 +5,7 @@ import (
 	"encoding"
 	"encoding/json"
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 
@@ -15,6 +16,17 @@ import (
 	"google.golang.org/genproto/googleapis/api/monitoredres"
 )
 
+// project, service, version
+func getGAEInfo() (string, string, string) {
+	return os.Getenv("GOOGLE_CLOUD_PROJECT"),
+		os.Getenv("GAE_SERVICE"),
+		os.Getenv("GAE_VERSION")
+}
+
+func isGAE() bool {
+	return os.Getenv("GAE_DEPLOYMENT_ID") != ""
+}
+
 type gaeLogger struct {
 	project string
 	monRes  *monitoredres.MonitoredResource

server/kit/kitserver.go

@@ -1,3 +1,4 @@
+// Package kit implements an opinionated server based on go-kit primitives.
 package kit
 
 import (
@@ -7,7 +8,6 @@ import (
 	"net"
 	"net/http"
 	"net/http/pprof"
-	"os"
 	"strings"
 
 	"cloud.google.com/go/errorreporting"
@@ -76,29 +76,20 @@ func NewServer(svc Service) *Server {
 		r = opt(r)
 	}
 
-	// check if we're running on GAE via env variables
-	projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
-	serviceID := os.Getenv("GAE_SERVICE")
-	svcVersion := os.Getenv("GAE_VERSION")
+	ctx := context.Background()
+
+	lg, logClose, err := NewLogger(ctx)
+	if err != nil {
+		stdlog.Fatalf("unable to start up logger: %s", err)
+	}
 
 	var (
-		err      error
-		lg       log.Logger
-		logClose func() error
-		errs     *errorreporting.Client
+		errs  *errorreporting.Client
+		propr propagation.HTTPFormat
 	)
-	var propr propagation.HTTPFormat
-	// use the version variable to determine if we're in the GAE environment
-	if svcVersion == "" {
-		lg = log.NewJSONLogger(log.NewSyncWriter(os.Stdout))
-	} else {
-		ctx := context.Background()
-		lg, logClose, err = newAppEngineLogger(ctx,
-			projectID, serviceID, svcVersion)
-		if err != nil {
-			stdlog.Fatalf("unable to start up app engine logger: %s", err)
-		}
-
+	// if in App Engine, initiate an error reporter and the stackdriver exporters
+	if isGAE() {
+		projectID, serviceID, svcVersion := getGAEInfo()
 		errs, err = errorreporting.NewClient(ctx, projectID, errorreporting.Config{
 			ServiceName:    serviceID,
 			ServiceVersion: svcVersion,

server/kit/log.go

@@ -2,12 +2,31 @@ package kit
 
 import (
 	"context"
+	"os"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/transport/http"
 	"google.golang.org/grpc/metadata"
 )
 
+// NewLogger will inspect the environment and, if running in the Google App Engine
+// environment, it will return a new Stackdriver logger annotated with the current
+// server's project ID, service ID and version. If not in App Engine, a normal JSON
+// logger pointing to stdout will be returned.
+// This function can be used for services that need to log information outside the
+// context of an inbound request.
+// When using the Stackdriver logger, any go-kit/log/levels will be translated to
+// Stackdriver severity levels.
+func NewLogger(ctx context.Context) (log.Logger, func() error, error) {
+	// running locally or in a non-GAE environment? use JSON
+	if !isGAE() {
+		return log.NewJSONLogger(log.NewSyncWriter(os.Stdout)), func() error { return nil }, nil
+	}
+
+	projectID, serviceID, svcVersion := getGAEInfo()
+	return newAppEngineLogger(ctx, projectID, serviceID, svcVersion)
+}
+
 // Logger will return a kit/log.Logger that has been injected into the context by the kit
 // server. This logger has had request headers and metadata added as key values.
 // This function will only work within the scope of a request initiated by the server.

server/kit/server.go

@@ -1,5 +1,3 @@
-// +build !appengine
-
 package kit
 
 import (
@@ -8,9 +6,6 @@ import (
 	"syscall"
 )
 
-// TODO(jprobinson): built in stackdriver error reporting
-// TODO(jprobinson): built in stackdriver tracing (sampling)
-
 // Run will use environment variables to configure the server then register the given
 // Service and start up the server(s).
 // This will block until the server shuts down.