diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..5ca46841 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.pb.go linguist-generated=true +*.pb.*.go linguist-generated=true +go.sum linguist-generated=true +buf.lock linguist-generated=true \ No newline at end of file diff --git a/Makefile b/Makefile index 75240d97..192ed39c 100644 --- a/Makefile +++ b/Makefile @@ -1,47 +1,57 @@ -.DEFAULT_GOAL := release +.DEFAULT_GOAL := build VERSION=$(shell cat version) LDFLAGS="-X main.Version=$(VERSION)" GOLANGCI_LINT = $(GOPATH)/bin/golangci-lint GOLANGCI_LINT_VERSION = 1.56.2 -$(GOLANGCI_LINT): +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +$(GOLANGCI_LINT): ## Download Go linter curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin $(GOLANGCI_LINT_VERSION) .PHONY: lint -lint: $(GOLANGCI_LINT) - $(GOLANGCI_LINT) run +lint: $(GOLANGCI_LINT) ## Run Go linter + $(GOLANGCI_LINT) run -v --fix -c .golangci.yml ./... .PHONY: test -test: +test: ## Run unit tests and measure code coverage (go test -v -race -p=1 -count=1 -tags holster_test_mode -coverprofile coverage.out ./...; ret=$$?; \ go tool cover -func coverage.out; \ go tool cover -html coverage.out -o coverage.html; \ exit $$ret) .PHONY: bench -bench: +bench: ## Run Go benchmarks go test ./... -bench . -benchtime 5s -timeout 0 -run=XXX -benchmem .PHONY: docker -docker: +docker: ## Build Docker image docker build --build-arg VERSION=$(VERSION) -t ghcr.io/mailgun/gubernator:$(VERSION) . docker tag ghcr.io/mailgun/gubernator:$(VERSION) ghcr.io/mailgun/gubernator:latest -.PHONY: release -release: +.PHONY: build +build: proto ## Build binary go build -v -ldflags $(LDFLAGS) -o gubernator ./cmd/gubernator/main.go .PHONY: clean -clean: +clean: ## Clean binaries rm -f gubernator gubernator-cli +.PHONY: clean-proto +clean-proto: ## Clean the generated source files from the protobuf sources + @echo "==> Cleaning up the go generated files from proto" + @find . -name "*.pb.go" -type f -delete + @find . -name "*.pb.*.go" -type f -delete + + .PHONY: proto -proto: - # Install buf: https://buf.build/docs/installation - buf generate +proto: ## Build protos + ./buf.gen.yaml .PHONY: certs -certs: +certs: ## Generate SSL certificates rm certs/*.key || rm certs/*.srl || rm certs/*.csr || rm certs/*.pem || rm certs/*.cert || true openssl genrsa -out certs/ca.key 4096 openssl req -new -x509 -key certs/ca.key -sha256 -subj "/C=US/ST=TX/O=Mailgun Technologies, Inc." -days 3650 -out certs/ca.cert diff --git a/buf.gen.yaml b/buf.gen.yaml old mode 100644 new mode 100755 index 65a065c5..5c62f51f --- a/buf.gen.yaml +++ b/buf.gen.yaml @@ -1,6 +1,8 @@ +#!/usr/bin/env -S buf generate --debug --template +--- version: v1 plugins: - - name: go + - plugin: buf.build/protocolbuffers/go:v1.32.0 out: ./ opt: paths=source_relative - plugin: buf.build/grpc/go:v1.3.0 @@ -8,6 +10,12 @@ plugins: opt: - paths=source_relative - require_unimplemented_servers=false + - plugin: buf.build/grpc-ecosystem/gateway:v2.18.0 # same version in go.mod + out: ./ + opt: + - paths=source_relative + - logtostderr=true + - generate_unbound_methods=true - plugin: buf.build/grpc/python:v1.57.0 out: ./python/gubernator - plugin: buf.build/protocolbuffers/python diff --git a/buf.lock b/buf.lock index 3435191d..b46be110 100644 --- a/buf.lock +++ b/buf.lock @@ -4,5 +4,5 @@ deps: - remote: buf.build owner: googleapis repository: googleapis - commit: 711e289f6a384c4caeebaff7c6931ade - digest: shake256:e08fb55dad7469f69df00304eed31427d2d1576e9aab31e6bf86642688e04caaf0372f15fe6974cf79432779a635b3ea401ca69c943976dc42749524e4c25d94 + commit: 7e6f6e774e29406da95bd61cdcdbc8bc + digest: shake256:fe43dd2265ea0c07d76bd925eeba612667cf4c948d2ce53d6e367e1b4b3cb5fa69a51e6acb1a6a50d32f894f054a35e6c0406f6808a483f2752e10c866ffbf73 diff --git a/gubernator.pb.go b/gubernator.pb.go index 1ae28afa..808a8814 100644 --- a/gubernator.pb.go +++ b/gubernator.pb.go @@ -109,7 +109,7 @@ const ( // distributed to each peer and cached locally. A rate limit request received from any peer in the // cluster will first check the local cache for a rate limit answer, if it exists the peer will // immediately return the answer to the client and asynchronously forward the aggregate hits to - // the peer coordinator. Because of GLOBALS async nature we lose some accuracy in rate limit + // the owner peer. Because of GLOBALS async nature we lose some accuracy in rate limit // reporting, which may result in allowing some requests beyond the chosen rate limit. However we // gain massive performance as every request coming into the system does not have to wait for a // single peer to decide if the rate limit has been reached. @@ -478,15 +478,15 @@ type RateLimitResp struct { // The status of the rate limit. Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=pb.gubernator.Status" json:"status,omitempty"` - // The currently configured request limit (Identical to RateLimitRequest.rate_limit_config.limit). + // The currently configured request limit (Identical to [[RateLimitReq.limit]]). Limit int64 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` - // This is the number of requests remaining before the limit is hit. + // This is the number of requests remaining before the rate limit is hit but after subtracting the hits from the current request Remaining int64 `protobuf:"varint,3,opt,name=remaining,proto3" json:"remaining,omitempty"` // This is the time when the rate limit span will be reset, provided as a unix timestamp in milliseconds. ResetTime int64 `protobuf:"varint,4,opt,name=reset_time,json=resetTime,proto3" json:"reset_time,omitempty"` // Contains the error; If set all other values should be ignored Error string `protobuf:"bytes,5,opt,name=error,proto3" json:"error,omitempty"` - // This is additional metadata that a client might find useful. (IE: Additional headers, corrdinator ownership, etc..) + // This is additional metadata that a client might find useful. (IE: Additional headers, coordinator ownership, etc..) Metadata map[string]string `protobuf:"bytes,6,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } diff --git a/gubernator.pb.gw.go b/gubernator.pb.gw.go index 1c67924b..bb460598 100644 --- a/gubernator.pb.gw.go +++ b/gubernator.pb.gw.go @@ -145,7 +145,7 @@ func RegisterV1HandlerServer(ctx context.Context, mux *runtime.ServeMux, server // RegisterV1HandlerFromEndpoint is same as RegisterV1Handler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterV1HandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { - conn, err := grpc.Dial(endpoint, opts...) + conn, err := grpc.DialContext(ctx, endpoint, opts...) if err != nil { return err } diff --git a/gubernator.proto b/gubernator.proto index 5fd040df..fea99a22 100644 --- a/gubernator.proto +++ b/gubernator.proto @@ -85,7 +85,7 @@ enum Behavior { // distributed to each peer and cached locally. A rate limit request received from any peer in the // cluster will first check the local cache for a rate limit answer, if it exists the peer will // immediately return the answer to the client and asynchronously forward the aggregate hits to - // the peer coordinator. Because of GLOBALS async nature we lose some accuracy in rate limit + // the owner peer. Because of GLOBALS async nature we lose some accuracy in rate limit // reporting, which may result in allowing some requests beyond the chosen rate limit. However we // gain massive performance as every request coming into the system does not have to wait for a // single peer to decide if the rate limit has been reached. @@ -178,15 +178,15 @@ enum Status { message RateLimitResp { // The status of the rate limit. Status status = 1; - // The currently configured request limit (Identical to RateLimitRequest.rate_limit_config.limit). + // The currently configured request limit (Identical to [[RateLimitReq.limit]]). int64 limit = 2; - // This is the number of requests remaining before the limit is hit. + // This is the number of requests remaining before the rate limit is hit but after subtracting the hits from the current request int64 remaining = 3; // This is the time when the rate limit span will be reset, provided as a unix timestamp in milliseconds. int64 reset_time = 4; // Contains the error; If set all other values should be ignored string error = 5; - // This is additional metadata that a client might find useful. (IE: Additional headers, corrdinator ownership, etc..) + // This is additional metadata that a client might find useful. (IE: Additional headers, coordinator ownership, etc..) map metadata = 6; } diff --git a/peers.pb.gw.go b/peers.pb.gw.go index 41f7d6e5..f0929765 100644 --- a/peers.pb.gw.go +++ b/peers.pb.gw.go @@ -161,7 +161,7 @@ func RegisterPeersV1HandlerServer(ctx context.Context, mux *runtime.ServeMux, se // RegisterPeersV1HandlerFromEndpoint is same as RegisterPeersV1Handler but // automatically dials to "endpoint" and closes the connection when "ctx" gets done. func RegisterPeersV1HandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { - conn, err := grpc.Dial(endpoint, opts...) + conn, err := grpc.DialContext(ctx, endpoint, opts...) if err != nil { return err } diff --git a/peers.proto b/peers.proto index 5caefae4..1ce2a431 100644 --- a/peers.proto +++ b/peers.proto @@ -26,10 +26,10 @@ import "gubernator.proto"; // NOTE: For use by gubernator peers only service PeersV1 { - // Used by peers to relay batches of requests to an authoritative peer + // Used by peers to relay batches of requests to an owner peer rpc GetPeerRateLimits (GetPeerRateLimitsReq) returns (GetPeerRateLimitsResp) {} - // Used by peers send global rate limit updates to other peers + // Used by owner peers to send global rate limit updates to non-owner peers rpc UpdatePeerGlobals (UpdatePeerGlobalsReq) returns (UpdatePeerGlobalsResp) {} } diff --git a/peers_grpc.pb.go b/peers_grpc.pb.go index 33db74af..e74a7d16 100644 --- a/peers_grpc.pb.go +++ b/peers_grpc.pb.go @@ -42,9 +42,9 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type PeersV1Client interface { - // Used by peers to relay batches of requests to an authoritative peer + // Used by peers to relay batches of requests to an owner peer GetPeerRateLimits(ctx context.Context, in *GetPeerRateLimitsReq, opts ...grpc.CallOption) (*GetPeerRateLimitsResp, error) - // Used by peers send global rate limit updates to other peers + // Used by owner peers to send global rate limit updates to non-owner peers UpdatePeerGlobals(ctx context.Context, in *UpdatePeerGlobalsReq, opts ...grpc.CallOption) (*UpdatePeerGlobalsResp, error) } @@ -78,9 +78,9 @@ func (c *peersV1Client) UpdatePeerGlobals(ctx context.Context, in *UpdatePeerGlo // All implementations should embed UnimplementedPeersV1Server // for forward compatibility type PeersV1Server interface { - // Used by peers to relay batches of requests to an authoritative peer + // Used by peers to relay batches of requests to an owner peer GetPeerRateLimits(context.Context, *GetPeerRateLimitsReq) (*GetPeerRateLimitsResp, error) - // Used by peers send global rate limit updates to other peers + // Used by owner peers to send global rate limit updates to non-owner peers UpdatePeerGlobals(context.Context, *UpdatePeerGlobalsReq) (*UpdatePeerGlobalsResp, error) } diff --git a/python/gubernator/gubernator_pb2.py b/python/gubernator/gubernator_pb2.py index cefa90ce..17351bb6 100644 --- a/python/gubernator/gubernator_pb2.py +++ b/python/gubernator/gubernator_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: gubernator.proto -# Protobuf Python Version: 4.25.1 +# Protobuf Python Version: 4.25.3 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/python/gubernator/peers_pb2.py b/python/gubernator/peers_pb2.py index 53b40758..b1451c7a 100644 --- a/python/gubernator/peers_pb2.py +++ b/python/gubernator/peers_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: peers.proto -# Protobuf Python Version: 4.25.1 +# Protobuf Python Version: 4.25.3 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/python/gubernator/peers_pb2_grpc.py b/python/gubernator/peers_pb2_grpc.py index 7b8f4c99..9ebb860d 100644 --- a/python/gubernator/peers_pb2_grpc.py +++ b/python/gubernator/peers_pb2_grpc.py @@ -32,14 +32,14 @@ class PeersV1Servicer(object): """ def GetPeerRateLimits(self, request, context): - """Used by peers to relay batches of requests to an authoritative peer + """Used by peers to relay batches of requests to an owner peer """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def UpdatePeerGlobals(self, request, context): - """Used by peers send global rate limit updates to other peers + """Used by owner peers to send global rate limit updates to non-owner peers """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!')