From da75cdf385d3eebfb2db363a9ad9447c70be58b1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Braun Date: Tue, 1 Mar 2022 11:32:55 +0100 Subject: [PATCH] pkg/tool/http: add tls settings tls.verify can be used to disable server certificate validation. tls.caCert can be used to provide a PEM encoded certificates to validate the server certificate. Closes #1558 Signed-off-by: Jean-Philippe Braun Change-Id: If8f0aa5d9f882675e84e2546faa510f7d3bcde1c Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/533845 Unity-Result: CUEcueckoo TryBot-Result: CUEcueckoo Reviewed-by: Marcel van Lohuizen --- cmd/cue/cmd/testdata/script/cmd_http.txt | 13 ++--- internal/task/task.go | 14 +++-- pkg/tool/http/doc.go | 8 +++ pkg/tool/http/http.cue | 8 +++ pkg/tool/http/http.go | 55 +++++++++++++++++--- pkg/tool/http/http_test.go | 65 ++++++++++++++++++++++++ pkg/tool/http/pkg.go | 4 ++ 7 files changed, 151 insertions(+), 16 deletions(-) diff --git a/cmd/cue/cmd/testdata/script/cmd_http.txt b/cmd/cue/cmd/testdata/script/cmd_http.txt index 4c240b1ad94..9ead053f395 100644 --- a/cmd/cue/cmd/testdata/script/cmd_http.txt +++ b/cmd/cue/cmd/testdata/script/cmd_http.txt @@ -4,24 +4,25 @@ cmp stdout cmd_http.out {"data":"I'll be back!","when":"now"} -- task_tool.cue -- - package home +import ( + h "tool/http" + "tool/cli" +) + command: http: { task: testserver: { kind: "testserver" url: string } - task: http: { - kind: "http" - method: "POST" + task: http: h.Post & { url: task.testserver.url request: body: "I'll be back!" response: body: string // TODO: allow this to be a struct, parsing the body. } - task: print: { - kind: "print" + task: print: cli.Print & { text: task.http.response.body } } diff --git a/internal/task/task.go b/internal/task/task.go index e8e585fd09b..135c56e485c 100644 --- a/internal/task/task.go +++ b/internal/task/task.go @@ -52,8 +52,6 @@ func (c *Context) Int64(field string) int64 { f := c.Obj.Lookup(field) value, err := f.Int64() if err != nil { - // TODO: use v for position for now, as f has not yet a - // position associated with it. c.addErr(f, err, "invalid integer argument") return 0 } @@ -64,8 +62,6 @@ func (c *Context) String(field string) string { f := c.Obj.Lookup(field) value, err := f.String() if err != nil { - // TODO: use v for position for now, as f has not yet a - // position associated with it. c.addErr(f, err, "invalid string argument") return "" } @@ -82,6 +78,16 @@ func (c *Context) Bytes(field string) []byte { return value } +func (c *Context) BoolPath(path cue.Path) bool { + f := c.Obj.LookupPath(path) + value, err := f.Bool() + if err != nil { + c.addErr(f, err, "invalid bool argument") + return false + } + return value +} + func (c *Context) addErr(v cue.Value, wrap error, format string, args ...interface{}) { err := &taskError{ diff --git a/pkg/tool/http/doc.go b/pkg/tool/http/doc.go index 9941311c543..0884f0fa1e4 100644 --- a/pkg/tool/http/doc.go +++ b/pkg/tool/http/doc.go @@ -15,6 +15,14 @@ // method: string // url: string // TODO: make url.URL type // +// tls: { +// // Whether the server certificate must be validated. +// verify: *true | bool +// // PEM encoded certificate(s) to validate the server certificate. +// // If not set the CA bundle of the system is used. +// caCert?: bytes | string +// } +// // request: { // body?: bytes | string // header: [string]: string | [...string] diff --git a/pkg/tool/http/http.cue b/pkg/tool/http/http.cue index 1526561d16e..19bcd033623 100644 --- a/pkg/tool/http/http.cue +++ b/pkg/tool/http/http.cue @@ -25,6 +25,14 @@ Do: { method: string url: string // TODO: make url.URL type + tls: { + // Whether the server certificate must be validated. + verify: *true | bool + // PEM encoded certificate(s) to validate the server certificate. + // If not set the CA bundle of the system is used. + caCert?: bytes | string + } + request: { body?: bytes | string header: [string]: string | [...string] diff --git a/pkg/tool/http/http.go b/pkg/tool/http/http.go index e6651bed70f..61bff5133a9 100644 --- a/pkg/tool/http/http.go +++ b/pkg/tool/http/http.go @@ -19,11 +19,15 @@ package http import ( "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" "io" "io/ioutil" "net/http" "cuelang.org/go/cue" + "cuelang.org/go/cue/errors" "cuelang.org/go/internal/task" ) @@ -43,8 +47,9 @@ func newHTTPCmd(v cue.Value) (task.Runner, error) { func (c *httpCmd) Run(ctx *task.Context) (res interface{}, err error) { var header, trailer http.Header var ( - method = ctx.String("method") - u = ctx.String("url") + method = ctx.String("method") + u = ctx.String("url") + tlsVerify = ctx.BoolPath(cue.ParsePath("tls.verify")) ) var r io.Reader if obj := ctx.Obj.Lookup("request"); obj.Exists() { @@ -63,10 +68,50 @@ func (c *httpCmd) Run(ctx *task.Context) (res interface{}, err error) { return nil, err } } + + var caCert []byte + caCertValue := ctx.Obj.LookupPath(cue.ParsePath("tls.caCert")) + if caCertValue.Exists() { + caCert, err = caCertValue.Bytes() + if err != nil { + return nil, errors.Wrapf(err, caCertValue.Pos(), "invalid bytes value") + } + } + if ctx.Err != nil { return nil, ctx.Err } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{} + + if !tlsVerify { + transport.TLSClientConfig.InsecureSkipVerify = true + } + if tlsVerify && len(caCert) > 0 { + pool := x509.NewCertPool() + for { + block, rest := pem.Decode(caCert) + if block == nil { + break + } + if block.Type == "PUBLIC KEY" { + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, errors.Wrapf(err, ctx.Obj.Pos(), "failed to parse caCert") + } + pool.AddCert(c) + } + caCert = rest + } + transport.TLSClientConfig.RootCAs = pool + } + + client := &http.Client{ + Transport: transport, + // TODO: timeout + } + req, err := http.NewRequest(method, u, r) if err != nil { return nil, err @@ -74,10 +119,8 @@ func (c *httpCmd) Run(ctx *task.Context) (res interface{}, err error) { req.Header = header req.Trailer = trailer - // TODO: - // - retry logic - // - TLS certs - resp, err := http.DefaultClient.Do(req) + // TODO: retry logic + resp, err := client.Do(req) if err != nil { return nil, err } diff --git a/pkg/tool/http/http_test.go b/pkg/tool/http/http_test.go index 6de2640350d..626746a8558 100644 --- a/pkg/tool/http/http_test.go +++ b/pkg/tool/http/http_test.go @@ -15,13 +15,78 @@ package http import ( + "encoding/pem" "fmt" + "net/http" + "net/http/httptest" "strings" "testing" "cuelang.org/go/cue" + "cuelang.org/go/cue/parser" + "cuelang.org/go/internal/task" + "cuelang.org/go/internal/value" ) +func newTLSServer() *httptest.Server { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := `{"foo": "bar"}` + w.Write([]byte(resp)) + })) + return server +} + +func parse(t *testing.T, kind, expr string) cue.Value { + t.Helper() + + x, err := parser.ParseExpr("test", expr) + if err != nil { + t.Fatal(err) + } + var r cue.Runtime + i, err := r.CompileExpr(x) + if err != nil { + t.Fatal(err) + } + return value.UnifyBuiltin(i.Value(), kind) +} + +func TestTLS(t *testing.T) { + s := newTLSServer() + defer s.Close() + + v1 := parse(t, "tool/http.Get", fmt.Sprintf(`{url: "%s"}`, s.URL)) + _, err := (*httpCmd).Run(nil, &task.Context{Obj: v1}) + if err == nil { + t.Fatal("http call should have failed") + } + + v2 := parse(t, "tool/http.Get", fmt.Sprintf(`{url: "%s", tls: verify: false}`, s.URL)) + _, err = (*httpCmd).Run(nil, &task.Context{Obj: v2}) + if err != nil { + t.Fatal(err) + } + + publicKeyBlock := pem.Block{ + Type: "PUBLIC KEY", + Bytes: s.Certificate().Raw, + } + publicKeyPem := pem.EncodeToMemory(&publicKeyBlock) + + v3 := parse(t, "tool/http.Get", fmt.Sprintf(` + { + url: "%s" + tls: caCert: ''' +%s +''' + }`, s.URL, publicKeyPem)) + + _, err = (*httpCmd).Run(nil, &task.Context{Obj: v3}) + if err != nil { + t.Fatal(err) + } +} + func TestParseHeaders(t *testing.T) { req := ` header: { diff --git a/pkg/tool/http/pkg.go b/pkg/tool/http/pkg.go index e1683ebb632..3b8b4e710af 100644 --- a/pkg/tool/http/pkg.go +++ b/pkg/tool/http/pkg.go @@ -35,6 +35,10 @@ var pkg = &internal.Package{ $id: *"tool/http.Do" | "http" method: string url: string + tls: { + verify: *true | bool + caCert?: bytes | string + } request: { body?: bytes | string header: {