pkg/tool/http: add tls settings

tls.verify can be used to disable server certificate validation.
tls.caCert can be used to provide a PEM encoded certificates to validate
the server certificate.

Closes #1558

Signed-off-by: Jean-Philippe Braun <[email protected]>
Change-Id: If8f0aa5d9f882675e84e2546faa510f7d3bcde1c
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/533845
Unity-Result: CUEcueckoo <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
Reviewed-by: Marcel van Lohuizen <[email protected]>

cmd/cue/cmd/testdata/script/cmd_http.txt

@@ -4,24 +4,25 @@ cmp stdout cmd_http.out
 {"data":"I'll be back!","when":"now"}
 
 -- task_tool.cue --
-
 package home
 
+import (
+	h "tool/http"
+	"tool/cli"
+)
+
 command: http: {
 	task: testserver: {
 		kind: "testserver"
 		url:  string
 	}
-	task: http: {
-		kind:   "http"
-		method: "POST"
+	task: http: h.Post & {
 		url:    task.testserver.url
 
 		request: body:  "I'll be back!"
 		response: body: string // TODO: allow this to be a struct, parsing the body.
 	}
-	task: print: {
-		kind: "print"
+	task: print: cli.Print & {
 		text: task.http.response.body
 	}
 }

internal/task/task.go

@@ -52,8 +52,6 @@ func (c *Context) Int64(field string) int64 {
 	f := c.Obj.Lookup(field)
 	value, err := f.Int64()
 	if err != nil {
-		// TODO: use v for position for now, as f has not yet a
-		// position associated with it.
 		c.addErr(f, err, "invalid integer argument")
 		return 0
 	}
@@ -64,8 +62,6 @@ func (c *Context) String(field string) string {
 	f := c.Obj.Lookup(field)
 	value, err := f.String()
 	if err != nil {
-		// TODO: use v for position for now, as f has not yet a
-		// position associated with it.
 		c.addErr(f, err, "invalid string argument")
 		return ""
 	}
@@ -82,6 +78,16 @@ func (c *Context) Bytes(field string) []byte {
 	return value
 }
 
+func (c *Context) BoolPath(path cue.Path) bool {
+	f := c.Obj.LookupPath(path)
+	value, err := f.Bool()
+	if err != nil {
+		c.addErr(f, err, "invalid bool argument")
+		return false
+	}
+	return value
+}
+
 func (c *Context) addErr(v cue.Value, wrap error, format string, args ...interface{}) {
 
 	err := &taskError{

pkg/tool/http/http.cue

@@ -25,6 +25,14 @@ Do: {
 	method: string
 	url:    string // TODO: make url.URL type
 
+	tls: {
+		// Whether the server certificate must be validated.
+		verify: *true | bool
+		// PEM encoded certificate(s) to validate the server certificate.
+		// If not set the CA bundle of the system is used.
+		caCert?: bytes | string
+	}
+
 	request: {
 		body?: bytes | string
 		header: [string]:  string | [...string]

pkg/tool/http/http.go

@@ -19,11 +19,15 @@ package http
 
 import (
 	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"io"
 	"io/ioutil"
 	"net/http"
 
 	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/errors"
 	"cuelang.org/go/internal/task"
 )
 
@@ -43,8 +47,9 @@ func newHTTPCmd(v cue.Value) (task.Runner, error) {
 func (c *httpCmd) Run(ctx *task.Context) (res interface{}, err error) {
 	var header, trailer http.Header
 	var (
-		method = ctx.String("method")
-		u      = ctx.String("url")
+		method    = ctx.String("method")
+		u         = ctx.String("url")
+		tlsVerify = ctx.BoolPath(cue.ParsePath("tls.verify"))
 	)
 	var r io.Reader
 	if obj := ctx.Obj.Lookup("request"); obj.Exists() {
@@ -63,21 +68,59 @@ func (c *httpCmd) Run(ctx *task.Context) (res interface{}, err error) {
 			return nil, err
 		}
 	}
+
+	var caCert []byte
+	caCertValue := ctx.Obj.LookupPath(cue.ParsePath("tls.caCert"))
+	if caCertValue.Exists() {
+		caCert, err = caCertValue.Bytes()
+		if err != nil {
+			return nil, errors.Wrapf(err, caCertValue.Pos(), "invalid bytes value")
+		}
+	}
+
 	if ctx.Err != nil {
 		return nil, ctx.Err
 	}
 
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.TLSClientConfig = &tls.Config{}
+
+	if !tlsVerify {
+		transport.TLSClientConfig.InsecureSkipVerify = true
+	}
+	if tlsVerify && len(caCert) > 0 {
+		pool := x509.NewCertPool()
+		for {
+			block, rest := pem.Decode(caCert)
+			if block == nil {
+				break
+			}
+			if block.Type == "PUBLIC KEY" {
+				c, err := x509.ParseCertificate(block.Bytes)
+				if err != nil {
+					return nil, errors.Wrapf(err, ctx.Obj.Pos(), "failed to parse caCert")
+				}
+				pool.AddCert(c)
+			}
+			caCert = rest
+		}
+		transport.TLSClientConfig.RootCAs = pool
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		// TODO: timeout
+	}
+
 	req, err := http.NewRequest(method, u, r)
 	if err != nil {
 		return nil, err
 	}
 	req.Header = header
 	req.Trailer = trailer
 
-	// TODO:
-	//  - retry logic
-	//  - TLS certs
-	resp, err := http.DefaultClient.Do(req)
+	// TODO: retry logic
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}

pkg/tool/http/http_test.go

@@ -15,13 +15,78 @@
 package http
 
 import (
+	"encoding/pem"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
 
 	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/parser"
+	"cuelang.org/go/internal/task"
+	"cuelang.org/go/internal/value"
 )
 
+func newTLSServer() *httptest.Server {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		resp := `{"foo": "bar"}`
+		w.Write([]byte(resp))
+	}))
+	return server
+}
+
+func parse(t *testing.T, kind, expr string) cue.Value {
+	t.Helper()
+
+	x, err := parser.ParseExpr("test", expr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var r cue.Runtime
+	i, err := r.CompileExpr(x)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return value.UnifyBuiltin(i.Value(), kind)
+}
+
+func TestTLS(t *testing.T) {
+	s := newTLSServer()
+	defer s.Close()
+
+	v1 := parse(t, "tool/http.Get", fmt.Sprintf(`{url: "%s"}`, s.URL))
+	_, err := (*httpCmd).Run(nil, &task.Context{Obj: v1})
+	if err == nil {
+		t.Fatal("http call should have failed")
+	}
+
+	v2 := parse(t, "tool/http.Get", fmt.Sprintf(`{url: "%s", tls: verify: false}`, s.URL))
+	_, err = (*httpCmd).Run(nil, &task.Context{Obj: v2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	publicKeyBlock := pem.Block{
+		Type:  "PUBLIC KEY",
+		Bytes: s.Certificate().Raw,
+	}
+	publicKeyPem := pem.EncodeToMemory(&publicKeyBlock)
+
+	v3 := parse(t, "tool/http.Get", fmt.Sprintf(`
+	{
+		url: "%s"
+		tls: caCert: '''
+%s
+'''
+	}`, s.URL, publicKeyPem))
+
+	_, err = (*httpCmd).Run(nil, &task.Context{Obj: v3})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestParseHeaders(t *testing.T) {
 	req := `
 	header: {