Add HTTPS support to Varnish cache using Hitch

cache-config/t3c-apply/t3c-apply.go

@@ -334,6 +334,25 @@ func Main() int {
 		}
 	}
 
+	if trops.HitchReload {
+		svcStatus, _, err := util.GetServiceStatus("hitch")
+		cmd := "start"
+		running := false
+		if err != nil {
+			log.Errorf("not starting 'hitch', error getting 'hitch' run status: %s\n", err)
+		} else if svcStatus != util.SvcNotRunning {
+			cmd = "reload"
+		}
+		running, err = util.ServiceStart("hitch", cmd)
+		if err != nil {
+			log.Errorf("'hitch' was not %sed: %s\n", cmd, err)
+		} else if running {
+			log.Infof("service 'hitch' %sed", cmd)
+		} else {
+			log.Infoln("service 'hitch' already running")
+		}
+	}
+
 	// reload sysctl
 	if trops.SysCtlReload == true {
 		runSysctl(cfg)

cache-config/t3c-apply/torequest/torequest.go

@@ -90,6 +90,7 @@
 	TeakdRestart         bool // a restart of teakd is required
 	TrafficServerRestart bool // a trafficserver restart is required
 	RemapConfigReload    bool // remap.config should be reloaded
+	HitchReload          bool // hitch should be reloaded
 }
 
 type ConfigFile struct {
@@ -208,33 +209,33 @@
 	}
 
 	if cfg.Dir == "" {
 		cfg.AuditFailed = true
 		return errors.New("No location information for " + cfg.Name)
 	}
 	// return if audit has already been done.
 	if cfg.AuditComplete {
 		return nil
 	}
 
 	if !util.MkDirWithOwner(cfg.Dir, r.Cfg.ReportOnly, &cfg.Uid, &cfg.Gid) {
 		cfg.AuditFailed = true
 		return errors.New("Unable to create the directory '" + cfg.Dir + " for " + "'" + cfg.Name + "'")
 	}
 
 	log.Debugf("======== Start processing config file: %s ========\n", cfg.Name)
 
 	if cfg.Name == "50-ats.rules" {
 		err := r.processUdevRules(cfg)
 		if err != nil {
 			cfg.AuditFailed = true
 			return errors.New("unable to process udev rules in '" + cfg.Name + "': " + err.Error())
 		}
 	}
 
 	if cfg.Name == "remap.config" {
 		err := r.processRemapOverrides(cfg)
 		if err != nil {
 			cfg.AuditFailed = true
 			return err
 		}
 	}
@@ -243,7 +244,7 @@
 	if cfg.Name == "remap.config" || cfg.Name == "plugin.config" {
 		if err := checkRefs(r.Cfg, cfg.Body, filesAdding); err != nil {
 			r.configFileWarnings[cfg.Name] = append(r.configFileWarnings[cfg.Name], "failed to verify '"+cfg.Name+"': "+err.Error())
 			cfg.AuditFailed = true
 			return errors.New("failed to verify '" + cfg.Name + "': " + err.Error())
 		}
 		log.Infoln("Successfully verified plugins used by '" + cfg.Name + "'")
@@ -263,7 +264,7 @@
 	changeNeeded, err := diff(r.Cfg, cfg.Body, cfg.Path, r.Cfg.ReportOnly, cfg.Perm, cfg.Uid, cfg.Gid)
 
 	if err != nil {
 		cfg.AuditFailed = true
 		return errors.New("getting diff: " + err.Error())
 	}
 	cfg.ChangeNeeded = changeNeeded
@@ -424,7 +425,7 @@
 			}
 		}
 	}
 	fs, err := os.ReadDir("/proc/fs/ext4")
 	if err != nil {
 		log.Errorln("unable to read /proc/fs/ext4, cannot audit disks for filesystem usage.")
 	} else {
@@ -520,6 +521,7 @@
 	trafficServerRestart := cfg.Name == "plugin.config"
 	ntpdRestart := cfg.Name == "ntpd.conf"
 	sysCtlReload := cfg.Name == "sysctl.conf"
+	hitchReload := cfg.Name == "hitch.conf"
 
 	log.Debugf("Reload state after %s: remap.config: %t reload: %t restart: %t ntpd: %t sysctl: %t", cfg.Name, remapConfigReload, trafficCtlReload, trafficServerRestart, ntpdRestart, sysCtlReload)
 
@@ -532,6 +534,7 @@
 			NtpdRestart:          ntpdRestart,
 			TrafficServerRestart: trafficServerRestart,
 			RemapConfigReload:    remapConfigReload,
+			HitchReload:          hitchReload,
 		},
 	}, nil
 }
@@ -748,7 +751,7 @@
 
 // CheckSyncDSState retrieves and returns the DS Update status from Traffic Ops.
 // The metaData is this run's metadata. It must not be nil, and this function may add to it.
 func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
 	updateStatus := UpdateTropsNotNeeded
 	randDispSec := time.Duration(0)
 	log.Debugln("Checking syncds state.")
@@ -785,7 +788,7 @@
 			}
 		} else if !r.Cfg.IgnoreUpdateFlag {
 			log.Errorln("no queued update needs to be applied.  Running revalidation before exiting.")
 			r.RevalidateWhileSleeping(metaData, cfg)
 			return UpdateTropsNotNeeded, nil
 		} else {
 			log.Errorln("Traffic Ops is signaling that no update is waiting to be applied.")
@@ -810,6 +813,7 @@
 		rd.TeakdRestart = rd.TeakdRestart || changedFile.TeakdRestart
 		rd.TrafficServerRestart = rd.TrafficServerRestart || changedFile.TrafficServerRestart
 		rd.RemapConfigReload = rd.RemapConfigReload || changedFile.RemapConfigReload
+		rd.HitchReload = rd.HitchReload || changedFile.HitchReload
 	}
 	return rd
 }
@@ -817,12 +821,12 @@
 // ProcessConfigFiles processes all config files retrieved from Traffic Ops.
 func (r *TrafficOpsReq) ProcessConfigFiles(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
 	var updateStatus UpdateStatus = UpdateTropsNotNeeded
 	var auditErrors []string
 
 	log.Infoln(" ======== Start processing config files ========")
 
 	filesAdding := []string{} // list of file names being added, needed for verification.
 	for fileName := range r.configFiles {
 		filesAdding = append(filesAdding, fileName)
 	}
 
@@ -848,7 +852,7 @@
 		err := r.checkConfigFile(cfg, filesAdding)
 		if err != nil {
 			log.Errorln(err)
 			r.configFiles[cfg.Name].AuditError = err.Error()
 		}
 	}
 
@@ -865,11 +869,11 @@
 			!cfg.AuditFailed {
 
 			changesRequired++
 			if cfg.Name == "plugin.config" && r.configFiles["remap.config"].PreReqFailed {
 				updateStatus = UpdateTropsFailed
 				log.Errorln("plugin.config changed however, prereqs failed for remap.config so I am skipping updates for plugin.config")
 				continue
 			} else if cfg.Name == "remap.config" && r.configFiles["plugin.config"].PreReqFailed {
 				updateStatus = UpdateTropsFailed
 				log.Errorln("remap.config changed however, prereqs failed for plugin.config so I am skipping updates for remap.config")
 				continue
@@ -884,16 +888,16 @@
 				}
 				shouldRestartReload.ReloadRestart = append(shouldRestartReload.ReloadRestart, *reData)
 			}
 		} else if cfg.AuditFailed {
 			auditErrors = append(auditErrors, cfg.AuditError)
 			log.Warnf("audit failed for config file: %v Error: %s", cfg.Name, cfg.AuditError)
 			updateStatus = UpdateTropsFailed
 		}
 	}
 
 	if updateStatus == UpdateTropsFailed {
 		return UpdateTropsFailed, errors.New(strings.Join(auditErrors, "\n"))
 	}
 
 	r.RestartData = r.CheckReloadRestart(shouldRestartReload.ReloadRestart)
 
@@ -1050,48 +1054,48 @@
 	return nil
 }
 
 func pkgMetaDataToMap(pmd []string) map[string]bool {
 	pkgMap := map[string]bool{}
 	for _, pkg := range pmd {
 		pkgMap[pkg] = true
 	}
 	return pkgMap
 }
 
 func pkgMatch(pkgMetaData []string, pk string) bool {
 	for _, pkg := range pkgMetaData {
 		if strings.Contains(pk, pkg) {
 			return true
 		}
 	}
 	return false
 
 }
 
 // ProcessPackagesWithMetaData will attempt to get installed package data from
 // t3c-apply-metadata.json and log the results.
 func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error {
 	pkgs, err := getPackages(r.Cfg)
 	pkgMdataMap := pkgMetaDataToMap(packageMetaData)
 	if err != nil {
 		return fmt.Errorf("getting packages: %w", err)
 	}
 	for _, pkg := range pkgs {
 		fullPackage := pkg.Name + "-" + pkg.Version
 		if pkgMdataMap[fullPackage] {
 			log.Infof("package %s is assumed to be installed according to metadata file", fullPackage)
 			r.Pkgs[fullPackage] = true
 		} else if pkgMatch(packageMetaData, pkg.Name) {
 			log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage)
 			r.Pkgs[fullPackage] = true
 		} else {
 			log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version)
 		}
 	}
 	return nil
 }
 
 func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
 	updateStatus, err := r.CheckRevalidateState(true)
 	if err != nil {
 		return updateStatus, err
@@ -1115,7 +1119,7 @@
 			t3cutil.WriteActionLog(t3cutil.ActionLogActionUpdateFilesReval, t3cutil.ActionLogStatusSuccess, metaData)
 		}
 
 		if err := r.StartServices(&updateStatus, metaData, cfg); err != nil {
 			return updateStatus, errors.New("failed to start services: " + err.Error())
 		}
 
@@ -1132,7 +1136,7 @@
 // StartServices reloads, restarts, or starts ATS as necessary,
 // according to the changed config files and run mode.
 // Returns nil on success or any error.
 func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData, cfg config.Cfg) error {
 	serviceNeeds := t3cutil.ServiceNeedsNothing
 	if r.Cfg.ServiceAction == t3cutil.ApplyServiceActionFlagRestart {
 		serviceNeeds = t3cutil.ServiceNeedsRestart
@@ -1154,17 +1158,17 @@
 			serviceNeeds = t3cutil.ServiceNeedsReload
 		}
 	}
 	packageName := "trafficserver"
 	if cfg.CacheType == "varnish" {
 		packageName = "varnish"
 	}
 
 	if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled(packageName) {
 		// TODO try to reload/restart anyway? To allow non-RPM installs?
 		return errors.New(packageName + " needs " + serviceNeeds.String() + " but is not installed.")
 	}
 
 	svcStatus, _, err := util.GetServiceStatus(packageName)
 	if err != nil {
 		return errors.New("getting trafficserver service status: " + err.Error())
 	}
@@ -1181,7 +1185,7 @@
 		if svcStatus != util.SvcRunning {
 			startStr = "start"
 		}
 		if _, err := util.ServiceStart(packageName, startStr); err != nil {
 			t3cutil.WriteActionLog(t3cutil.ActionLogActionATSRestart, t3cutil.ActionLogStatusFailure, metaData)
 			return errors.New("failed to restart trafficserver")
 		}
@@ -1208,13 +1212,13 @@
 			log.Errorln("ATS configuration has changed.  The new config will be picked up the next time ATS is started.")
 		} else if serviceNeeds == t3cutil.ServiceNeedsReload {
 			log.Infoln("ATS configuration has changed, Running 'traffic_ctl config reload' now.")
 			reloadCommand := config.TSHome + config.TrafficCtl
 			reloadArgs := []string{"config", "reload"}
 			if cfg.CacheType == "varnish" {
 				reloadCommand = "varnishreload"
 				reloadArgs = []string{}
 			}
 			if _, _, err := util.ExecCommand(reloadCommand, reloadArgs...); err != nil {
 				t3cutil.WriteActionLog(t3cutil.ActionLogActionATSReload, t3cutil.ActionLogStatusFailure, metaData)
 
 				if *syncdsUpdate == UpdateTropsNeeded {

cache-config/t3c-generate/cfgfile/varnish.go

@@ -20,6 +20,9 @@
  */
 
 import (
+	"errors"
+	"path/filepath"
+
 	"github.com/apache/trafficcontrol/cache-config/t3c-generate/config"
 	"github.com/apache/trafficcontrol/cache-config/t3cutil"
 	"github.com/apache/trafficcontrol/lib/varnishcfg"
@@ -27,20 +30,44 @@
 
 // GetVarnishConfigs returns varnish configuration files
 // TODO: add varnishncsa and hitch configs
 func GetVarnishConfigs(toData *t3cutil.ConfigData, cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) {
 	vclBuilder := varnishcfg.NewVCLBuilder(toData)
 	vcl, warnings, err := vclBuilder.BuildVCLFile()
 	logWarnings("Generating varnish configuration files: ", warnings)
 
 	configs := make([]t3cutil.ATSConfigFile, 0)
 	// TODO: should be parameterized and generated from varnishcfg
 	configs = append(configs, t3cutil.ATSConfigFile{
 		Name:        "default.vcl",
 		Text:        vcl,
 		Path:        cfg.Dir,
 		ContentType: "text/plain; charset=us-ascii",
 		LineComment: "//",
 		Secure:      false,
 	})
-	return configs, err
+	txt, hitchWarnings := varnishcfg.GetHitchConfig(toData.DeliveryServices, filepath.Join(cfg.Dir, "ssl/"))
+	warnings = append(warnings, hitchWarnings...)
+	logWarnings("Generating hitch configuration files: ", hitchWarnings)
+
+	configs = append(configs, t3cutil.ATSConfigFile{
+		Name:        "hitch.conf",
+		Text:        txt,
+		Path:        cfg.Dir,
+		ContentType: "text/plain; charset=us-ascii",
+		LineComment: "//",
+		Secure:      false,
+	})
+
+	sslConfigs, err := GetSSLCertsAndKeyFiles(toData)
+	if err != nil {
+		return nil, errors.New("getting ssl key and cert config files: " + err.Error())
+	}
+	for i := range sslConfigs {
+		// path changed manually because GetSSLCertsAndKeyFiles hardcodes the directory certs and keys are written to.
+		// will be removed once GetSSLCertsAndKeyFiles uses proxy.config.ssl.server.cert.path parameter.
+		sslConfigs[i].Path = filepath.Join(cfg.Dir, "ssl/")
+	}
+	configs = append(configs, sslConfigs...)
+
+	return configs, nil
 }

infrastructure/cdn-in-a-box/varnish/Dockerfile

@@ -28,6 +28,8 @@ RUN curl -s https://packagecloud.io/install/repositories/varnishcache/varnish73/
 
 RUN yum install varnish-7.3.0 -y
 
+RUN yum install -y hitch-1.5.2
+
 RUN dnf install -y bind-utils kyotocabinet-libs initscripts iproute net-tools nmap-ncat gettext autoconf automake libtool gcc-c++ cronie glibc-devel openssl-devel git perl && \
     dnf install -y jq logrotate findutils && \
     dnf clean all

infrastructure/cdn-in-a-box/varnish/systemctl.sh

@@ -18,6 +18,7 @@
 # under the License.
 
 VARNISHD_EXECUTABLE="/usr/sbin/varnishd"
+HITCH_EXECUTABLE="/usr/sbin/hitch"
 
 is_varnishd_running() {
   pgrep -x "$(basename "$VARNISHD_EXECUTABLE")" >/dev/null
@@ -28,7 +29,7 @@ start_varnishd() {
     echo "varnishd is already running."
   else
     echo "Starting varnishd..."
-    "$VARNISHD_EXECUTABLE" -f /opt/cache/etc/varnish/default.vcl
+    "$VARNISHD_EXECUTABLE" -f /opt/cache/etc/varnish/default.vcl -a :80 -a :6081,PROXY
     echo "varnishd is now running."
   fi
 }
@@ -67,28 +68,90 @@ restart_varnishd() {
   start_varnishd
 }
 
-case "$1" in
-  enable)
-    exit 0
-    ;;
-  start)
-    start_varnishd
-    ;;
-  stop)
-    stop_varnishd
-    ;;
-  restart)
-    restart_varnishd
-    ;;
-  status)
-    if is_varnishd_running; then
-      exit 0
-    fi
-    exit 3
-    ;;
-  *)
-    echo "Usage: $0 {start|stop|restart|enable|status}"
+is_hitch_running() {
+  pgrep -x "$(basename "$HITCH_EXECUTABLE")" >/dev/null
+}
+
+
+start_hitch() {
+  if is_hitch_running; then
+    echo "hitch is already running."
+  else
+    echo "Starting hitch..."
+    "$HITCH_EXECUTABLE" --config /opt/cache/etc/varnish/hitch.conf --daemon
+    echo "hitch is now running."
+  fi
+
+}
+
+reload_hitch() {
+  if is_hitch_running; then
+    pkill -HUP "$(basename "$HITCH_EXECUTABLE")"
+  else
+    echo "hitch is not running"
     exit 1
-esac
+  fi
+}
+
+handle_varnish_service() {
+  case "$1" in
+    enable)
+      exit 0
+      ;;
+    start)
+      start_varnishd
+      ;;
+    stop)
+      stop_varnishd
+      ;;
+    restart)
+      restart_varnishd
+      ;;
+    status)
+      if is_varnishd_running; then
+        # t3c-apply looks for this specific string
+        echo "Active: active"
+        exit 0
+      fi
+      exit 3
+      ;;
+    *)
+      echo "Usage: $0 {start|stop|restart|enable|status}"
+      exit 1
+  esac
+}
+
+handle_hitch_service() {
+  case "$1" in
+    enable)
+      exit 0
+      ;;
+    start)
+      start_hitch
+      ;;
+    reload)
+      reload_hitch
+      ;;
+    status)
+      if is_hitch_running; then
+        # t3c-apply looks for this specific string
+        echo "Active: active"
+        exit 0
+      fi
+      exit 3
+      ;;
+    *)
+      echo "Usage: $0 {start|stop|restart|reload|enable|status}"
+      exit 1
+  esac
+}
+
+if [[ $2 == "varnish.service" ]]
+then
+  handle_varnish_service $1
+elif [[ $2 == "hitch.service" ]]
+then
+  handle_hitch_service $1
+fi
 
 exit 0

lib/varnishcfg/hitch.go

@@ -0,0 +1,61 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/apache/trafficcontrol/lib/go-atscfg"
+)
+
+// GetHitchConfig returns Hitch config using TO data
+func GetHitchConfig(deliveryServices []atscfg.DeliveryService, sslDir string) (string, []string) {
+	warnings := make([]string, 0)
+	lines := []string{
+		`frontend = {`,
+		`	host = "*"`,
+		`	port = "443"`,
+		`}`,
+		`backend = "[127.0.0.1]:6081"`,
+		`write-proxy-v2 = on`,
+		// TODO: change root user
+		`user = "root"`,
+	}
+
+	dses, dsWarns := atscfg.DeliveryServicesToSSLMultiCertDSes(deliveryServices)
+	warnings = append(warnings, dsWarns...)
+
+	dses = atscfg.GetSSLMultiCertDotConfigDeliveryServices(dses)
+
+	for dsName, ds := range dses {
+		cerName, keyName := atscfg.GetSSLMultiCertDotConfigCertAndKeyName(dsName, ds)
+		lines = append(lines, []string{
+			`pem-file = {`,
+			`	cert = "` + filepath.Join(sslDir, cerName) + `"`,
+			`	private-key = "` + filepath.Join(sslDir, keyName) + `"`,
+			`}`,
+		}...)
+	}
+
+	txt := strings.Join(lines, "\n")
+	txt += "\n"
+	return txt, warnings
+}

lib/varnishcfg/hitch_test.go

@@ -0,0 +1,60 @@
+package varnishcfg
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/apache/trafficcontrol/lib/go-atscfg"
+	"github.com/apache/trafficcontrol/lib/go-tc"
+	"github.com/apache/trafficcontrol/lib/go-util"
+)
+
+func TestGetHitchConfig(t *testing.T) {
+	ds1 := &atscfg.DeliveryService{}
+	ds1.XMLID = util.StrPtr("ds1")
+	ds1.Protocol = util.IntPtr(1)
+	ds1Type := tc.DSTypeHTTP
+	ds1.Type = &ds1Type
+	ds1.ExampleURLs = []string{"https://ds1.example.org"}
+	deliveryServices := []atscfg.DeliveryService{*ds1}
+	txt, warnings := GetHitchConfig(deliveryServices, "/ssl")
+	expectedTxt := strings.Join([]string{
+		`frontend = {`,
+		`	host = "*"`,
+		`	port = "443"`,
+		`}`,
+		`backend = "[127.0.0.1]:6081"`,
+		`write-proxy-v2 = on`,
+		`user = "root"`,
+		`pem-file = {`,
+		`	cert = "/ssl/ds1_example_org_cert.cer"`,
+		`	private-key = "/ssl/ds1.example.org.key"`,
+		`}`,
+	}, "\n")
+	expectedTxt += "\n"
+	if len(warnings) != 0 {
+		t.Errorf("expected no warnings got %v", warnings)
+	}
+	if txt != expectedTxt {
+		t.Errorf("expected: %s got: %s", expectedTxt, txt)
+	}
+}