Remove pgzip and use tar directly

This is basically a revert of
#336.

We found that the bosh cli with the pgzip code produces releases that
cannot be compiled by the bosh agent. In addition, there were many many
edge cases that we found we had to fix and/or paper over that have been
solved in the tar executable over the past decades.

Maybe in the future we can give this another go, as the performance
improvements would be welcome.

Signed-off-by: Chris Selzo <[email protected]>

agent/bootstrap_test.go

@@ -1103,7 +1103,7 @@ var _ = Describe("bootstrap", func() {
 				linuxCdrom := boshcdrom.NewLinuxCdrom("/dev/sr0", udev, runner)
 				linuxCdutil := boshcdrom.NewCdUtil(dirProvider.SettingsDir(), fs, linuxCdrom, logger)
 
-				compressor := boshcmd.NewTarballCompressor(fs)
+				compressor := boshcmd.NewTarballCompressor(runner, fs)
 				copier := boshcmd.NewGenericCpCopier(fs, logger)
 				logsTarProvider := boshlogstarprovider.NewLogsTarProvider(compressor, copier, dirProvider)
 

go.mod

@@ -9,15 +9,14 @@ require (
 	github.com/charlievieth/fs v0.0.3
 	github.com/cloudfoundry/bosh-cli/v7 v7.8.3
 	github.com/cloudfoundry/bosh-davcli v0.0.384
-	github.com/cloudfoundry/bosh-utils v0.0.513
+	github.com/cloudfoundry/bosh-utils v0.0.515
 	github.com/cloudfoundry/gosigar v1.3.79
 	github.com/containerd/cgroups v1.1.0
 	github.com/coreos/go-iptables v0.8.0
 	github.com/gofrs/uuid v4.4.0+incompatible
 	github.com/golang/mock v1.6.0
 	github.com/google/uuid v1.6.0
 	github.com/kevinburke/ssh_config v1.2.0
-	github.com/klauspost/pgzip v1.2.6
 	github.com/masterzen/winrm v0.0.0-20240702205601-3fad6e106085
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.9.0
 	github.com/mitchellh/mapstructure v1.5.0

go.sum

@@ -32,8 +32,8 @@ github.com/cloudfoundry/bosh-cli/v7 v7.8.3 h1:IVi2Oyo69WLxJuBpqcHJUTkoy1JDM0y5xx
 github.com/cloudfoundry/bosh-cli/v7 v7.8.3/go.mod h1:beOLxNr/c16x5hTFWyheVVNBIhLdE14r/zkBxuEI3HE=
 github.com/cloudfoundry/bosh-davcli v0.0.384 h1:dL0SjRVLrYfZQx8GeyX8ygiHZd0N8UgYNqqy4AS4w9c=
 github.com/cloudfoundry/bosh-davcli v0.0.384/go.mod h1:gTVXrUtySvUqqp4U++OxtBhfMKw2prKlonTmNrzFXa0=
-github.com/cloudfoundry/bosh-utils v0.0.513 h1:PV3qhdX5ZcAB8870hqeB9TB/4pEkRsefig9vLtBUZP4=
-github.com/cloudfoundry/bosh-utils v0.0.513/go.mod h1:GCh00+zqULOUWDhIeYYjXP0vghoQgi1uX+RzC8q9rMU=
+github.com/cloudfoundry/bosh-utils v0.0.515 h1:hAQlg5mGVjHf3f4DKTyIA/wT4l91q7FOb3uIlj+KM3o=
+github.com/cloudfoundry/bosh-utils v0.0.515/go.mod h1:Yl7mT/Fy99deXXwaZZVUTf9VC878kUuoTS3h1F9Dzho=
 github.com/cloudfoundry/go-socks5 v0.0.0-20240831012420-2590b55236ee h1:88ruSYvCUKX2YcF2CMYVTmPGITvNdRbzaBRk2c/iMds=
 github.com/cloudfoundry/go-socks5 v0.0.0-20240831012420-2590b55236ee/go.mod h1:72EEm1oq5oXqGfu9XGtaRPWEcAFYd/P10cMNln0QhA8=
 github.com/cloudfoundry/gosigar v1.3.79 h1:ut3L/8A7rj4QYo8114MVDTToLFQgGlzHRVGXAWQNIMI=
@@ -153,8 +153,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
-github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
-github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

integration/windows/windows_test.go

@@ -66,8 +66,9 @@ var _ = Describe("An Agent running on Windows", func() {
 		blobstoreClient = utils.NewBlobstore(windowsutils.BlobstoreURI())
 
 		logger := boshlog.NewLogger(boshlog.LevelNone)
+		cmdRunner := boshsys.NewExecCmdRunner(logger)
 		fs = boshsys.NewOsFileSystem(logger)
-		compressor := boshfileutil.NewTarballCompressor(fs)
+		compressor := boshfileutil.NewTarballCompressor(cmdRunner, fs)
 
 		natsClient = utils.NewNatsClient(compressor, blobstoreClient, windowsutils.FakeDirectorIP())
 		err := natsClient.Setup()

platform/dummy_platform.go

@@ -69,7 +69,7 @@ func NewDummyPlatform(
 		fs:                 fs,
 		cmdRunner:          cmdRunner,
 		collector:          collector,
-		compressor:         boshcmd.NewTarballCompressor(fs),
+		compressor:         boshcmd.NewTarballCompressor(cmdRunner, fs),
 		copier:             boshcmd.NewGenericCpCopier(fs, logger),
 		dirProvider:        dirProvider,
 		devicePathResolver: devicePathResolver,

platform/linux_platform_test.go

@@ -19,6 +19,10 @@ import (
 
 	. "github.com/cloudfoundry/bosh-agent/platform"
 
+	fakeretry "github.com/cloudfoundry/bosh-utils/retrystrategy/fakes"
+	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
+	fakeuuidgen "github.com/cloudfoundry/bosh-utils/uuid/fakes"
+
 	fakedpresolv "github.com/cloudfoundry/bosh-agent/infrastructure/devicepathresolver/fakes"
 	fakecdrom "github.com/cloudfoundry/bosh-agent/platform/cdrom/fakes"
 	"github.com/cloudfoundry/bosh-agent/platform/cert/certfakes"
@@ -28,17 +32,15 @@ import (
 	fakenet "github.com/cloudfoundry/bosh-agent/platform/net/fakes"
 	fakestats "github.com/cloudfoundry/bosh-agent/platform/stats/fakes"
 	"github.com/cloudfoundry/bosh-agent/servicemanager/servicemanagerfakes"
-	fakeretry "github.com/cloudfoundry/bosh-utils/retrystrategy/fakes"
-	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
-	fakeuuidgen "github.com/cloudfoundry/bosh-utils/uuid/fakes"
+
+	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	boshsys "github.com/cloudfoundry/bosh-utils/system"
 
 	boshdisk "github.com/cloudfoundry/bosh-agent/platform/disk"
 	boshvitals "github.com/cloudfoundry/bosh-agent/platform/vitals"
 	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
 	boshdirs "github.com/cloudfoundry/bosh-agent/settings/directories"
-	boshcmd "github.com/cloudfoundry/bosh-utils/fileutil"
-	boshlog "github.com/cloudfoundry/bosh-utils/logger"
-	boshsys "github.com/cloudfoundry/bosh-utils/system"
 )
 
 var _ = Describe("LinuxPlatform", func() {
@@ -85,7 +87,7 @@ var _ = Describe("LinuxPlatform", func() {
 		cmdRunner = fakesys.NewFakeCmdRunner()
 		dirProvider = boshdirs.NewProvider("/fake-dir")
 		cdutil = fakecdrom.NewFakeCDUtil()
-		compressor = boshcmd.NewTarballCompressor(fs)
+		compressor = boshcmd.NewTarballCompressor(cmdRunner, fs)
 		copier = boshcmd.NewGenericCpCopier(fs, logger)
 		netManager = &fakenet.FakeManager{}
 		certManager = new(certfakes.FakeManager)

platform/provider.go

@@ -70,7 +70,7 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl
 	linuxCdrom := boshcdrom.NewLinuxCdrom("/dev/sr0", udev, runner)
 	linuxCdutil := boshcdrom.NewCdUtil(dirProvider.SettingsDir(), fs, linuxCdrom, logger)
 
-	compressor := boshcmd.NewTarballCompressor(fs)
+	compressor := boshcmd.NewTarballCompressor(runner, fs)
 	copier := boshcmd.NewGenericCpCopier(fs, logger)
 
 	// Kick of stats collection as soon as possible

platform/windows_platform.go

@@ -87,7 +87,7 @@ func NewWindowsPlatform(
 		fs:                     fs,
 		cmdRunner:              cmdRunner,
 		collector:              collector,
-		compressor:             boshcmd.NewTarballCompressor(fs),
+		compressor:             boshcmd.NewTarballCompressor(cmdRunner, fs),
 		copier:                 boshcmd.NewGenericCpCopier(fs, logger),
 		dirProvider:            dirProvider,
 		netManager:             netManager,

releasetarball/compile.go

@@ -12,6 +12,7 @@ import (
 	"bufio"
 	"bytes"
 	"cmp"
+	"compress/gzip"
 	"errors"
 	"fmt"
 	"io"
@@ -25,8 +26,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/klauspost/pgzip"
-
 	"gopkg.in/yaml.v3"
 
 	"code.cloudfoundry.org/clock"
@@ -60,7 +59,7 @@ func NewCompiler(dirProvider directories.Provider) (boshcomp.Compiler, error) {
 	logger := boshlog.New(boshlog.LevelWarn, log.Default())
 	cmdRunner := boshsys.NewExecCmdRunner(logger)
 	filesystem := boshsys.NewOsFileSystem(logger)
-	compressor := boshcmd.NewTarballCompressor(filesystem)
+	compressor := boshcmd.NewTarballCompressor(cmdRunner, filesystem)
 	blobstoreProvider := boshblob.NewProvider(filesystem, cmdRunner, dirProvider.EtcDir(), logger)
 	db, err := blobstoreProvider.Get("local", map[string]any{"blobstore_path": dirProvider.BlobsDir()})
 	if err != nil {
@@ -221,7 +220,7 @@ func writeCompiledRelease(m manifest.Manifest, outputDirectory, stemcellFilename
 		return "", err
 	}
 	defer closeAndIgnoreErr(outputFile)
-	gw := pgzip.NewWriter(outputFile)
+	gw := gzip.NewWriter(outputFile)
 	defer closeAndIgnoreErr(gw)
 	tw := tar.NewWriter(gw)
 	defer closeAndIgnoreErr(tw)
@@ -321,7 +320,7 @@ func walkTarballFiles(releaseFilePath string, file tarballWalkFunc) error {
 		return nil
 	}
 	defer closeAndIgnoreErr(f)
-	gr, err := pgzip.NewReader(bufio.NewReader(f))
+	gr, err := gzip.NewReader(bufio.NewReader(f))
 	if err != nil {
 		return err
 	}