Fixing tests and fix cleanup.

Signed-off-by: Christoph Fiehe <[email protected]>

api/publish.go

@@ -438,8 +438,9 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		}
 
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
-			err = collection.CleanupPrefixComponentFiles(context, published, result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(),
-				collectionFactory, out)
+			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
@@ -846,8 +847,9 @@ func apiPublishUpdate(c *gin.Context) {
 		}
 
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
-			err = collection.CleanupPrefixComponentFiles(context, published,
-				result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, out)
+			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}

cmd/mirror_update.go

@@ -267,7 +267,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
 	return err
 }
 

cmd/publish_switch.go

@@ -115,8 +115,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published,
-			[]string{}, components, []string{}, collectionFactory, context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}

cmd/publish_update.go

@@ -76,14 +76,15 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published,
-			result.AddedComponents(), result.UpdatedComponents(), result.RemovedComponents(), collectionFactory, context.Progress())
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublished %s repository %s has been successfully updated.\n", published.SourceKind, published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
 
 	return err
 }

deb/publish.go

@@ -1462,73 +1462,127 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
-	published *PublishedRepo, addedComponents, updatedComponents, removedComponents []string,
-	collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+
+	var err error
 
 	collection.loadList()
 
+	storage := published.Storage
 	prefix := published.Prefix
 	distribution := published.Distribution
-	multiDist := published.MultiDist
+
+	rootPath := filepath.Join(prefix, "dists", distribution)
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
 
-	components := make([]string, 0, len(addedComponents)+len(updatedComponents)+len(removedComponents))
-	components = append(append(append(components, addedComponents...), updatedComponents...), removedComponents...)
-	sort.Strings(components)
+	sort.Strings(cleanComponents)
+	publishedComponents := published.Components()
+	removedComponents := utils.StrSlicesSubstract(cleanComponents, publishedComponents)
+	updatedComponents := utils.StrSlicesSubstract(cleanComponents, removedComponents)
+
+	if progress != nil {
+		progress.Printf("Cleaning up published repository %s/%s...\n", published.StoragePrefix(), distribution)
+	}
 
 	for _, component := range removedComponents {
 		if progress != nil {
-			progress.Printf("Removing component %q from prefix %q...\n", component, prefix)
+			progress.Printf("Removing component %q...\n", component)
 		}
 
-		err := publishedStorage.RemoveDirs(filepath.Join(prefix, "dists", distribution, component), progress)
+		err = publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
 		if err != nil {
 			return err
 		}
 
-		if multiDist {
-			for _, component := range removedComponents {
-				err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), progress)
+		// Ensure that component does not exist in multi distribution pool
+		err = publishedStorage.RemoveDirs(filepath.Join(prefix, "pool", distribution, component), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	referencedFiles := map[string][]string{}
+
+	if published.MultiDist {
+		rootPath = filepath.Join(prefix, "pool", distribution)
+
+		// Get all referenced files by component for determining orphaned pool files.
+		for _, component := range publishedComponents {
+			packageList, err := NewPackageListFromRefList(published.RefList(component), collectionFactory.PackageCollection(), progress)
+			if err != nil {
+				return err
+			}
+
+			packageList.ForEach(func(p *Package) error {
+				poolDir, err := p.PoolDirectory()
 				if err != nil {
 					return err
 				}
+
+				for _, file := range p.Files() {
+					referencedFiles[component] = append(referencedFiles[component], filepath.Join(poolDir, file.Filename))
+				}
+
+				return nil
+			})
+		}
+	} else {
+		rootPath = filepath.Join(prefix, "pool")
+
+		// In case of a shared component pool directory, we must check, if a component is no longer referenced by any other
+		// published repository within the same prefix.
+		referencedComponents := map[string]struct{}{}
+		for _, p := range collection.list {
+			if p.Prefix == prefix && p.Storage == storage && !p.MultiDist {
+				for _, component := range p.Components() {
+					referencedComponents[component] = struct{}{}
+				}
 			}
 		}
-	}
 
-	components = make([]string, 0, len(updatedComponents)+len(removedComponents))
-	components = append(append(components, addedComponents...), updatedComponents...)
+		// Remove orphaned component pool directories in the prefix.
+		for _, component := range removedComponents {
+			_, exists := referencedComponents[component]
+			if !exists {
+				err := publishedStorage.RemoveDirs(filepath.Join(rootPath, component), progress)
+				if err != nil {
+					return err
+				}
+			}
+		}
 
-	referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress)
-	if err != nil {
-		return err
+		// Get all referenced files by component for determining orphaned pool files.
+		referencedFiles, err = collection.listReferencedFilesByComponent(prefix, publishedComponents, collectionFactory, progress)
+		if err != nil {
+			return err
+		}
 	}
 
-	for _, component := range components {
+	for _, component := range updatedComponents {
 		if progress != nil {
-			progress.Printf("Cleaning up component %q in prefix %q...\n", component, prefix)
+			progress.Printf("Cleaning up component %q...\n", component)
 		}
 		sort.Strings(referencedFiles[component])
 
-		rootPath := filepath.Join(prefix, "pool", component)
-		existingFiles, err := publishedStorage.Filelist(rootPath)
+		path := filepath.Join(rootPath, component)
+		existingFiles, err := publishedStorage.Filelist(path)
 		if err != nil {
 			return err
 		}
 
 		sort.Strings(existingFiles)
 
-		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
+		orphanedFiles := utils.StrSlicesSubstract(existingFiles, referencedFiles[component])
 
-		for _, file := range filesToDelete {
-			err = publishedStorage.Remove(filepath.Join(rootPath, file))
+		for _, file := range orphanedFiles {
+			err = publishedStorage.Remove(filepath.Join(path, file))
 			if err != nil {
 				return err
 			}
 		}
 	}
 
-	return nil
+	return err
 }
 
 // Remove removes published repository, cleaning up directories, files
@@ -1579,8 +1633,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	if !skipCleanup && len(cleanComponents) > 0 {
-		err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, []string{}, cleanComponents, []string{},
-			collectionFactory, progress)
+		err = collection.CleanupPrefixComponentFiles(publishedStorageProvider, repo, cleanComponents, collectionFactory, progress)
 		if err != nil {
 			if !force {
 				return fmt.Errorf("cleanup failed, use -force-drop to override: %s", err)

deb/publish_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -489,6 +490,30 @@ func (s *PublishedRepoSuite) TestEncodeDecode(c *C) {
 	c.Assert(repo2, DeepEquals, s.repo2)
 }
 
+func (s *PublishedRepoSuite) TestPublishedRepoRevision(c *C) {
+	revision := s.repo2.ObtainRevision()
+	c.Assert(revision, NotNil)
+
+	sources := revision.Sources
+	c.Assert(sources, NotNil)
+	c.Assert(sources, DeepEquals, map[string]string{"main": "local1"})
+
+	sources["test1"] = "snap1"
+	sources["test2"] = "snap2"
+
+	c.Assert(revision.Components(), DeepEquals, []string{"main", "test1", "test2"})
+	c.Assert(revision.SourceNames(), DeepEquals, []string{"local1", "snap1", "snap2"})
+
+	bytes, err := json.Marshal(revision)
+	c.Assert(err, IsNil)
+
+	json_expected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}`
+	c.Assert(string(bytes), Equals, json_expected)
+
+	c.Assert(s.repo2.DropRevision(), DeepEquals, revision)
+	c.Assert(s.repo2.Revision, IsNil)
+}
+
 type PublishedRepoCollectionSuite struct {
 	PackageListMixinSuite
 	db                                database.Storage

system/t04_mirror/UpdateMirror10Test_gold

@@ -18,7 +18,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_i386.deb
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_amd64.deb
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_i386.deb
-Mirror `flat-src` has been successfully updated.
+Mirror `flat-src` has been updated successfully.
 Packages filtered: 110 -> 11.
 gpgv:                using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41
 gpgv: Good signature from "Johannes Ranke <[email protected]>"

system/t04_mirror/UpdateMirror11FTPTest_gold

@@ -13,7 +13,7 @@ Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/deb
 Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/pool/main/s/sensible-utils/sensible-utils_0.0.9+deb9u1_all.deb
 Downloading: http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/pool/main/s/sysvinit/sysvinit-utils_2.88dsf-59.9_i386.deb
 Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/dists/stretch/InRelease
-Mirror `stretch-main` has been successfully updated.
+Mirror `stretch-main` has been updated successfully.
 Packages filtered: 50604 -> 3.
 Retrying 0 http://repo.aptly.info/system-tests/snapshot.debian.org/archive/debian/20220201T025006Z/dists/stretch/InRelease...
 gpgv:                issuer "[email protected]"

system/t04_mirror/UpdateMirror12Test_gold

@@ -37,7 +37,7 @@ Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archi
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/g/gnupg2/scdaemon_2.1.18-8~deb9u4_amd64.deb
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/g/gnupg2/scdaemon_2.1.18-8~deb9u4_i386.deb
 Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
-Mirror `stretch` has been successfully updated.
+Mirror `stretch` has been updated successfully.
 Packages filtered: 78248 -> 20.
 Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
 gpgv:                issuer "[email protected]"

system/t04_mirror/UpdateMirror13Test_gold

@@ -58,4 +58,4 @@ Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/va
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb
-Mirror `varnish` has been successfully updated.
+Mirror `varnish` has been updated successfully.

system/t04_mirror/UpdateMirror14Test_gold

@@ -6,4 +6,4 @@ Downloading & parsing package files...
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/Release
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-amd64/Packages.bz2
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/dists/wheezy/main/binary-i386/Packages.bz2
-Mirror `varnish` has been successfully updated.
+Mirror `varnish` has been updated successfully.

system/t04_mirror/UpdateMirror15Test_gold

@@ -6,4 +6,4 @@ Downloading & parsing package files...
 Downloading https://dl.bintray.com/smira/deb/Packages.bz2...
 Downloading https://dl.bintray.com/smira/deb/Release...
 Downloading https://dl.bintray.com/smira/deb/libboost-program-options-dev_1.49.0.1_i386.deb...
-Mirror `bintray` has been successfully updated.
+Mirror `bintray` has been updated successfully.

system/t04_mirror/UpdateMirror16Test_gold

@@ -6,4 +6,4 @@ Downloading & parsing package files...
 Downloading https://dl.bintray.com/smira/deb/Packages.bz2...
 Downloading https://dl.bintray.com/smira/deb/Release...
 Downloading https://dl.bintray.com/smira/deb/libboost-program-options-dev_1.49.0.1_i386.deb...
-Mirror `bintray` has been successfully updated.
+Mirror `bintray` has been updated successfully.

system/t04_mirror/UpdateMirror17Test_gold

@@ -6,5 +6,5 @@ Download queue: 0 items (0 B)
 Downloading & parsing package files...
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
-Mirror `stretch` has been successfully updated.
+Mirror `stretch` has been updated successfully.
 Packages filtered: 50604 -> 1.

system/t04_mirror/UpdateMirror18Test_gold

@@ -7,5 +7,5 @@ Downloading & parsing package files...
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/main/binary-i386/Packages.gz
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/pool/main/b/boost-defaults/libboost-program-options-dev_1.62.0.1_i386.deb
-Mirror `stretch` has been successfully updated.
+Mirror `stretch` has been updated successfully.
 Packages filtered: 50604 -> 1.

system/t04_mirror/UpdateMirror19Test_gold

@@ -12,4 +12,4 @@ Downloading: http://repo.aptly.info/system-tests/packages.pagerduty.com/pdagent/
 Building download queue...
 Download queue: 23 items (3.46 MiB)
 
-Mirror `pagerduty` has been successfully updated.
+Mirror `pagerduty` has been updated successfully.

system/t04_mirror/UpdateMirror1Test_gold

@@ -58,4 +58,4 @@ Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/va
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish-dbg_3.0.3-1~wheezy_i386.deb
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_amd64.deb
 Downloading: http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/pool/wheezy/main/v/varnish/varnish_3.0.3-1~wheezy_i386.deb
-Mirror `varnish` has been successfully updated.
+Mirror `varnish` has been updated successfully.

system/t04_mirror/UpdateMirror20Test_gold

@@ -8,7 +8,7 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/Packages.bz2
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/r-cran-class_7.3-22-2~bullseyecran.0_amd64.deb
 Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/r-cran-class_7.3-22-2~bullseyecran.0_i386.deb
-Mirror `flat` has been successfully updated.
+Mirror `flat` has been updated successfully.
 Packages filtered: 89 -> 2.
 openpgp: Good signature from "Johannes Ranke <[email protected]>"
 openpgp:  RSA key ID B8F25A8A73EACF41

system/t04_mirror/UpdateMirror21Test_gold

@@ -11,4 +11,4 @@ Downloading: http://repo.aptly.info/system-tests/packages.pagerduty.com/pdagent/
 Building download queue...
 Download queue: 23 items (3.46 MiB)
 
-Mirror `pagerduty` has been successfully updated.
+Mirror `pagerduty` has been updated successfully.

system/t04_mirror/UpdateMirror22Test_gold

@@ -8,4 +8,4 @@ Applying filter...
 Building download queue...
 Download queue: 0 items (0 B)
 
-Mirror `libnvidia-container` has been successfully updated.
+Mirror `libnvidia-container` has been updated successfully.

system/t04_mirror/UpdateMirror23Test_gold

@@ -23,7 +23,7 @@ Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archi
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS
 Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
 Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS
-Mirror `stretch` has been successfully updated.
+Mirror `stretch` has been updated successfully.
 Packages filtered: 49256 -> 1.
 Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
 Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/non-free/installer-s390x/current/images/SHA256SUMS...