Filter mirrors for dependencies of other mirrors/repos

Distributions contain a hugh set of packages, mirroring them
requires a large amount of storage and bandwidth. To mitigate this
issues aptly provides an filter feature to reduce the set of
mirrored packages. Especially it supports to limit the downloaded
packages to a set of packages and their needed dependencies as well.

As this filter is self-contained within each mirror, it has
two limitations:

1. its difficult to partial mirror a security or updates mirror to
   match a filtered main repository. Update and security repositories
   contain (newer) packages for a (small) subset of the whole
   distribution to fix bugs/security issues. As the repository
   contains not all packages, filtering breaks due to cuts within
   the dependency tree.

2. integration of packages from other sources
   It is not possible to filter for all dependencies of a package
   if this package itself is not part of the mirror repository
   but instead published within another (local) repository.

In both cases all dependencies between repositories/mirrors have to
be included manually in the mirror filter.

This commit introduces `-deps-from-mirrors` and `-deps-from-repos`
options for mirrors. They are comma-separated lists referencing
mirror/repository names.

If `-filter-with-deps` is activated all dependencies of packages
from the references mirrors/repositories that are part of the
current mirror are included in addition to the filter result
itself. This resolves 2.

If in addition `-dep-follow-all-variants` is activated, all
packages that are included in the refereneced mirrors/repositories
and present in the current mirror (matched by package name only)
are selected to download, too. This resolves 1.

api/api.go

@@ -145,7 +145,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		list.PrepareIndex()
 
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+			nil, nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
 			c.Fail(500, fmt.Errorf("unable to search: %s", err))
 			return

bash_completion.d/aptly

@@ -156,15 +156,15 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-deps-from-mirrors= -deps-from-repos= -filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
           ;;
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-deps-from-mirrors= -deps-from-repos= -filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

cmd/mirror_create.go

@@ -46,6 +46,15 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
+	extraDepsFromMirrors := context.Flags().Lookup("deps-from-mirrors").Value.String()
+	if extraDepsFromMirrors != "" {
+		repo.DepsFromMirrors = strings.Split(extraDepsFromMirrors, ",")
+	}
+	extraDepsFromRepos := context.Flags().Lookup("deps-from-repos").Value.String()
+	if extraDepsFromRepos != "" {
+		repo.DepsFromRepos = strings.Split(extraDepsFromRepos, ",")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -101,6 +110,8 @@ Example:
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
+	cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")
 
 	return cmd
 }

cmd/mirror_edit.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
@@ -35,6 +36,20 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "deps-from-repos":
+			repoNames := flag.Value.String()
+			if repoNames != "" {
+				repo.DepsFromRepos = strings.Split(repoNames, ",")
+			} else {
+				repo.DepsFromRepos = make([]string, 0)
+			}
+		case "deps-from-mirrors":
+			mirrorNames := flag.Value.String()
+			if mirrorNames != "" {
+				repo.DepsFromMirrors = strings.Split(mirrorNames, ",")
+			} else {
+				repo.DepsFromMirrors = make([]string, 0)
+			}
 		}
 	})
 
@@ -87,6 +102,8 @@ Example:
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
+	cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -54,6 +54,14 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 			filterWithDeps = Yes
 		}
 		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+		if repo.FilterWithDeps {
+			if len(repo.DepsFromMirrors) > 0 {
+				fmt.Printf("Include Dependencies Of Mirrors: %s\n", strings.Join(repo.DepsFromMirrors, ", "))
+			}
+			if len(repo.DepsFromRepos) > 0 {
+				fmt.Printf("Include Dependencies Of Repositories: %s\n", strings.Join(repo.DepsFromRepos, ", "))
+			}
+		}
 	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")

cmd/mirror_update.go

@@ -64,14 +64,50 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	if repo.Filter != "" {
 		context.Progress().Printf("Applying filter...\n")
 		var filterQuery deb.PackageQuery
+		var packagesWithExtraDeps *deb.PackageList
 
 		filterQuery, err = query.Parse(repo.Filter)
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
 
+		if len(repo.DepsFromMirrors)+len(repo.DepsFromRepos) > 0 {
+			packagesWithExtraDeps = deb.NewPackageList()
+			for _, mirrorName := range repo.DepsFromMirrors {
+				extraMirror, lerr := context.CollectionFactory().RemoteRepoCollection().ByName(mirrorName)
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				lerr = context.CollectionFactory().RemoteRepoCollection().LoadComplete(extraMirror)
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				packageList, lerr := deb.NewPackageListFromRefList(extraMirror.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				packagesWithExtraDeps.Append(packageList)
+			}
+			for _, repoName := range repo.DepsFromRepos {
+				extraRepo, lerr := context.CollectionFactory().LocalRepoCollection().ByName(repoName)
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				lerr = context.CollectionFactory().LocalRepoCollection().LoadComplete(extraRepo)
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				packageList, lerr := deb.NewPackageListFromRefList(extraRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+				if lerr != nil {
+					return fmt.Errorf("unable to update: %s", lerr)
+				}
+				packagesWithExtraDeps.Append(packageList)
+			}
+			context.Progress().Printf("Extra packages with deps: %d\n", packagesWithExtraDeps.Len())
+		}
+
 		var oldLen, newLen int
-		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, packagesWithExtraDeps, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}

cmd/repo_move.go

@@ -115,7 +115,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}

cmd/repo_remove.go

@@ -44,7 +44,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	}
 
 	list.PrepareIndex()
-	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}

cmd/snapshot_filter.go

@@ -66,7 +66,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}

cmd/snapshot_pull.go

@@ -96,7 +96,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}

cmd/snapshot_search.go

@@ -103,7 +103,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList, context.Progress())
+		nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}

deb/list.go

@@ -472,12 +472,12 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
-func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
-	return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
+func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+	return l.FilterWithProgress(queries, withDependencies, source, extraDeps, dependencyOptions, architecturesList, nil)
 }
 
 // FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
-func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
+func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -493,18 +493,43 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
 		result.PrepareIndex()
 
 		dependencySource := NewPackageList()
+		validationSet := NewPackageList()
 		if source != nil {
 			dependencySource.Append(source)
 		}
+		if extraDeps != nil {
+			added += extraDeps.Len()
+			dependencySource.Append(extraDeps)
+			validationSet.Append(extraDeps)
+			if dependencyOptions&DepFollowAllVariants > 0 {
+				// keep all packages that are also present in extraDeps
+				// --> they are updates
+				extraDeps.ForEachIndexed(func(p *Package) error {
+					var err error
+					searchResults := l.Search(Dependency{Pkg: p.Name, Relation: VersionDontCare}, true)
+					if searchResults != nil {
+						for _, p := range searchResults {
+							if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+								progress.ColoredPrintf("@{g}Include package as update@|: %s", p)
+							}
+							result.Add(p)
+							added++
+						}
+					}
+					return err
+				})
+			}
+		}
 		dependencySource.Append(result)
+		validationSet.Append(result)
 		dependencySource.PrepareIndex()
+		validationSet.PrepareIndex()
 
 		// while some new dependencies were discovered
 		for added > 0 {
 			added = 0
 
-			// find missing dependencies
-			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
+			missing, err := validationSet.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
 			if err != nil {
 				return nil, err
 			}
@@ -525,6 +550,7 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
 						}
 						result.Add(p)
 						dependencySource.Add(p)
+						validationSet.Add(p)
 						added++
 						if dependencyOptions&DepFollowAllVariants == 0 {
 							break
@@ -534,7 +560,27 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
 					if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
 						progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
 					}
+				}
 
+				if extraDeps == nil {
+					continue
+				}
+				// the missing dependency might come from the extraDeps package list
+				// if we can fulfil the dependency from extraDeps, include these packages
+				// within the validationSet -> we want to satisfy their dependencies as well
+				extraSearchResults := extraDeps.Search(dep, true)
+				if extraSearchResults != nil {
+					for _, p := range extraSearchResults {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{g}Injecting extra package@|: %s", p)
+						}
+						dependencySource.Add(p)
+						validationSet.Add(p)
+						added++
+						if dependencyOptions&DepFollowAllVariants == 0 {
+							break
+						}
+					}
 				}
 			}
 		}