Skip to content

Commit

Permalink
Filter mirrors for dependencies of other mirrors/repos
Browse files Browse the repository at this point in the history
Distributions contain a hugh set of packages, mirroring them
requires a large amount of storage and bandwidth. To mitigate this
issues aptly provides an filter feature to reduce the set of
mirrored packages. Especially it supports to limit the downloaded
packages to a set of packages and their needed dependencies as well.

As this filter is self-contained within each mirror, it has
two limitations:

1. its difficult to partial mirror a security or updates mirror to
   match a filtered main repository. Update and security repositories
   contain (newer) packages for a (small) subset of the whole
   distribution to fix bugs/security issues. As the repository
   contains not all packages, filtering breaks due to cuts within
   the dependency tree.

2. integration of packages from other sources
   It is not possible to filter for all dependencies of a package
   if this package itself is not part of the mirror repository
   but instead published within another (local) repository.

In both cases all dependencies between repositories/mirrors have to
be included manually in the mirror filter.

This commit introduces `-deps-from-mirrors` and `-deps-from-repos`
options for mirrors. They are comma-separated lists referencing
mirror/repository names.

If `-filter-with-deps` is activated all dependencies of packages
from the references mirrors/repositories that are part of the
current mirror are included in addition to the filter result
itself. This resolves 2.

If in addition `-dep-follow-all-variants` is activated, all
packages that are included in the refereneced mirrors/repositories
and present in the current mirror (matched by package name only)
are selected to download, too. This resolves 1.
  • Loading branch information
mswart committed Aug 18, 2017
1 parent d9607cf commit c824199
Show file tree
Hide file tree
Showing 18 changed files with 198 additions and 34 deletions.
2 changes: 1 addition & 1 deletion api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
list.PrepareIndex()

list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
nil, nil, context.DependencyOptions(), architecturesList)
if err != nil {
c.Fail(500, fmt.Errorf("unable to search: %s", err))
return
Expand Down
4 changes: 2 additions & 2 deletions bash_completion.d/aptly
Original file line number Diff line number Diff line change
Expand Up @@ -156,15 +156,15 @@ _aptly()
"create")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
COMPREPLY=($(compgen -W "-deps-from-mirrors= -deps-from-repos= -filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
return 0
fi
fi
;;
"edit")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
COMPREPLY=($(compgen -W "-deps-from-mirrors= -deps-from-repos= -filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi
Expand Down
11 changes: 11 additions & 0 deletions cmd/mirror_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,15 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)

extraDepsFromMirrors := context.Flags().Lookup("deps-from-mirrors").Value.String()
if extraDepsFromMirrors != "" {
repo.DepsFromMirrors = strings.Split(extraDepsFromMirrors, ",")
}
extraDepsFromRepos := context.Flags().Lookup("deps-from-repos").Value.String()
if extraDepsFromRepos != "" {
repo.DepsFromRepos = strings.Split(extraDepsFromRepos, ",")
}

if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
Expand Down Expand Up @@ -101,6 +110,8 @@ Example:
cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")

return cmd
}
17 changes: 17 additions & 0 deletions cmd/mirror_edit.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package cmd

import (
"fmt"
"strings"

"github.com/smira/aptly/query"
"github.com/smira/commander"
Expand Down Expand Up @@ -35,6 +36,20 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
repo.DownloadSources = flag.Value.Get().(bool)
case "with-udebs":
repo.DownloadUdebs = flag.Value.Get().(bool)
case "deps-from-repos":
repoNames := flag.Value.String()
if repoNames != "" {
repo.DepsFromRepos = strings.Split(repoNames, ",")
} else {
repo.DepsFromRepos = make([]string, 0)
}
case "deps-from-mirrors":
mirrorNames := flag.Value.String()
if mirrorNames != "" {
repo.DepsFromMirrors = strings.Split(mirrorNames, ",")
} else {
repo.DepsFromMirrors = make([]string, 0)
}
}
})

Expand Down Expand Up @@ -87,6 +102,8 @@ Example:
cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
cmd.Flag.String("deps-from-mirrors", "", "list of mirrors thats packages's dependencies should be fulfilled (comma-separated), default to none")
cmd.Flag.String("deps-from-repos", "", "list of repositories thats packages's dependencies should be fulfilled (comma-separated), default to none")

return cmd
}
8 changes: 8 additions & 0 deletions cmd/mirror_show.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
filterWithDeps = Yes
}
fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
if repo.FilterWithDeps {
if len(repo.DepsFromMirrors) > 0 {
fmt.Printf("Include Dependencies Of Mirrors: %s\n", strings.Join(repo.DepsFromMirrors, ", "))
}
if len(repo.DepsFromRepos) > 0 {
fmt.Printf("Include Dependencies Of Repositories: %s\n", strings.Join(repo.DepsFromRepos, ", "))
}
}
}
if repo.LastDownloadDate.IsZero() {
fmt.Printf("Last update: never\n")
Expand Down
38 changes: 37 additions & 1 deletion cmd/mirror_update.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,50 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
if repo.Filter != "" {
context.Progress().Printf("Applying filter...\n")
var filterQuery deb.PackageQuery
var packagesWithExtraDeps *deb.PackageList

filterQuery, err = query.Parse(repo.Filter)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}

if len(repo.DepsFromMirrors)+len(repo.DepsFromRepos) > 0 {
packagesWithExtraDeps = deb.NewPackageList()
for _, mirrorName := range repo.DepsFromMirrors {
extraMirror, lerr := context.CollectionFactory().RemoteRepoCollection().ByName(mirrorName)
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
lerr = context.CollectionFactory().RemoteRepoCollection().LoadComplete(extraMirror)
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
packageList, lerr := deb.NewPackageListFromRefList(extraMirror.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
packagesWithExtraDeps.Append(packageList)
}
for _, repoName := range repo.DepsFromRepos {
extraRepo, lerr := context.CollectionFactory().LocalRepoCollection().ByName(repoName)
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
lerr = context.CollectionFactory().LocalRepoCollection().LoadComplete(extraRepo)
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
packageList, lerr := deb.NewPackageListFromRefList(extraRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
if lerr != nil {
return fmt.Errorf("unable to update: %s", lerr)
}
packagesWithExtraDeps.Append(packageList)
}
context.Progress().Printf("Extra packages with deps: %d\n", packagesWithExtraDeps.Len())
}

var oldLen, newLen int
oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, packagesWithExtraDeps, context.Progress())
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/repo_move.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
}
}

toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, nil, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return fmt.Errorf("unable to %s: %s", command, err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/repo_remove.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
}

list.PrepareIndex()
toRemove, err := list.Filter(queries, false, nil, 0, nil)
toRemove, err := list.Filter(queries, false, nil, nil, 0, nil)
if err != nil {
return fmt.Errorf("unable to remove: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/snapshot_filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
}

// Filter with dependencies as requested
result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
result, err := packageList.FilterWithProgress(queries, withDeps, nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return fmt.Errorf("unable to filter: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/snapshot_pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
}

// Filter with dependencies as requested
result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, nil, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return fmt.Errorf("unable to pull: %s", err)
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/snapshot_search.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
}

result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList, context.Progress())
nil, nil, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return fmt.Errorf("unable to search: %s", err)
}
Expand Down
56 changes: 51 additions & 5 deletions deb/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,12 +472,12 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
}

// Filter filters package index by specified queries (ORed together), possibly pulling dependencies
func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
return l.FilterWithProgress(queries, withDependencies, source, extraDeps, dependencyOptions, architecturesList, nil)
}

// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source, extraDeps *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
if !l.indexed {
panic("list not indexed, can't filter")
}
Expand All @@ -493,18 +493,43 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
result.PrepareIndex()

dependencySource := NewPackageList()
validationSet := NewPackageList()
if source != nil {
dependencySource.Append(source)
}
if extraDeps != nil {
added += extraDeps.Len()
dependencySource.Append(extraDeps)
validationSet.Append(extraDeps)
if dependencyOptions&DepFollowAllVariants > 0 {
// keep all packages that are also present in extraDeps
// --> they are updates
extraDeps.ForEachIndexed(func(p *Package) error {
var err error
searchResults := l.Search(Dependency{Pkg: p.Name, Relation: VersionDontCare}, true)
if searchResults != nil {
for _, p := range searchResults {
if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
progress.ColoredPrintf("@{g}Include package as update@|: %s", p)
}
result.Add(p)
added++
}
}
return err
})
}
}
dependencySource.Append(result)
validationSet.Append(result)
dependencySource.PrepareIndex()
validationSet.PrepareIndex()

// while some new dependencies were discovered
for added > 0 {
added = 0

// find missing dependencies
missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
missing, err := validationSet.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
if err != nil {
return nil, err
}
Expand All @@ -525,6 +550,7 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
}
result.Add(p)
dependencySource.Add(p)
validationSet.Add(p)
added++
if dependencyOptions&DepFollowAllVariants == 0 {
break
Expand All @@ -534,7 +560,27 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
}
}

if extraDeps == nil {
continue
}
// the missing dependency might come from the extraDeps package list
// if we can fulfil the dependency from extraDeps, include these packages
// within the validationSet -> we want to satisfy their dependencies as well
extraSearchResults := extraDeps.Search(dep, true)
if extraSearchResults != nil {
for _, p := range extraSearchResults {
if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
progress.ColoredPrintf("@{g}Injecting extra package@|: %s", p)
}
dependencySource.Add(p)
validationSet.Add(p)
added++
if dependencyOptions&DepFollowAllVariants == 0 {
break
}
}
}
}
}
Expand Down
Loading

0 comments on commit c824199

Please sign in to comment.