From c530bf8f68206aa7b00ad5e23bd767ed86ca2911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sat, 6 Jan 2024 18:38:42 +0100 Subject: [PATCH] s3: call s3.ListFiles only on publish path in LinkFromPool instead of caching the whole s3 bucket, cache only the pool path. this requires an additional parameter, and since this is an interface, all implementations need to follow. might help in other backends too. closes #1181 --- aptly/interfaces.go | 2 +- azure/public.go | 3 ++- deb/package.go | 4 +--- files/public.go | 3 ++- s3/public.go | 5 +++-- swift/public.go | 3 ++- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/aptly/interfaces.go b/aptly/interfaces.go index e912daba..9f28ff31 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -70,7 +70,7 @@ type PublishedStorage interface { // Remove removes single file under public path Remove(path string) error // LinkFromPool links package file from pool to dist's pool location - LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error + LinkFromPool(prefix string, path string, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error // Filelist returns list of files under prefix Filelist(prefix string) ([]string, error) // RenameFile renames (moves) file diff --git a/azure/public.go b/azure/public.go index b5c1e357..1d210f77 100644 --- a/azure/public.go +++ b/azure/public.go @@ -179,9 +179,10 @@ func (storage *PublishedStorage) Remove(path string) error { // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { + publishedDirectory := filepath.Join(prefix, path) relPath := filepath.Join(publishedDirectory, fileName) poolPath := filepath.Join(storage.prefix, relPath) diff --git a/deb/package.go b/deb/package.go index ce16adf2..5d372b2b 100644 --- a/deb/package.go +++ b/deb/package.go @@ -621,9 +621,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP return err } - publishedDirectory := filepath.Join(prefix, relPath) - - err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force) + err = publishedStorage.LinkFromPool(prefix, relPath, f.Filename, packagePool, sourcePoolPath, f.Checksums, force) if err != nil { return err } diff --git a/files/public.go b/files/public.go index fc87b410..74ea39bc 100644 --- a/files/public.go +++ b/files/public.go @@ -123,9 +123,10 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress // sourcePath is a relative path to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { + publishedDirectory := filepath.Join(prefix, path) baseName := filepath.Base(fileName) poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName)) diff --git a/s3/public.go b/s3/public.go index 78a4a917..2663e1e9 100644 --- a/s3/public.go +++ b/s3/public.go @@ -310,14 +310,15 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { + publishedDirectory := filepath.Join(prefix, path) relPath := filepath.Join(publishedDirectory, fileName) poolPath := filepath.Join(storage.prefix, relPath) if storage.pathCache == nil { - paths, md5s, err := storage.internalFilelist("", true) + paths, md5s, err := storage.internalFilelist(filepath.Join(storage.prefix, prefix, "pool"), true) if err != nil { return errors.Wrap(err, "error caching paths under prefix") } diff --git a/swift/public.go b/swift/public.go index 4f8c7b8a..6d55e600 100644 --- a/swift/public.go +++ b/swift/public.go @@ -193,9 +193,10 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // sourcePath is filepath to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(prefix string, path string, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { + publishedDirectory := filepath.Join(prefix, path) relPath := filepath.Join(publishedDirectory, fileName) poolPath := filepath.Join(storage.prefix, relPath)