feat: add concurrency for s5cmd download (#140)

bentoml · Nov 29, 2024 · 9eeaf03 · 9eeaf03
1 parent b072c29
commit 9eeaf03
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/bento-image-snapshotter/fs/s3.go b/bento-image-snapshotter/fs/s3.go
@@ -228,7 +228,7 @@ func (o *S3FileSystem) downloadLayerFromS3(ctx context.Context, bucketName, laye
 
 	startTime := time.Now()
 	var stderr bytes.Buffer
-	cmd := exec.CommandContext(ctx, "sh", "-c", fmt.Sprintf("s5cmd cat %s | pzstd -d | tar -xf -", s3Path)) // nolint:gosec
+	cmd := exec.CommandContext(ctx, "sh", "-c", fmt.Sprintf("s5cmd cat --concurrency=12 --part-size=400 %s | pzstd -d | tar -xf -", s3Path)) // nolint:gosec
 	cmd.Stderr = &stderr
 	cmd.Dir = tempName