add basic appstream support

Gomfile

@@ -22,6 +22,7 @@ gom 'github.com/smira/lzma', :commit => '7f0af6269940baa2c938fabe73e0d7ba4120568
 gom 'github.com/golang/snappy', :commit => '723cc1e459b8eea2dea4583200fd60757d40097a'
 gom 'github.com/syndtr/goleveldb/leveldb', :commit => '917f41c560270110ceb73c5b38be2a9127387071'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
+gom 'github.com/ulikunitz/xz', :commit => 'f852725cf3ed4e7dca9bd458fe881ff02b27eae4'
 gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'

deb/index_files.go

@@ -23,8 +23,11 @@ type indexFiles struct {
 type indexFile struct {
 	parent       *indexFiles
 	discardable  bool
-	compressable bool
-	onlyGzip     bool
+	ignoreFlat   bool
+	compressOnly bool
+	compressGzip bool
+	compressBzip bool
+	compressGxz  bool
 	signable     bool
 	relativePath string
 	tempFilename string
@@ -61,7 +64,7 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 		return fmt.Errorf("unable to write to index file: %s", err)
 	}
 
-	if file.compressable {
+	if file.compressGzip || file.compressBzip || file.compressGxz {
 		err = utils.CompressFile(file.tempFile)
 		if err != nil {
 			file.tempFile.Close()
@@ -71,12 +74,18 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 
 	file.tempFile.Close()
 
-	exts := []string{""}
-	if file.compressable {
-		exts = append(exts, ".gz", ".bz2")
-		if file.onlyGzip {
-			exts = []string{".gz"}
-		}
+	exts := []string{}
+	if !file.ignoreFlat {
+		exts = append(exts, "")
+	}
+	if file.compressGzip {
+		exts = append(exts, ".gz")
+	}
+	if file.compressBzip {
+		exts = append(exts, ".bz2")
+	}
+	if file.compressGxz {
+		exts = append(exts, ".xz")
 	}
 
 	for _, ext := range exts {
@@ -95,6 +104,10 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	}
 
 	for _, ext := range exts {
+		if file.compressOnly && !(ext == ".gz" || ext == ".bz2" || ext == ".xz") {
+			continue
+		}
+
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
 		if err != nil {
@@ -175,7 +188,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		file = &indexFile{
 			parent:       files,
 			discardable:  false,
-			compressable: true,
+			compressGzip: true,
+			compressBzip: true,
 			signable:     false,
 			relativePath: relativePath,
 		}
@@ -208,7 +222,6 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		file = &indexFile{
 			parent:       files,
 			discardable:  udeb,
-			compressable: false,
 			signable:     false,
 			relativePath: relativePath,
 		}
@@ -237,8 +250,37 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		file = &indexFile{
 			parent:       files,
 			discardable:  true,
-			compressable: true,
-			onlyGzip:     true,
+			ignoreFlat:   true,
+			compressOnly: true,
+			compressGzip: true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) AppStreamIndex(component, name string) *indexFile {
+	key := fmt.Sprintf("ai-%s-%s", component, name)
+	file, ok := files.indexes[key]
+	if !ok {
+		relativePath := filepath.Join(component, "dep11", name)
+		ext := filepath.Ext(name)
+
+		xz := false
+		if ext == ".yml" {
+			xz = true
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressOnly: true,
+			compressGzip: true,
+			compressGxz:  xz,
 			signable:     false,
 			relativePath: relativePath,
 		}
@@ -251,11 +293,10 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 
 func (files *indexFiles) ReleaseFile() *indexFile {
 	return &indexFile{
-		parent:       files,
-		discardable:  false,
-		compressable: false,
-		signable:     true,
-		relativePath: "Release",
+		parent:        files,
+		discardable:   false,
+		signable:      true,
+		relativePath:  "Release",
 	}
 }
 

deb/publish.go

@@ -4,12 +4,14 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -452,6 +454,36 @@ func (p *PublishedRepo) GetLabel() string {
 	return p.Label
 }
 
+// GetAppStream returns full path list of appstream files
+func (p *PublishedRepo) GetAppStream(component string) ([]string, error) {
+	if utils.Config.AppStreamDir == "" {
+		return nil, errors.New("AppStream directory unconfigured")
+	}
+
+	appStreamDir := filepath.Join(utils.Config.AppStreamDir, p.Prefix, "dists", p.Distribution, component)
+	files := []string{}
+
+	err := filepath.Walk(appStreamDir, func(path string, info os.FileInfo, err error) error {
+		stat, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		ext := filepath.Ext(path)
+		if stat.Mode().IsRegular() && (ext == ".yml" || ext == ".tar") {
+			files = append(files, path)
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	return files, nil
+}
+
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error {
@@ -629,6 +661,34 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 			}
 		}
 
+		// AppStream file colection
+		if utils.Config.AppStreamDir != "" {
+			appstream, err := p.GetAppStream(component)
+			if err != nil {
+				return fmt.Errorf("Unable to get AppStream files: %s...\n", err)
+			}
+
+			for _, file := range appstream {
+				base := filepath.Base(file)
+
+				fi, err := os.Open(file)
+				if err != nil {
+					return fmt.Errorf("unable to open %s file: %v", base, err)
+				}
+				defer fi.Close()
+
+				w, err := indexes.AppStreamIndex(component, base).BufWriter()
+				if err != nil {
+					return fmt.Errorf("unable to write %s file: %v", base, err)
+				}
+
+				_, err = io.Copy(w, fi)
+				if err != nil {
+					return fmt.Errorf("unable to copy %s file: %v", base, err)
+				}
+			}
+		}
+
 		// For all architectures, generate Release files
 		for _, arch := range p.Architectures {
 			for _, udeb := range udebs {

system/t02_config/ConfigShowTest_gold

@@ -13,6 +13,7 @@
     "ppaDistributorID": "ubuntu",
     "ppaCodename": "",
     "skipContentsPublishing": false,
+    "appStreamDir": "",
     "S3PublishEndpoints": {},
     "SwiftPublishEndpoints": {}
 }

system/t02_config/CreateConfigTest_gold

@@ -13,6 +13,7 @@
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",
   "skipContentsPublishing": false,
+  "appStreamDir": "",
   "S3PublishEndpoints": {},
   "SwiftPublishEndpoints": {}
-}
+}

utils/compress.go

@@ -2,16 +2,18 @@ package utils
 
 import (
 	"compress/gzip"
+	"github.com/ulikunitz/xz"
 	"io"
 	"os"
 	"os/exec"
 )
 
-// CompressFile compresses file specified by source to .gz & .bz2
+// CompressFile compresses file specified by source to .gz, .xz, and .bz2
 //
-// It uses internal gzip and external bzip2, see:
+// It uses internal gzip, external xz, and external bzip2, see:
 // https://code.google.com/p/go/issues/detail?id=4828
 func CompressFile(source *os.File) error {
+	// gz compression
 	gzPath := source.Name() + ".gz"
 	gzFile, err := os.Create(gzPath)
 	if err != nil {
@@ -28,6 +30,24 @@ func CompressFile(source *os.File) error {
 		return err
 	}
 
+	// xz compression
+	xzPath := source.Name() + ".xz"
+	xzFile, err := os.Create(xzPath)
+	if err != nil {
+		return err
+	}
+	defer xzFile.Close()
+
+	xzWriter := xz.NewWriter(xzFile)
+	defer xzWriter.Close()
+
+	source.Seek(0, 0)
+	_, err = io.Copy(xzWriter, source)
+	if err != nil {
+		return err
+	}
+
+	// bzip compression
 	cmd := exec.Command("bzip2", "-k", "-f", source.Name())
 	return cmd.Run()
 }

utils/compress_test.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"compress/bzip2"
 	"compress/gzip"
+	"github.com/ulikunitz/xz"
 	"io/ioutil"
 	"os"
 
@@ -41,7 +42,20 @@ func (s *CompressSuite) TestCompress(c *C) {
 	_, err = gzReader.Read(buf)
 	c.Assert(err, IsNil)
 
+	file.Close()
 	gzReader.Close()
+
+	c.Check(string(buf), Equals, testString)
+
+	file, err = os.Open(s.tempfile.Name() + ".xz")
+	c.Assert(err, IsNil)
+
+	xzReader, err := xz.NewReader(file)
+	c.Assert(err, IsNil)
+
+	_, err = xzReader.Read(buf)
+	c.Assert(err, IsNil)
+
 	file.Close()
 
 	c.Check(string(buf), Equals, testString)

utils/config.go

@@ -22,6 +22,7 @@ type ConfigStructure struct {
 	PpaDistributorID       string                      `json:"ppaDistributorID"`
 	PpaCodename            string                      `json:"ppaCodename"`
 	SkipContentsPublishing bool                        `json:"skipContentsPublishing"`
+	AppStreamDir           string                      `json:"appStreamDir"`
 	S3PublishRoots         map[string]S3PublishRoot    `json:"S3PublishEndpoints"`
 	SwiftPublishRoots      map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"`
 }
@@ -70,6 +71,7 @@ var Config = ConfigStructure{
 	DownloadSourcePackages: false,
 	PpaDistributorID:       "ubuntu",
 	PpaCodename:            "",
+	AppStreamDir:           "",
 	S3PublishRoots:         map[string]S3PublishRoot{},
 	SwiftPublishRoots:      map[string]SwiftPublishRoot{},
 }

utils/config_test.go

@@ -63,6 +63,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"  \"ppaDistributorID\": \"\",\n"+
 		"  \"ppaCodename\": \"\",\n"+
 		"  \"skipContentsPublishing\": false,\n"+
+		"  \"appStreamDir\": \"\",\n"+
 		"  \"S3PublishEndpoints\": {\n"+
 		"    \"test\": {\n"+
 		"      \"region\": \"us-east-1\",\n"+