Merge pull request #7 from vpoluyaktov/feature_Search_Paging

Search result paging implemented

.gitignore

@@ -5,4 +5,4 @@ dist/
 output/
 __debug_bin*
 abb_ia.log
-config.yaml
+abb_ia.config.yaml

internal/config/config.go

@@ -7,6 +7,7 @@ import (
 
 	"abb_ia/internal/logger"
 	"abb_ia/internal/utils"
+
 	"gopkg.in/yaml.v3"
 )
 
@@ -17,7 +18,7 @@ var (
 
 // global vars
 var (
-	configFile            = "config.yaml"
+	configFile            = "abb_ia.config.yaml"
 	appVersion, buildDate string
 	repoOwner             string = "vpoluyaktov"
 	repoName              string = "abb_ia"
@@ -26,7 +27,7 @@ var (
 // Fields of this stuct should to be private but I have to make them public because yaml.Marshal/Unmarshal can't work with private fields
 type Config struct {
 	SearchCondition        string        `yaml:"SearchCondition"`
-	SearchRowsMax          int           `yaml:"SearchRowsMax"`
+	RowsPerPage            int           `yaml:"RowsPerPage"`
 	LogFileName            string        `yaml:"LogFileName"`
 	OutputDir              string        `yaml:"Outputdir"`
 	CopyToOutputDir        bool          `yaml:"CopyToOutputDir"`
@@ -73,7 +74,7 @@ func Load() {
 	config.CopyToOutputDir = true
 	config.OutputDir = "/mnt/NAS/Audiobooks/Internet Archive"
 	config.LogLevel = "INFO"
-	config.SearchRowsMax = 25
+	config.RowsPerPage = 25
 	config.UseMock = false
 	config.SaveMock = false
 	config.SearchCondition = ""
@@ -181,12 +182,12 @@ func (c *Config) GetLogLevel() string {
 	return c.LogLevel
 }
 
-func (c *Config) SetSearchRowsMax(r int) {
-	c.SearchRowsMax = r
+func (c *Config) SetRowsPerPage(r int) {
+	c.RowsPerPage = r
 }
 
-func (c *Config) GetSearchRowsMax() int {
-	return c.SearchRowsMax
+func (c *Config) GetRowsPerPage() int {
+	return c.RowsPerPage
 }
 
 func (c *Config) SetUseMock(b bool) {

internal/controller/build_controller.go

@@ -230,7 +230,7 @@ func (c *BuildController) buildAudiobookPart(ab *dto.Audiobook, partId int) {
 		return
 	}
 
-	// clean up 
+	// clean up
 	os.Remove(part.AACFile)
 
 	// add tags and cover image
@@ -312,7 +312,7 @@ func (c *BuildController) updateFileProgress(fileId int, l net.Listener) {
 			c.files[fileId].encodingSpeed = encodingSpeed
 			c.files[fileId].progress = percent
 			c.files[fileId].complete = complete
-			c.mq.SendMessage(mq.BuildController, mq.BuildPage, &dto.BuildFileProgress{FileId: fileId, FileName: c.files[fileId].fileName, Percent: percent}, true)
+			c.mq.SendMessage(mq.BuildController, mq.BuildPage, &dto.FileBuildProgress{FileId: fileId, FileName: c.files[fileId].fileName, Percent: percent}, true)
 		}
 	}
 }
@@ -361,7 +361,7 @@ func (c *BuildController) updateTotalProgress() {
 			speedH := fmt.Sprintf("%.0fx", speed)
 			etaH := utils.SecondsToTime(eta)
 
-			c.mq.SendMessage(mq.BuildController, mq.BuildPage, &dto.BuildProgress{Elapsed: elapsedH, Percent: percent, Files: filesH, Speed: speedH, ETA: etaH}, true)
+			c.mq.SendMessage(mq.BuildController, mq.BuildPage, &dto.TotalBuildProgress{Elapsed: elapsedH, Percent: percent, Files: filesH, Speed: speedH, ETA: etaH}, true)
 		}
 		time.Sleep(mq.PullFrequency)
 	}

internal/controller/download_controller.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"abb_ia/internal/dto"
-	"abb_ia/internal/ia"
+	ia_client "abb_ia/internal/ia"
 	"abb_ia/internal/logger"
 	"abb_ia/internal/mq"
 	"abb_ia/internal/utils"
@@ -79,7 +79,7 @@ func (c *DownloadController) startDownload(cmd *dto.DownloadCommand) {
 	c.mq.SendMessage(mq.DownloadController, mq.DownloadPage, &dto.DisplayBookInfoCommand{Audiobook: c.ab}, true)
 
 	// download files
-	ia := ia_client.New(c.ab.Config.GetSearchRowsMax(), c.ab.Config.IsUseMock(), c.ab.Config.IsSaveMock())
+	ia := ia_client.New(c.ab.Config.GetRowsPerPage(), c.ab.Config.IsUseMock(), c.ab.Config.IsSaveMock())
 	c.stopFlag = false
 	c.files = make([]fileDownload, len(item.AudioFiles))
 	jd := utils.NewJobDispatcher(c.ab.Config.GetConcurrentDownloaders())
@@ -111,7 +111,7 @@ func (c *DownloadController) updateFileProgress(fileId int, fileName string, siz
 		}
 
 		// sent a message only if progress changed
-		c.mq.SendMessage(mq.DownloadController, mq.DownloadPage, &dto.DownloadFileProgress{FileId: fileId, FileName: fileName, Percent: percent}, false)
+		c.mq.SendMessage(mq.DownloadController, mq.DownloadPage, &dto.FileDownloadProgress{FileId: fileId, FileName: fileName, Percent: percent}, false)
 	}
 	c.files[fileId].fileId = fileId
 	c.files[fileId].fileSize = size
@@ -169,7 +169,7 @@ func (c *DownloadController) updateTotalProgress() {
 			speedH := utils.SpeedToHuman(speed)
 			etaH := utils.SecondsToTime(eta)
 
-			c.mq.SendMessage(mq.DownloadController, mq.DownloadPage, &dto.DownloadProgress{Elapsed: elapsedH, Percent: percent, Files: filesH, Bytes: bytesH, Speed: speedH, ETA: etaH}, false)
+			c.mq.SendMessage(mq.DownloadController, mq.DownloadPage, &dto.TotalDownloadProgress{Elapsed: elapsedH, Percent: percent, Files: filesH, Bytes: bytesH, Speed: speedH, ETA: etaH}, false)
 		}
 		time.Sleep(mq.PullFrequency)
 	}

internal/controller/search_controller.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"fmt"
 	"net/url"
 	"sort"
 	"strconv"
@@ -16,8 +17,17 @@ import (
 	"github.com/vpoluyaktov/tview"
 )
 
+var (
+	// mp3 format list ranged by priority
+	Mp3Formats = []string{"16Kbps MP3", "24Kbps MP3", "32Kbps MP3", "40Kbps MP3", "48Kbps MP3", "56Kbps MP3", "64Kbps MP3", "80Kbps MP3", "96Kbps MP3", "112Kbps MP3", "128Kbps MP3", "144Kbps MP3", "160Kbps MP3", "224Kbps MP3", "256Kbps MP3", "320Kbps MP3", "VBR MP3"}
+	// audiobook cover formats
+	CoverFormats = []string{"JPEG", "JPEG Thumb"}
+)
+
 type SearchController struct {
-	mq *mq.Dispatcher
+	mq                *mq.Dispatcher
+	ia                *ia_client.IAClient
+	totalItemsFetched int
 }
 
 func NewSearchController(dispatcher *mq.Dispatcher) *SearchController {
@@ -37,22 +47,57 @@ func (c *SearchController) checkMQ() {
 func (c *SearchController) dispatchMessage(m *mq.Message) {
 	switch dto := m.Dto.(type) {
 	case *dto.SearchCommand:
-		go c.performSearch(dto)
+		go c.search(dto)
+	case *dto.GetNextPageCommand:
+		go c.getGetNextPage(dto)
 	default:
 		m.UnsupportedTypeError(mq.SearchController)
 	}
 }
 
-func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
+func (c *SearchController) search(cmd *dto.SearchCommand) {
+	logger.Info(mq.SearchController + " received " + cmd.String())
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: "Fetching Internet Archive items..."}, false)
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: true}, false)
+	c.totalItemsFetched = 0
+	c.ia = ia_client.New(config.Instance().GetRowsPerPage(), config.Instance().IsUseMock(), config.Instance().IsSaveMock())
+	resp := c.ia.Search(cmd.SearchCondition, "audio")
+	if resp == nil {
+		logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.SearchCondition)
+	}
+	itemsFetched, err := c.fetchDetails(resp)
+	if err != nil {
+		logger.Error(mq.SearchController + ": Failed to fetch item details: " + err.Error())
+	}
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: false}, false)
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: ""}, false)
+	c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{SearchCondition: cmd.SearchCondition}, false)
+	if itemsFetched == 0 {
+		c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.NothingFoundError{SearchCondition: cmd.SearchCondition}, false)
+	}
+}
+
+func (c *SearchController) getGetNextPage(cmd *dto.GetNextPageCommand) {
 	logger.Info(mq.SearchController + " received " + cmd.String())
 	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: "Fetching Internet Archive items..."}, false)
 	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: true}, false)
-	ia := ia_client.New(config.Instance().GetSearchRowsMax(), config.Instance().IsUseMock(), config.Instance().IsSaveMock())
-	resp := ia.Search(cmd.SearchCondition, "audio")
+	resp := c.ia.GetNextPage(cmd.SearchCondition, "audio")
 	if resp == nil {
 		logger.Error(mq.SearchController + ": Failed to perform IA search with condition: " + cmd.SearchCondition)
 	}
+	itemsFetched, err := c.fetchDetails(resp)
+	if err != nil {
+		logger.Error(mq.SearchController + ": Failed to fetch item details: " + err.Error())
+	}
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: false}, false)
+	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: ""}, false)
+	c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.SearchComplete{SearchCondition: cmd.SearchCondition}, false)
+	if itemsFetched == 0 {
+		c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.LastPageMessage{SearchCondition: cmd.SearchCondition}, false)
+	}
+}
 
+func (c *SearchController) fetchDetails(resp *ia_client.SearchResponse) (int, error) {
 	itemsTotal := resp.Response.NumFound
 	itemsFetched := 0
 
@@ -67,7 +112,7 @@ func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
 		item.AudioFiles = make([]dto.AudioFile, 0)
 		var totalSize int64 = 0
 		var totalLength float64 = 0.0
-		d := ia.GetItemDetails(doc.Identifier)
+		d := c.ia.GetItemDetails(doc.Identifier)
 		if d != nil {
 			item.Server = d.Server
 			item.Dir = d.Dir
@@ -82,18 +127,19 @@ func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
 			}
 
 			if len(d.Metadata.Description) > 0 {
-				item.Description = tview.Escape(ia.Html2Text(d.Metadata.Description[0]))
+				item.Description = tview.Escape(c.ia.Html2Text(d.Metadata.Description[0]))
 			}
 
 			for name, metadata := range d.Files {
 				format := metadata.Format
 				// collect mp3 files
 
-				if utils.Contains(dto.Mp3Formats, format) {
+				if utils.Contains(Mp3Formats, format) {
 					size, sErr := strconv.ParseInt(metadata.Size, 10, 64)
 					length, lErr := utils.TimeToSeconds(metadata.Length)
 					if sErr != nil || lErr != nil {
 						logger.Error("Can't parse the file metadata: " + name)
+						return 0, fmt.Errorf("can't parse file metadata: " + name)
 					} else {
 						file := dto.AudioFile{}
 						file.Name = strings.TrimPrefix(name, "/")
@@ -110,8 +156,8 @@ func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
 						addNewFile := true
 						for i, oldFile := range item.AudioFiles {
 							if file.Title == oldFile.Title {
-								oldFilePriority := utils.GetIndex(dto.Mp3Formats, oldFile.Format)
-								newFilePriority := utils.GetIndex(dto.Mp3Formats, file.Format)
+								oldFilePriority := utils.GetIndex(Mp3Formats, oldFile.Format)
+								newFilePriority := utils.GetIndex(Mp3Formats, file.Format)
 								if newFilePriority > oldFilePriority {
 									// remove old file from the list
 									item.AudioFiles = append(item.AudioFiles[:i], item.AudioFiles[i+1:]...)
@@ -137,7 +183,7 @@ func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
 				}
 
 				// collect image files
-				if utils.Contains(dto.CoverFormats, format) {
+				if utils.Contains(CoverFormats, format) {
 					size, err := strconv.ParseInt(metadata.Size, 10, 64)
 					if err == nil {
 						file := dto.ImageFile{}
@@ -172,17 +218,13 @@ func (c *SearchController) performSearch(cmd *dto.SearchCommand) {
 
 			if len(item.AudioFiles) > 0 {
 				itemsFetched++
-				sp := &dto.SearchProgress{ItemsTotal: itemsTotal, ItemsFetched: itemsFetched}
+				c.totalItemsFetched++
+				sp := &dto.SearchProgress{ItemsTotal: itemsTotal, ItemsFetched: c.totalItemsFetched}
 				c.mq.SendMessage(mq.SearchController, mq.SearchPage, sp, false)
 				c.mq.SendMessage(mq.SearchController, mq.SearchPage, item, false)
 			}
 		}
-		logger.Debug(mq.SearchController + " fetched first " + strconv.Itoa(itemsFetched) + " items from " + strconv.Itoa(itemsTotal) + " total")
-	}
-	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.SetBusyIndicator{Busy: false}, false)
-	c.mq.SendMessage(mq.SearchController, mq.Footer, &dto.UpdateStatus{Message: ""}, false)
-
-	if itemsFetched == 0 {
-		c.mq.SendMessage(mq.SearchController, mq.SearchPage, &dto.NothingFoundError{SearchCondition: cmd.SearchCondition}, false)
+		logger.Debug(mq.SearchController + " fetched first " + strconv.Itoa(c.totalItemsFetched) + " items from " + strconv.Itoa(itemsTotal) + " total")
 	}
+	return itemsFetched, nil
 }

internal/dto/build.go

@@ -10,29 +10,28 @@ func (c *BuildCommand) String() string {
 	return fmt.Sprintf("BuildCommand: %s", c.Audiobook.String())
 }
 
-type BuildFileProgress struct {
+type FileBuildProgress struct {
 	FileId   int
 	FileName string
 	Percent  int
 }
 
-func (c *BuildFileProgress) String() string {
-	return fmt.Sprintf("BuildFileProgress: %d, %s, %d", c.FileId, c.FileName, c.Percent)
+func (c *FileBuildProgress) String() string {
+	return fmt.Sprintf("FileBuildProgress: %d, %s, %d", c.FileId, c.FileName, c.Percent)
 }
 
-type BuildProgress struct {
+type TotalBuildProgress struct {
 	Elapsed string // time since started
 	Percent int
 	Files   string // files encoded
 	Speed   string // encode speed bytes/s
 	ETA     string // ETA in seconds
 }
 
-func (c *BuildProgress) String() string {
-	return fmt.Sprintf("BuildProgress: %d", c.Percent)
+func (c *TotalBuildProgress) String() string {
+	return fmt.Sprintf("TotalBuildProgress: %d", c.Percent)
 }
 
-
 type BuildComplete struct {
 	Audiobook *Audiobook
 }

internal/dto/download.go

@@ -18,17 +18,17 @@ func (c *DisplayBookInfoCommand) String() string {
 	return fmt.Sprintf("DisplayBookInfoCommand: %s", c.Audiobook.String())
 }
 
-type DownloadFileProgress struct {
+type FileDownloadProgress struct {
 	FileId   int
 	FileName string
 	Percent  int
 }
 
-func (c *DownloadFileProgress) String() string {
-	return fmt.Sprintf("DownloadFileProgress: %d, %s, %d", c.FileId, c.FileName, c.Percent)
+func (c *FileDownloadProgress) String() string {
+	return fmt.Sprintf("FileDownloadProgress: %d, %s, %d", c.FileId, c.FileName, c.Percent)
 }
 
-type DownloadProgress struct {
+type TotalDownloadProgress struct {
 	Elapsed string // time since started
 	Percent int
 	Files   string // files downloaded
@@ -37,8 +37,8 @@ type DownloadProgress struct {
 	ETA     string // ETA in seconds
 }
 
-func (c *DownloadProgress) String() string {
-	return fmt.Sprintf("DownloadProgress: %d", c.Percent)
+func (c *TotalDownloadProgress) String() string {
+	return fmt.Sprintf("TotalDownloadProgress: %d", c.Percent)
 }
 
 type DownloadComplete struct {