fix pushing algolia indexes (#166)

fix pushing algolia indexes

.github/workflows/algolia.yml

@@ -29,23 +29,19 @@ jobs:
           mkdir -p public
           hugo-algolia --config config/_default/config.yaml
 
-      - name: Build Go binary
+      - name: Build adjust-algolia-output and adjust algolia.json
         run: |
-          cd tools/adjust-algolia-output
-          go build -o /usr/local/bin/adjust-algolia-output 
-
-      - name: Adjust algolia.json using binary
-        run: adjust-algolia-output < ./public/algolia.json | jq > public/algolia_final.json
-
-      - name: Install Node.js
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        uses: actions/setup-node@v2
-        with:
-          node-version: '14'
-
-      - name: Push to Algolia
+          pushd tools/adjust-algolia-output
+          go build -o /usr/local/bin/adjust-algolia-output
+          popd
+          adjust-algolia-output < ./public/algolia.json | jq > public/algolia_final.json
+          echo "Final Algolia JSON:"
+          cat public/algolia_final.json
+
+      - name: Build push-to-algolia and push to Algolia
         if: github.event_name == 'push' && github.ref == 'refs/heads/main'
         run: |
-          cd scripts
-          npm install algoliasearch
-          APPLICATION_ID=${{ secrets.ALGOLIA_APPLICATION_ID }} API_KEY=${{ secrets.ALGOLIA_API_KEY }} node push-to-algolia.js ../public/algolia_final.json
+          pushd tools/push-to-algolia
+          go build -o /usr/local/bin/push-to-algolia
+          popd
+          APPLICATION_ID=${{ secrets.ALGOLIA_APPLICATION_ID }} API_KEY=${{ secrets.ALGOLIA_API_KEY }} push-to-algolia ./public/algolia_final.json

.gitignore

@@ -15,4 +15,5 @@ trace.out
 .idea
 node_modules
 .hugo_build.lock
-adjust-algolia-output
+tools/adjust-algolia-output/adjust-algolia-output
+tools/push-to-algolia/push-to-algolia

tools/adjust-algolia-output/main.go

@@ -15,85 +15,109 @@ import (
 type arrayFlags []string
 
 func (i *arrayFlags) String() string {
-	return "my string representation"
+	return strings.Join(*i, ", ")
 }
 
 func (i *arrayFlags) Set(value string) error {
 	*i = append(*i, value)
 	return nil
 }
 
-// WriteTo writes v to writer.
-func WriteTo(w io.Writer, v interface{}) error {
+// JSON encoding and decoding helpers
+func writeJSON(w io.Writer, v interface{}) error {
 	var h codec.JsonHandle
 	h.BasicHandle.Canonical = true
-	err := codec.NewEncoder(w, &h).Encode(v)
-	if err != nil {
-		return fmt.Errorf("JSON encoder failed: %w", err)
-	}
-	return nil
+	return codec.NewEncoder(w, &h).Encode(v)
 }
 
-// ReadFrom reads and stores the result in v.
-func ReadFrom(w io.Reader, v interface{}) error {
+func readJSON(r io.Reader, v interface{}) error {
 	var h codec.JsonHandle
-	err := codec.NewDecoder(w, &h).Decode(v)
-	if err != nil {
-		return fmt.Errorf("JSON decoder failed: %w", err)
-	}
-	return nil
+	return codec.NewDecoder(r, &h).Decode(v)
 }
 
+// getParentID returns the parent directory of the objectID
 func getParentID(objectID string) string {
 	return filepath.Dir(objectID)
 }
 
+// adjustURIs removes prefixes from the "uri" field in each item
+func adjustURIs(data []map[string]interface{}, prefixes arrayFlags) {
+	for _, item := range data {
+		for _, prefix := range prefixes {
+			item["uri"] = strings.TrimPrefix(item["uri"].(string), prefix)
+		}
+	}
+}
+
+// adjustTitles recursively adjusts the title field for objects that match the given regex
+func adjustTitles(data []map[string]interface{}, regex *regexp.Regexp, adjustTitle string, mapping map[string]map[string]interface{}) {
+	for _, item := range data {
+		if regex == nil {
+			continue
+		}
+		parentID := getParentID(item["objectID"].(string))
+		for {
+			if regex.MatchString(parentID) {
+				break
+			}
+			parent, exists := mapping[parentID]
+			if !exists {
+				break
+			}
+			item["title"] = fmt.Sprintf("%v%s%v", parent["title"], adjustTitle, item["title"])
+			parentID = getParentID(parentID)
+		}
+	}
+}
+
+// filterNonEmptyContent filters items that have non-empty "content"
+func filterNonEmptyContent(data []map[string]interface{}) []map[string]interface{} {
+	filtered := []map[string]interface{}{}
+	for _, item := range data {
+		if len(item["content"].(string)) > 0 {
+			filtered = append(filtered, item)
+		}
+	}
+	return filtered
+}
+
 func main() {
-	rootObjectIDRegex := flag.String("root-object-id", "[a-z0-9A-Z]+/docs$", "Root object ID for the adjust title - if empty, no adjustment is made, if set the string will be used as separator")
-	adjustTitle := flag.String("adjust-title", " / ", "Adjust title - if empty, no adjustment is made, if set the string will be used as separator")
+	// Parse flags
+	rootObjectIDRegex := flag.String("root-object-id", "[a-z0-9A-Z]+/docs$", "Regex to match root object IDs for title adjustment")
+	adjustTitle := flag.String("adjust-title", " / ", "String to use as separator when adjusting titles")
 	trimPrefixesInURI := arrayFlags{"en/"}
-	flag.Var(&trimPrefixesInURI, "trim-prefixes-in-uri", "Remove the given prefixes in the URI")
+	flag.Var(&trimPrefixesInURI, "trim-prefixes-in-uri", "Prefixes to remove from URI")
 	flag.Parse()
 
+	// Read input data
 	var data []map[string]interface{}
-	err := ReadFrom(os.Stdin, &data)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+	if err := readJSON(os.Stdin, &data); err != nil {
+		fmt.Fprintf(os.Stderr, "Error reading JSON: %v\n", err)
 		os.Exit(1)
 	}
-	var regexpRootObjectID *regexp.Regexp
+
+	// Create regex if needed
+	var regex *regexp.Regexp
 	if *rootObjectIDRegex != "" && *adjustTitle != "" {
-		regexpRootObjectID = regexp.MustCompile(*rootObjectIDRegex)
+		regex = regexp.MustCompile(*rootObjectIDRegex)
 	}
+
+	// Create objectID to item map
 	mapping := make(map[string]map[string]interface{})
 	for _, item := range data {
-		objectID := item["objectID"].(string)
-		mapping[objectID] = item
+		mapping[item["objectID"].(string)] = item
 	}
-	for idx, item := range data {
-		for _, prefix := range trimPrefixesInURI {
-			item["uri"] = strings.TrimPrefix(item["uri"].(string), prefix)
-		}
-		if regexpRootObjectID != nil {
-			parentID := getParentID(item["objectID"].(string))
-			for {
-				if regexpRootObjectID.Match([]byte(parentID)) {
-					break
-				}
-				parent, ok := mapping[parentID]
-				if !ok {
-					break
-				}
-				item["title"] = fmt.Sprintf("%v%s%v", parent["title"], *adjustTitle, item["title"])
-				parentID = getParentID(parentID)
-			}
-		}
-		data[idx] = item
-	}
-	err = WriteTo(os.Stdout, data)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+
+	// Adjust URIs and titles
+	adjustURIs(data, trimPrefixesInURI)
+	adjustTitles(data, regex, *adjustTitle, mapping)
+
+	// Filter items with non-empty content
+	filteredData := filterNonEmptyContent(data)
+
+	// Write output
+	if err := writeJSON(os.Stdout, filteredData); err != nil {
+		fmt.Fprintf(os.Stderr, "Error writing JSON: %v\n", err)
 		os.Exit(1)
 	}
-
 }

tools/push-to-algolia/go.mod

@@ -0,0 +1,17 @@
+module github.com/plgd-dev/doc/tools/push-to-algolia
+
+go 1.22.0
+
+require github.com/algolia/algoliasearch-client-go/v4 v4.3.5
+
+require (
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.22.1 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+)

tools/push-to-algolia/go.sum

@@ -0,0 +1,30 @@
+github.com/algolia/algoliasearch-client-go/v4 v4.3.5 h1:8DM9QmRDnqzIH9ae5jFu80zlHPOzgyQAAPmgGCmMdD4=
+github.com/algolia/algoliasearch-client-go/v4 v4.3.5/go.mod h1:WpoGIDI9om1sNFTsisBJdBF46W0bUvAXA75l+4+UrTE=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
+github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA=
+github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/push-to-algolia/main.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/algolia/algoliasearch-client-go/v4/algolia/search"
+)
+
+func main() {
+	// Retrieve environment variables
+	applicationID := os.Getenv("APPLICATION_ID")
+	if applicationID == "" {
+		log.Fatal("APPLICATION_ID environment variable is required")
+	}
+	apiKey := os.Getenv("API_KEY")
+	if apiKey == "" {
+		log.Fatal("API_KEY environment variable is required")
+	}
+
+	// Initialize Algolia client
+	client, err := search.NewClient(applicationID, apiKey)
+	if err != nil {
+		log.Fatalf("Error initializing Algolia client: %v", err)
+	}
+
+	// Check if file path is provided
+	if len(os.Args) < 2 {
+		log.Fatal("Please provide the path to the JSON file")
+	}
+
+	// Read JSON file
+	filePath := os.Args[1]
+	jsonData, err := os.ReadFile(filePath)
+	if err != nil {
+		log.Fatalf("Error reading JSON file: %v", err)
+	}
+
+	// Parse JSON data
+	var records []map[string]interface{}
+	if err := json.Unmarshal(jsonData, &records); err != nil {
+		log.Fatalf("Error parsing JSON data: %v", err)
+	}
+
+	// Clear the existing index
+	if _, err := client.ClearObjects(client.NewApiClearObjectsRequest("doc")); err != nil {
+		log.Fatalf("Error clearing index: %v", err)
+	}
+	fmt.Println("Existing records cleared")
+
+	// Add or update records in Algolia index
+	res, err := client.SaveObjects("doc", records, search.WithCreateIfNotExists(true))
+	if err != nil {
+		log.Fatalf("Error adding/updating records: %v", err)
+	}
+
+	fmt.Println("Records added/updated:", res)
+}