make build

.gitignore

@@ -18,4 +18,5 @@
 /test
 /*.internal
 /run.sh
-/.env*
+/.env*
+/build

Makefile

@@ -8,8 +8,11 @@ v:
 clean:
 	rm -rf build/ out/
 
+.PHONY: build
 build:
-	go build -trimpath -ldflags "-X main.version=${VERSION}" -v -o your-project main.go
+	@mkdir ./build || true
+	go build -trimpath -ldflags "-X main.version=${VERSION}" -v -o ./build/collector cmd/collector/main.go
+	go build -trimpath -ldflags "-X main.version=${VERSION}" -v -o ./build/summerizer cmd/summarizer/main.go
 
 test:
 	go test ./...

README.md

@@ -12,7 +12,7 @@ Dump mempool transactions from EL nodes, and archive them in [Parquet](https://g
 
 **Notes:**
 
-- This is work in progress and under heavy development.
+- This is work in progress and under heavy development (mempool collector is relatively stable now though!)
 - Seeing about 90k - 140k unique new mempool transactions per hour, on average ~1.2kb per rawTx (as of 2023-08-07).
 - See also: [discussion about compression](https://github.com/flashbots/mempool-archiver/issues/2) and [storage](https://github.com/flashbots/mempool-archiver/issues/1)
 
@@ -110,8 +110,13 @@ make fmt
 
 Lots, this is WIP
 
-maybe:
+should:
 
+- collector support multiple `-node` cli args (like [mev-boost](https://github.com/flashbots/mev-boost/blob/ci-fix/cli/main.go#L87))
+
+could:
+
+- stats about which node saw how many tx first
 - http server to add/remove nodes, see stats, pprof?
 
 ---

cmd/collector/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"os"
 	"os/signal"
 	"strings"
@@ -21,6 +22,7 @@ var (
 	defaultLogService = os.Getenv("LOG_SERVICE")
 
 	// Flags
+	printVersion  = flag.Bool("version", false, "only print version")
 	debugPtr      = flag.Bool("debug", defaultDebug, "print debug output")
 	logProdPtr    = flag.Bool("log-prod", defaultLogProd, "log in production mode (json)")
 	logServicePtr = flag.String("log-service", defaultLogService, "'service' tag to logs")
@@ -31,6 +33,12 @@ var (
 func main() {
 	flag.Parse()
 
+	// perhaps only print the version
+	if *printVersion {
+		fmt.Printf("mempool-collector %s\n", version)
+		return
+	}
+
 	// Logger setup
 	var logger *zap.Logger
 	zapLevel := zap.NewAtomicLevel()

cmd/summarizer/main.go

@@ -33,10 +33,11 @@ var (
 	defaultLogProd = os.Getenv("LOG_PROD") == "1"
 
 	// Flags
-	debugPtr   = flag.Bool("debug", defaultDebug, "print debug output")
-	logProdPtr = flag.Bool("log-prod", defaultLogProd, "log in production mode (json)")
-	dirPtr     = flag.String("dir", "", "which path to archive")
-	outDirPtr  = flag.String("out", "", "where to save output files")
+	printVersion = flag.Bool("version", false, "only print version")
+	debugPtr     = flag.Bool("debug", defaultDebug, "print debug output")
+	logProdPtr   = flag.Bool("log-prod", defaultLogProd, "log in production mode (json)")
+	dirPtr       = flag.String("dir", "", "which path to archive")
+	outDirPtr    = flag.String("out", "", "where to save output files")
 	// saveCSV    = flag.Bool("csv", false, "save a csv summary")
 	// limit = flag.Int("limit", 0, "max number of txs to process")
 
@@ -51,6 +52,12 @@ var (
 func main() {
 	flag.Parse()
 
+	// perhaps only print the version
+	if *printVersion {
+		fmt.Printf("mempool-summarizer %s\n", version)
+		return
+	}
+
 	// Logger setup
 	var logger *zap.Logger
 	zapLevel := zap.NewAtomicLevel()

scripts/stats.sh

@@ -1,15 +1,25 @@
 #!/bin/bash
+#
+# print stats (lines and disk usage) for all CSV files in a directory/subdirectory
+#
 
-# Iterate over files in current directory, and print filename, lines and size
-for x in $(ls -1); do
+# requires directory as argument
+if [ $# -eq 0 ]
+  then
+    echo "No arguments supplied"
+    exit 1
+fi
+
+# requires directory to exist
+if [ ! -d "$1" ]; then
+  echo "Directory $1 does not exist"
+  exit 1
+fi
+
+# iterate over all CSV files in any subdirectory
+for x in $(ls -1 -- $1/**/*.csv); do
 	size=$( du --si -s $x | cut -f1 )
-	size_app=$( du --si -s --apparent-size $x | cut -f1 )
 	lines=$( cat $x | wc -l )
-	printf "%s \t %'10d \t %s \t %s \n" $x $lines $size $size_app
+	printf "%s \t %'10d \t %8s \n" $x $lines $size
 done
 
-#for x in $(ls -d -1 -- */); do
-#	size=$( du --si -s $x | cut -f1 )
-#	files=$( find $x -type f | wc -l )
-#	echo -e "$x \t $size \t $files"
-#done