sourcelog

cmd/collector/main.go

@@ -33,7 +33,7 @@ var (
 	outDirPtr     = flag.String("out", "", "path to collect raw transactions into")
 	uidPtr        = flag.String("uid", "", "collector uid (part of output CSV filename)")
 	blxAuthToken  = flag.String("blx-token", defaultblxAuthToken, "bloxroute auth token (optional)")
-	txlog         = flag.Bool("txlog", false, "record txlog (all received transactions with source)")
+	sourcelog     = flag.Bool("source-log", false, "write a CSV with all received transactions from any source (timestamp_ms,hash,source)")
 )
 
 func main() {
@@ -99,7 +99,7 @@ func main() {
 	}
 
 	// Start service components
-	collector.Start(log, nodes, *outDirPtr, *uidPtr, *blxAuthToken, *txlog)
+	collector.Start(log, nodes, *outDirPtr, *uidPtr, *blxAuthToken, *sourcelog)
 
 	// Wwait for termination signal
 	exit := make(chan os.Signal, 1)

cmd/txlog/main.go → cmd/sourcelog/main.go

@@ -9,7 +9,7 @@ import (
 	"sort"
 
 	"github.com/flashbots/mempool-dumpster/common"
-	"github.com/flashbots/mempool-dumpster/txlog"
+	"github.com/flashbots/mempool-dumpster/sourcelog"
 	"go.uber.org/zap"
 )
 
@@ -45,7 +45,7 @@ func main() {
 
 	// perhaps only print the version
 	if *printVersion {
-		fmt.Printf("mempool-dumpster txlog %s\n", version)
+		fmt.Printf("mempool-dumpster sourcelog %s\n", version)
 		return
 	}
 
@@ -60,7 +60,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	log.Infow("mempool-dumpster txlog", "version", version)
+	log.Infow("mempool-dumpster sourcelog", "version", version)
 
 	// print possible aliases, for debugging
 	aliases := common.SourceAliasesFromEnv()
@@ -90,26 +90,26 @@ func main() {
 	checkInputFiles(files)
 
 	// Load transaction log files
-	txLog := txlog.LoadTxLog(log, files)
+	sourceLog := sourcelog.LoadSourceLogFiles(log, files)
 
 	// Write output file
 	if *outDirPtr != "" {
-		err := writeTxCSV(txLog)
+		err := writeTxCSV(sourceLog)
 		if err != nil {
 			log.Errorw("writeTxCSV", "error", err)
 		}
 	}
 
 	// Analyze
 	log.Info("Analyzing...")
-	analyzer := txlog.NewAnalyzer(txLog)
+	analyzer := sourcelog.NewAnalyzer(sourceLog)
 	analyzer.Print()
 }
 
 func getOutFilename() string {
-	fnOut := filepath.Join(*outDirPtr, "txlog.csv")
+	fnOut := filepath.Join(*outDirPtr, "sourcelog.csv")
 	if *outDatePtr != "" {
-		fnOut = filepath.Join(*outDirPtr, fmt.Sprintf("%s_txlog.csv", *outDatePtr))
+		fnOut = filepath.Join(*outDirPtr, fmt.Sprintf("%s_sourcelog.csv", *outDatePtr))
 	}
 	return fnOut
 }

cmd/summarizer/main.go

@@ -1,3 +1,4 @@
+// Loads many raw transaction CSV files (produced by the collector), creates summary files in CSV and Parquet, and writes a single CSV file with all raw transactions
 package main
 
 import (

collector/tx_processor.go

@@ -180,8 +180,8 @@ func (p *TxProcessor) getOutputCSVFiles(timestamp int64) (fTx, fSrcStats *os.Fil
 	}
 
 	if p.recSrcStats && !fSrcStatsOk {
-		// open txlog for writing
-		dir := filepath.Join(p.outDir, t.Format(time.DateOnly), "txlog")
+		// open sourcelog for writing
+		dir := filepath.Join(p.outDir, t.Format(time.DateOnly), "sourcelog")
 		err = os.MkdirAll(dir, os.ModePerm)
 		if err != nil {
 			p.log.Error(err)

txlog/analyzer.go → sourcelog/analyzer.go

@@ -1,4 +1,4 @@
-package txlog
+package sourcelog
 
 import (
 	"fmt"

txlog/txlog.go → sourcelog/sourcelog.go

@@ -1,9 +1,9 @@
-// Package txlog deals with loading and analyzing the transaction log.
+// Package sourcelog deals with loading and analyzing the transaction source logs.
 //
 // Input: CSV file(s) with the following format:
 //
 //	<timestamp_ms>,<tx_hash>,<source>
-package txlog
+package sourcelog
 
 import (
 	"bufio"
@@ -18,8 +18,8 @@ import (
 	"go.uber.org/zap"
 )
 
-// LoadTxLog loads all input CSV files and returns a map[hash][source] = timestampMs
-func LoadTxLog(log *zap.SugaredLogger, files []string) (txs map[string]map[string]int64) { //nolint:gocognit
+// LoadSourceLogFiles loads all input CSV files and returns a map[hash][source] = timestampMs
+func LoadSourceLogFiles(log *zap.SugaredLogger, files []string) (txs map[string]map[string]int64) { //nolint:gocognit
 	txs = make(map[string]map[string]int64)
 
 	timestampFirst, timestampLast := int64(0), int64(0)