Merge pull request #112 from spq/correct-stats

improve the statistics:

internal/index/builder/builder.go

@@ -23,6 +23,7 @@ type (
 	Builder struct {
 		snapshots        []*snapshot
 		knownPcaps       []*pcapmetadata.PcapInfo
+		packetCount      uint
 		indexDir         string
 		snapshotDir      string
 		snapshotFilename string
@@ -61,6 +62,7 @@ func New(pcapDir, indexDir, snapshotDir string, cachedKnownPcaps []*pcapmetadata
 			}
 		}
 		b.knownPcaps = append(b.knownPcaps, info)
+		b.packetCount += info.PacketCount
 	}
 	// load the snapshot file with the most packets covered
 	snapshotFiles, err := os.ReadDir(snapshotDir)
@@ -541,6 +543,9 @@ outer:
 	}
 
 	b.knownPcaps = append(b.knownPcaps, newPcapInfos...)
+	for _, pi := range newPcapInfos {
+		b.packetCount += pi.PacketCount
+	}
 	b.snapshots = newSnapshots
 
 	outputFiles := []string{}
@@ -551,6 +556,10 @@ outer:
 	return nProcessedPcaps, indexes, nil
 }
 
+func (b *Builder) PacketCount() uint {
+	return b.packetCount
+}
+
 func (b *Builder) KnownPcaps() []*pcapmetadata.PcapInfo {
 	return b.knownPcaps
 }

internal/index/manager/manager.go

@@ -42,10 +42,13 @@ const (
 
 type (
 	PcapStatistics struct {
-		PcapCount      int
-		ImportJobCount int
-		StreamCount    int
-		PacketCount    int
+		PcapCount         int
+		PacketCount       int
+		ImportJobCount    int
+		IndexCount        int
+		StreamCount       int
+		StreamRecordCount int
+		PacketRecordCount int
 	}
 
 	Event struct {
@@ -108,7 +111,8 @@ type (
 
 		builder             *builder.Builder
 		indexes             []*index.Reader
-		nStreams, nPackets  int
+		nStreamRecords      int
+		nPacketRecords      int
 		nextStreamID        uint64
 		nUnmergeableIndexes int
 		stateFilename       string
@@ -140,6 +144,8 @@ type (
 		PcapCount           int
 		StreamCount         int
 		PacketCount         int
+		StreamRecordCount   int
+		PacketRecordCount   int
 		MergeJobRunning     bool
 		TaggingJobRunning   bool
 		ConverterJobRunning bool
@@ -249,8 +255,8 @@ func New(pcapDir, indexDir, snapshotDir, stateDir, converterDir string) (*Manage
 			continue
 		}
 		mgr.indexes = append(mgr.indexes, idx)
-		mgr.nStreams += idx.StreamCount()
-		mgr.nPackets += idx.PacketCount()
+		mgr.nStreamRecords += idx.StreamCount()
+		mgr.nPacketRecords += idx.PacketCount()
 		if next := idx.MaxStreamID() + 1; mgr.nextStreamID < next {
 			mgr.nextStreamID = next
 		}
@@ -596,8 +602,8 @@ func (mgr *Manager) importPcapJob(filenames []string, nextStreamID uint64, exist
 		// add new indexes if some were created
 		if len(createdIndexes) > 0 {
 			mgr.indexes = append(mgr.indexes, createdIndexes...)
-			mgr.nStreams += newStreamCount
-			mgr.nPackets += newPacketCount
+			mgr.nStreamRecords += newStreamCount
+			mgr.nPacketRecords += newPacketCount
 			mgr.nextStreamID = newNextStreamID
 			mgr.lock(createdIndexes)
 			mgr.addedStreamsDuringTaggingJob.Or(addedStreams)
@@ -623,10 +629,13 @@ func (mgr *Manager) importPcapJob(filenames []string, nextStreamID uint64, exist
 		mgr.event(Event{
 			Type: "pcapProcessed",
 			PcapStats: PcapStatistics{
-				PcapCount:      len(mgr.builder.KnownPcaps()),
-				ImportJobCount: len(mgr.importJobs),
-				StreamCount:    mgr.nStreams,
-				PacketCount:    mgr.nPackets,
+				PcapCount:         len(mgr.builder.KnownPcaps()),
+				ImportJobCount:    len(mgr.importJobs),
+				StreamCount:       int(mgr.nextStreamID),
+				PacketCount:       int(mgr.builder.PacketCount()),
+				IndexCount:        len(mgr.indexes),
+				StreamRecordCount: mgr.nStreamRecords,
+				PacketRecordCount: mgr.nPacketRecords,
 			},
 		})
 		mgr.triggerPcapProcessedWebhooks(filenames[:processedFiles])
@@ -643,7 +652,7 @@ func (mgr *Manager) startMergeJobIfNeeded() {
 			return
 		}
 	}
-	nStreams := mgr.nStreams
+	nStreams := mgr.nStreamRecords
 	for i, idx := range mgr.indexes {
 		c := idx.StreamCount()
 		nStreams -= c
@@ -715,14 +724,23 @@ func (mgr *Manager) mergeIndexesJob(offset int, indexes []*index.Reader, release
 			mgr.lock(mergedIndexes)
 			mgr.indexes = append(mgr.indexes[:offset], append(mergedIndexes, mgr.indexes[offset+len(indexes):]...)...)
 			mgr.nUnmergeableIndexes += len(mergedIndexes) - 1
-			mgr.nStreams += streamsDiff
-			mgr.nPackets += packetsDiff
+			mgr.nStreamRecords += streamsDiff
+			mgr.nPacketRecords += packetsDiff
 		}
 		mgr.mergeJobRunning = false
 		mgr.startMergeJobIfNeeded()
 		releaser.release(mgr)
 		mgr.event(Event{
 			Type: "indexesMerged",
+			PcapStats: PcapStatistics{
+				PcapCount:         len(mgr.builder.KnownPcaps()),
+				ImportJobCount:    len(mgr.importJobs),
+				StreamCount:       int(mgr.nextStreamID),
+				PacketCount:       int(mgr.builder.PacketCount()),
+				IndexCount:        len(mgr.indexes),
+				StreamRecordCount: mgr.nStreamRecords,
+				PacketRecordCount: mgr.nPacketRecords,
+			},
 		})
 	}
 }
@@ -813,8 +831,10 @@ func (mgr *Manager) Status() Statistics {
 			IndexLockCount:      locks,
 			PcapCount:           len(mgr.builder.KnownPcaps()),
 			ImportJobCount:      len(mgr.importJobs),
-			StreamCount:         mgr.nStreams,
-			PacketCount:         mgr.nPackets,
+			StreamRecordCount:   mgr.nStreamRecords,
+			PacketRecordCount:   mgr.nPacketRecords,
+			StreamCount:         int(mgr.nextStreamID),
+			PacketCount:         int(mgr.builder.PacketCount()),
 			MergeJobRunning:     mgr.mergeJobRunning,
 			TaggingJobRunning:   mgr.taggingJobRunning,
 			ConverterJobRunning: mgr.converterJobRunning,

web/src/apiClient.guard.ts

@@ -124,7 +124,9 @@ export function isStatistics(obj: unknown): obj is Statistics {
         typeof typedObj["PcapCount"] === "number" &&
         typeof typedObj["ImportJobCount"] === "number" &&
         typeof typedObj["StreamCount"] === "number" &&
+        typeof typedObj["StreamRecordCount"] === "number" &&
         typeof typedObj["PacketCount"] === "number" &&
+        typeof typedObj["PacketRecordCount"] === "number" &&
         typeof typedObj["MergeJobRunning"] === "boolean" &&
         typeof typedObj["TaggingJobRunning"] === "boolean" &&
         typeof typedObj["ConverterJobRunning"] === "boolean"

web/src/apiClient.ts

@@ -74,7 +74,9 @@ export type Statistics = {
   PcapCount: number;
   ImportJobCount: number;
   StreamCount: number;
+  StreamRecordCount: number;
   PacketCount: number;
+  PacketRecordCount: number;
   MergeJobRunning: boolean;
   TaggingJobRunning: boolean;
   ConverterJobRunning: boolean;

web/src/components/Status.vue

@@ -14,7 +14,7 @@
       <v-card-title>Status</v-card-title>
       <v-simple-table>
         <tbody>
-          <tr v-for="(value, name) in status" :key="name">
+          <tr v-for="(value, name) in store.status || []" :key="name">
             <th>{{ name }}</th>
             <td width="100%">{{ value }}</td>
           </tr>
@@ -26,12 +26,11 @@
 
 <script lang="ts" setup>
 import ToolBar from "./ToolBar.vue";
-import { computed, onMounted } from "vue";
+import { onMounted } from "vue";
 import { useRootStore } from "@/stores";
 import { EventBus } from "./EventBus";
 
 const store = useRootStore();
-const status = computed(() => store.status);
 
 onMounted(() => {
   updateStatus();

web/src/stores/websocket.guard.ts

@@ -2,7 +2,7 @@
  * Generated type guards for "websocket.ts".
  * WARNING: Do not manually change this file.
  */
-import { Event, TagEvent, ConverterEvent, PcapProcessedEvent } from "./websocket";
+import { Event, TagEvent, ConverterEvent, PcapStatsEvent } from "./websocket";
 
 export function isEvent(obj: unknown): obj is Event {
     const typedObj = obj as Event
@@ -68,19 +68,23 @@ export function isConverterEvent(obj: unknown): obj is ConverterEvent {
     )
 }
 
-export function isPcapProcessedEvent(obj: unknown): obj is PcapProcessedEvent {
-    const typedObj = obj as PcapProcessedEvent
+export function isPcapStatsEvent(obj: unknown): obj is PcapStatsEvent {
+    const typedObj = obj as PcapStatsEvent
     return (
         (typedObj !== null &&
             typeof typedObj === "object" ||
             typeof typedObj === "function") &&
-        typedObj["Type"] === "pcapProcessed" &&
+        (typedObj["Type"] === "indexesMerged" ||
+            typedObj["Type"] === "pcapProcessed") &&
         (typedObj["PcapStats"] !== null &&
             typeof typedObj["PcapStats"] === "object" ||
             typeof typedObj["PcapStats"] === "function") &&
         typeof typedObj["PcapStats"]["PcapCount"] === "number" &&
+        typeof typedObj["PcapStats"]["PacketCount"] === "number" &&
         typeof typedObj["PcapStats"]["ImportJobCount"] === "number" &&
+        typeof typedObj["PcapStats"]["IndexCount"] === "number" &&
         typeof typedObj["PcapStats"]["StreamCount"] === "number" &&
-        typeof typedObj["PcapStats"]["PacketCount"] === "number"
+        typeof typedObj["PcapStats"]["StreamRecordCount"] === "number" &&
+        typeof typedObj["PcapStats"]["PacketRecordCount"] === "number"
     )
 }

web/src/stores/websocket.ts

@@ -5,7 +5,7 @@ import { useStreamsStore } from "./streams";
 import {
   isConverterEvent,
   isEvent,
-  isPcapProcessedEvent,
+  isPcapStatsEvent,
   isTagEvent,
 } from "./websocket.guard";
 
@@ -45,14 +45,17 @@ export type ConverterEvent = {
 
 export type PcapStats = {
   PcapCount: number;
+  PacketCount: number;
   ImportJobCount: number;
+  IndexCount: number;
   StreamCount: number;
-  PacketCount: number;
+  StreamRecordCount: number;
+  PacketRecordCount: number;
 };
 
-/** @see {isPcapProcessedEvent} ts-auto-guard:type-guard */
-export type PcapProcessedEvent = {
-  Type: "pcapProcessed";
+/** @see {isPcapStatsEvent} ts-auto-guard:type-guard */
+export type PcapStatsEvent = {
+  Type: "pcapProcessed" | "indexesMerged";
   PcapStats: PcapStats;
 };
 
@@ -174,16 +177,20 @@ export function setupWebsocket() {
           }
           break;
         case "pcapProcessed":
-          if (!isPcapProcessedEvent(e)) {
-            console.error("Invalid pcap processed event:", e);
+        case "indexesMerged":
+          if (!isPcapStatsEvent(e)) {
+            console.error("Invalid pcap stats event:", e);
             return;
           }
-          streamsStore.outdated = true;
+          if (e.Type == "pcapProcessed") streamsStore.outdated = true;
           if (store.status != null) {
             store.status.PcapCount = e.PcapStats.PcapCount;
+            store.status.PacketCount = e.PcapStats.PacketCount;
             store.status.ImportJobCount = e.PcapStats.ImportJobCount;
+            store.status.IndexCount = e.PcapStats.IndexCount;
             store.status.StreamCount = e.PcapStats.StreamCount;
-            store.status.PacketCount = e.PcapStats.PacketCount;
+            store.status.StreamRecordCount = e.PcapStats.StreamRecordCount;
+            store.status.PacketRecordCount = e.PcapStats.PacketRecordCount;
           }
           break;
         default: