diff --git a/internal/index/builder/builder.go b/internal/index/builder/builder.go index 0257a88..f240454 100644 --- a/internal/index/builder/builder.go +++ b/internal/index/builder/builder.go @@ -23,6 +23,7 @@ type ( Builder struct { snapshots []*snapshot knownPcaps []*pcapmetadata.PcapInfo + packetCount uint indexDir string snapshotDir string snapshotFilename string @@ -61,6 +62,7 @@ func New(pcapDir, indexDir, snapshotDir string, cachedKnownPcaps []*pcapmetadata } } b.knownPcaps = append(b.knownPcaps, info) + b.packetCount += info.PacketCount } // load the snapshot file with the most packets covered snapshotFiles, err := os.ReadDir(snapshotDir) @@ -541,6 +543,9 @@ outer: } b.knownPcaps = append(b.knownPcaps, newPcapInfos...) + for _, pi := range newPcapInfos { + b.packetCount += pi.PacketCount + } b.snapshots = newSnapshots outputFiles := []string{} @@ -551,6 +556,10 @@ outer: return nProcessedPcaps, indexes, nil } +func (b *Builder) PacketCount() uint { + return b.packetCount +} + func (b *Builder) KnownPcaps() []*pcapmetadata.PcapInfo { return b.knownPcaps } diff --git a/internal/index/manager/manager.go b/internal/index/manager/manager.go index 78ae1b6..cdcf90f 100644 --- a/internal/index/manager/manager.go +++ b/internal/index/manager/manager.go @@ -42,10 +42,13 @@ const ( type ( PcapStatistics struct { - PcapCount int - ImportJobCount int - StreamCount int - PacketCount int + PcapCount int + PacketCount int + ImportJobCount int + IndexCount int + StreamCount int + StreamRecordCount int + PacketRecordCount int } Event struct { @@ -108,7 +111,8 @@ type ( builder *builder.Builder indexes []*index.Reader - nStreams, nPackets int + nStreamRecords int + nPacketRecords int nextStreamID uint64 nUnmergeableIndexes int stateFilename string @@ -140,6 +144,8 @@ type ( PcapCount int StreamCount int PacketCount int + StreamRecordCount int + PacketRecordCount int MergeJobRunning bool TaggingJobRunning bool ConverterJobRunning bool @@ -249,8 +255,8 @@ func New(pcapDir, indexDir, snapshotDir, stateDir, converterDir string) (*Manage continue } mgr.indexes = append(mgr.indexes, idx) - mgr.nStreams += idx.StreamCount() - mgr.nPackets += idx.PacketCount() + mgr.nStreamRecords += idx.StreamCount() + mgr.nPacketRecords += idx.PacketCount() if next := idx.MaxStreamID() + 1; mgr.nextStreamID < next { mgr.nextStreamID = next } @@ -596,8 +602,8 @@ func (mgr *Manager) importPcapJob(filenames []string, nextStreamID uint64, exist // add new indexes if some were created if len(createdIndexes) > 0 { mgr.indexes = append(mgr.indexes, createdIndexes...) - mgr.nStreams += newStreamCount - mgr.nPackets += newPacketCount + mgr.nStreamRecords += newStreamCount + mgr.nPacketRecords += newPacketCount mgr.nextStreamID = newNextStreamID mgr.lock(createdIndexes) mgr.addedStreamsDuringTaggingJob.Or(addedStreams) @@ -623,10 +629,13 @@ func (mgr *Manager) importPcapJob(filenames []string, nextStreamID uint64, exist mgr.event(Event{ Type: "pcapProcessed", PcapStats: PcapStatistics{ - PcapCount: len(mgr.builder.KnownPcaps()), - ImportJobCount: len(mgr.importJobs), - StreamCount: mgr.nStreams, - PacketCount: mgr.nPackets, + PcapCount: len(mgr.builder.KnownPcaps()), + ImportJobCount: len(mgr.importJobs), + StreamCount: int(mgr.nextStreamID), + PacketCount: int(mgr.builder.PacketCount()), + IndexCount: len(mgr.indexes), + StreamRecordCount: mgr.nStreamRecords, + PacketRecordCount: mgr.nPacketRecords, }, }) mgr.triggerPcapProcessedWebhooks(filenames[:processedFiles]) @@ -643,7 +652,7 @@ func (mgr *Manager) startMergeJobIfNeeded() { return } } - nStreams := mgr.nStreams + nStreams := mgr.nStreamRecords for i, idx := range mgr.indexes { c := idx.StreamCount() nStreams -= c @@ -715,14 +724,23 @@ func (mgr *Manager) mergeIndexesJob(offset int, indexes []*index.Reader, release mgr.lock(mergedIndexes) mgr.indexes = append(mgr.indexes[:offset], append(mergedIndexes, mgr.indexes[offset+len(indexes):]...)...) mgr.nUnmergeableIndexes += len(mergedIndexes) - 1 - mgr.nStreams += streamsDiff - mgr.nPackets += packetsDiff + mgr.nStreamRecords += streamsDiff + mgr.nPacketRecords += packetsDiff } mgr.mergeJobRunning = false mgr.startMergeJobIfNeeded() releaser.release(mgr) mgr.event(Event{ Type: "indexesMerged", + PcapStats: PcapStatistics{ + PcapCount: len(mgr.builder.KnownPcaps()), + ImportJobCount: len(mgr.importJobs), + StreamCount: int(mgr.nextStreamID), + PacketCount: int(mgr.builder.PacketCount()), + IndexCount: len(mgr.indexes), + StreamRecordCount: mgr.nStreamRecords, + PacketRecordCount: mgr.nPacketRecords, + }, }) } } @@ -813,8 +831,10 @@ func (mgr *Manager) Status() Statistics { IndexLockCount: locks, PcapCount: len(mgr.builder.KnownPcaps()), ImportJobCount: len(mgr.importJobs), - StreamCount: mgr.nStreams, - PacketCount: mgr.nPackets, + StreamRecordCount: mgr.nStreamRecords, + PacketRecordCount: mgr.nPacketRecords, + StreamCount: int(mgr.nextStreamID), + PacketCount: int(mgr.builder.PacketCount()), MergeJobRunning: mgr.mergeJobRunning, TaggingJobRunning: mgr.taggingJobRunning, ConverterJobRunning: mgr.converterJobRunning, diff --git a/web/src/apiClient.guard.ts b/web/src/apiClient.guard.ts index 87030e4..bbe0ab1 100644 --- a/web/src/apiClient.guard.ts +++ b/web/src/apiClient.guard.ts @@ -124,7 +124,9 @@ export function isStatistics(obj: unknown): obj is Statistics { typeof typedObj["PcapCount"] === "number" && typeof typedObj["ImportJobCount"] === "number" && typeof typedObj["StreamCount"] === "number" && + typeof typedObj["StreamRecordCount"] === "number" && typeof typedObj["PacketCount"] === "number" && + typeof typedObj["PacketRecordCount"] === "number" && typeof typedObj["MergeJobRunning"] === "boolean" && typeof typedObj["TaggingJobRunning"] === "boolean" && typeof typedObj["ConverterJobRunning"] === "boolean" diff --git a/web/src/apiClient.ts b/web/src/apiClient.ts index a1268ca..c200f72 100644 --- a/web/src/apiClient.ts +++ b/web/src/apiClient.ts @@ -74,7 +74,9 @@ export type Statistics = { PcapCount: number; ImportJobCount: number; StreamCount: number; + StreamRecordCount: number; PacketCount: number; + PacketRecordCount: number; MergeJobRunning: boolean; TaggingJobRunning: boolean; ConverterJobRunning: boolean; diff --git a/web/src/components/Status.vue b/web/src/components/Status.vue index d599e66..286def0 100644 --- a/web/src/components/Status.vue +++ b/web/src/components/Status.vue @@ -14,7 +14,7 @@ Status - + {{ name }} {{ value }} @@ -26,12 +26,11 @@