From e273f5e4afda129a56fe3d6218dc368802791146 Mon Sep 17 00:00:00 2001 From: Thad House Date: Wed, 21 Feb 2024 22:13:32 -0800 Subject: [PATCH] Start adding most of DataLogManager --- codehelp/CodeHelpers/SymbolExtensions.cs | 3 +- src/ntcore/NetworkTableInstance.cs | 2 +- src/wpilibsharp/DataLogManager.cs | 377 ++++++++++++++++++ src/wpilibsharp/DriverStation.cs | 19 + src/wpilibsharp/Filesystem.cs | 6 + src/wpilibsharp/RobotBase.cs | 2 + src/wpilibsharp/RobotController.cs | 6 + src/wpilibsharp/RuntimeType.cs | 8 + src/wpiutil/Concurrent/Event.cs | 10 +- src/wpiutil/Concurrent/Synchronization.cs | 4 + src/wpiutil/Handles/SynchronizationHandles.cs | 15 + src/wpiutil/Logging/DataLog.cs | 7 +- src/wpiutil/Natives/SynchronizationNative.cs | 34 +- 13 files changed, 469 insertions(+), 24 deletions(-) create mode 100644 src/wpilibsharp/DataLogManager.cs create mode 100644 src/wpilibsharp/DriverStation.cs create mode 100644 src/wpilibsharp/Filesystem.cs create mode 100644 src/wpilibsharp/RobotController.cs create mode 100644 src/wpilibsharp/RuntimeType.cs create mode 100644 src/wpiutil/Handles/SynchronizationHandles.cs diff --git a/codehelp/CodeHelpers/SymbolExtensions.cs b/codehelp/CodeHelpers/SymbolExtensions.cs index cb170669..3684c97f 100644 --- a/codehelp/CodeHelpers/SymbolExtensions.cs +++ b/codehelp/CodeHelpers/SymbolExtensions.cs @@ -16,7 +16,8 @@ public static string GetFullTypeName(this ITypeSymbol symbol) return symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); } - public static string? GetNamespace(this ITypeSymbol symbol) { + public static string? GetNamespace(this ITypeSymbol symbol) + { // TODO Stop using ToDisplayString return symbol.ContainingNamespace is { IsGlobalNamespace: false } ns ? ns.ToDisplayString() : null; } diff --git a/src/ntcore/NetworkTableInstance.cs b/src/ntcore/NetworkTableInstance.cs index 4f2da7aa..ed56bb02 100644 --- a/src/ntcore/NetworkTableInstance.cs +++ b/src/ntcore/NetworkTableInstance.cs @@ -325,7 +325,7 @@ private void StartThread() m_thread = new Thread(() => { bool wasInterrupted = false; - ReadOnlySpan handles = [m_poller.Handle, m_waitQueueEvent.Handle]; + ReadOnlySpan handles = [m_poller.Handle, m_waitQueueEvent.Handle.Handle]; Span signaledStorage = [0, 0]; while (true) { diff --git a/src/wpilibsharp/DataLogManager.cs b/src/wpilibsharp/DataLogManager.cs new file mode 100644 index 00000000..3efc8808 --- /dev/null +++ b/src/wpilibsharp/DataLogManager.cs @@ -0,0 +1,377 @@ +using System.Text; +using NetworkTables; +using NetworkTables.Handles; +using WPIUtil.Concurrent; +using WPIUtil.Logging; + +namespace WPILib; + +public static class DataLogManger +{ + private static DataLog? m_log = null; + private static bool m_stopped; + private static string? m_logDir; + private static bool m_filenameOverride; + private static Thread? m_thread; + private static TimeZoneInfo m_utc = TimeZoneInfo.Utc; + private static bool m_ntLoggerEnabled = true; + private static NtDataLogger m_ntEntryLogger; + private static NtConnectionDataLogger m_ntConnLogger; + private static StringLogEntry? m_messageLog; + private static readonly object m_lockObject = new(); + private static volatile int m_runThread = 1; + + // if less than this much free space, delete log files until there is this much free space + // OR there are this many files remaining. + private const long FreeSpaceThreshold = 50000000L; + private const int FileCountThreshold = 10; + + public static void Start(string dir = "", string filename = "", double period = 0.25) + { + lock (m_lockObject) + { + if (m_log is null) + { + m_logDir = MakeLogDir(dir); + m_filenameOverride = !string.IsNullOrWhiteSpace(filename); + + // Delete all previously existing FRC_TBD_*.wpilog files. These only exist when the robot + // never connects to the DS, so they are very unlikely to have useful data and just clutter + // the filesystem. + var files = Directory.GetFiles(m_logDir).Where(name => Path.GetFileName(name).StartsWith("FRC_TBD_") && name.EndsWith(".wpilog")); + foreach (var file in files) + { + try + { + File.Delete(file); + } + catch (IOException) + { + Console.Error.WriteLine($"DataLogManager: could not delete {file}"); + } + } + m_log = new DataLog(m_logDir, MakeLogFilename(filename), period); + m_messageLog = new StringLogEntry(m_log, "messages"); + + if (m_ntLoggerEnabled) + { + StartNtLog(); + } + } + else if (m_stopped) + { + m_log.Filename = MakeLogFilename(filename); + m_log.Resume(); + m_stopped = false; + } + + if (m_thread is null) + { + m_thread = new Thread(LogMain) + { + Name = "DataLogDS", + IsBackground = true + }; + m_thread.Start(); + } + } + } + + public static void Stop() + { + lock (m_lockObject) + { + m_thread = null; + if (m_log is not null) + { + m_log.Stop(); + m_stopped = true; + } + } + } + + public static void Log(string message) + { + lock (m_lockObject) + { + if (m_messageLog is null) + { + Start(); + } + m_messageLog!.Append(message); + Console.WriteLine(message); + } + } + + public static DataLog DataLog + { + get + { + lock (m_lockObject) + { + if (m_log is null) + { + Start(); + } + return m_log!; + } + } + } + + public static string LogDir + { + get + { + lock (m_lockObject) + { + return m_logDir ?? ""; + } + } + } + + public static void LogNetworkTables(bool enabled) + { + lock (m_lockObject) + { + bool wasEnabled = m_ntLoggerEnabled; + m_ntLoggerEnabled = enabled; + if (m_log is null) + { + Start(); + return; + } + if (enabled && !wasEnabled) + { + StartNtLog(); + } + else if (!enabled && wasEnabled) + { + StopNtLog(); + } + } + } + + private static string MakeLogDir(string dir) + { + if (!string.IsNullOrWhiteSpace(dir)) + { + return dir; + } + + if (RobotBase.IsReal) + { + try + { + Directory.CreateDirectory("/u/logs"); + return "/u/logs"; + } + catch (IOException) + { + + } + if (RobotBase.RuntimeType == RuntimeType.RoboRIO) + { + DriverStation.ReportWarning("DataLogManager: Logging to RoboRIO 1 internal storage is not recommended!, Plug in a FAT32 formatted flash drive!", false); + } + try + { + Directory.CreateDirectory("/home/lvuser/logs"); + } + catch (IOException) + { + + } + return "/home/lvuser/logs"; + } + string logDir = Filesystem.OperatingDirectory; + try + { + Directory.CreateDirectory(logDir); + } + catch (IOException) + { + + } + return logDir; + } + + private static string MakeLogFilename(string filenameOverride) + { + if (!string.IsNullOrWhiteSpace(filenameOverride)) + { + return filenameOverride; + } + + Random rnd = new Random(); + StringBuilder filename = new(); + filename.Append("FRC_TBD_"); + for (int i = 0; i < 4; i++) + { + filename.Append(rnd.Next(0x10000).ToString("x4")); + } + filename.Append(".wpilog"); + return filename.ToString(); + } + + private static void StartNtLog() + { + var inst = NetworkTableInstance.Default; + m_ntEntryLogger = inst.StartEntryDataLog(m_log!, "", "NT:"); + m_ntConnLogger = inst.StartConnectionDataLog(m_log!, "NTConnection"); + } + + private static void StopNtLog() + { + NetworkTableInstance.StopEntryDataLog(m_ntEntryLogger); + NetworkTableInstance.StopConnectionDataLog(m_ntConnLogger); + } + + private static void LogMain() + { + // based on free disk space, scan for "old" FRC_*.wpilog files and remove + { + string logDir = m_logDir!; + var fileInfo = new DirectoryInfo(logDir); + var driveInfo = new DriveInfo(fileInfo.FullName); + var freeSpace = driveInfo.AvailableFreeSpace; + if (freeSpace < FreeSpaceThreshold) + { + var files = Directory.GetFiles(logDir).Where(name => + { + name = Path.GetFileName(name); + return name.StartsWith("FRC_") + && name.EndsWith(".wpilog") + && !name.StartsWith("FRC_TBD_"); + }).OrderBy(File.GetLastWriteTimeUtc); + int count = 0; + foreach (var file in files) + { + --count; + if (count < FileCountThreshold) + { + break; + } + try + { + long length = new FileInfo(file).Length; + DriverStation.ReportWarning($"DataLogManager: Deleted {Path.GetFileName(file)}", false); + freeSpace += length; + if (freeSpace >= FreeSpaceThreshold) + { + break; + } + } + catch (IOException) + { + Console.Error.WriteLine($"DataLogManager: could not delete {file}"); + } + } + } + else if (freeSpace < 2 * FreeSpaceThreshold) + { + DriverStation.ReportWarning( + "DataLogManager: Log storage device has " + + freeSpace / 1000000 + + " MB of free space remaining! Logs will get deleted below " + + FreeSpaceThreshold / 1000000 + + " MB of free space." + + "Consider deleting logs off the storage device.", + false); + } + } + + int timeoutCount = 0; + bool paused = false; + int dsAttachCount = 0; + int fmsAttachCount = 0; + bool dsRenamed = m_filenameOverride; + bool fmsRenamed = m_filenameOverride; + int sysTimeCount = 0; + IntegerLogEntry sysSimteEntry = new IntegerLogEntry(m_log!, "systemTime", "{\"source\":\"DataLogManager\",\"format\":\"time_t_us\"}"); + using Event newDataEvent = new(); + DriverStation.ProvideRefreshedDataEventHandle(newDataEvent.Handle); + while (Interlocked.CompareExchange(ref m_thread, null, null) != null) + { + var result = Synchronization.WaitForObject(newDataEvent.Handle.Handle, TimeSpan.FromSeconds(0.25)); + if (result == SynchronizationResult.Cancelled || Interlocked.CompareExchange(ref m_thread, null, null) != null) + { + break; + } + if (result == SynchronizationResult.TimedOut) + { + timeoutCount++; + // pause logging after being disconnected for 10 seconds + if (timeoutCount > 40 && !paused) + { + timeoutCount = 0; + paused = true; + m_log!.Pause(); + } + continue; + } + // when we connect to the DS, resume logging + timeoutCount = 0; + if (paused) + { + paused = false; + m_log!.Resume(); + } + + if (!dsRenamed) + { + // track ds attach + if (DriverStation.IsDSAttached) + { + dsAttachCount++; + + } + else + { + dsAttachCount = 0; + } + if (dsAttachCount > 50) // 1 second + { + if (RobotController.IsSystemTimeValid) + { + var now = DateTime.UtcNow; + // TODO make this time match + m_log!.Filename = $"FRC_{now}.wpilog"; + dsRenamed = true; + } + else + { + dsAttachCount = 0; // wait a bit and try again + } + } + } + + if (!fmsRenamed) + { + // track FMS attach + if (DriverStation.IsFMSAttached) + { + fmsAttachCount++; + } + else + { + fmsAttachCount = 0; + } + if (fmsAttachCount > 250) // 5 seconds + { + } + } + + // Write system time every ~5 seconds + sysTimeCount++; + if (sysTimeCount > 250) + { + sysTimeCount = 0; + if (RobotController.IsSystemTimeValid) + { + // Write time + } + } + } + + } +} diff --git a/src/wpilibsharp/DriverStation.cs b/src/wpilibsharp/DriverStation.cs new file mode 100644 index 00000000..2d688154 --- /dev/null +++ b/src/wpilibsharp/DriverStation.cs @@ -0,0 +1,19 @@ +using WPIUtil.Handles; + +namespace WPILib; + +public static class DriverStation +{ + public static void ReportWarning(string warning, bool printTrace) + { + + } + + public static void ProvideRefreshedDataEventHandle(WpiEventHandle handle) + { + + } + + public static bool IsDSAttached => true; + public static bool IsFMSAttached => true; +} diff --git a/src/wpilibsharp/Filesystem.cs b/src/wpilibsharp/Filesystem.cs new file mode 100644 index 00000000..a55ba0b3 --- /dev/null +++ b/src/wpilibsharp/Filesystem.cs @@ -0,0 +1,6 @@ +namespace WPILib; + +public static class Filesystem +{ + public static string OperatingDirectory => "/home/lvuser"; +} diff --git a/src/wpilibsharp/RobotBase.cs b/src/wpilibsharp/RobotBase.cs index 59640b24..9d3716b7 100644 --- a/src/wpilibsharp/RobotBase.cs +++ b/src/wpilibsharp/RobotBase.cs @@ -2,5 +2,7 @@ namespace WPILib; public abstract class RobotBase { + public static bool IsReal => true; + public static RuntimeType RuntimeType => RuntimeType.RoboRIO; } diff --git a/src/wpilibsharp/RobotController.cs b/src/wpilibsharp/RobotController.cs new file mode 100644 index 00000000..bd648971 --- /dev/null +++ b/src/wpilibsharp/RobotController.cs @@ -0,0 +1,6 @@ +namespace WPILib; + +public static class RobotController +{ + public static bool IsSystemTimeValid => true; +} diff --git a/src/wpilibsharp/RuntimeType.cs b/src/wpilibsharp/RuntimeType.cs new file mode 100644 index 00000000..7db22fb9 --- /dev/null +++ b/src/wpilibsharp/RuntimeType.cs @@ -0,0 +1,8 @@ +namespace WPILib; + +public enum RuntimeType +{ + RoboRIO, + RoboRIO2, + Simulation +} diff --git a/src/wpiutil/Concurrent/Event.cs b/src/wpiutil/Concurrent/Event.cs index 486a8e72..dce976e1 100644 --- a/src/wpiutil/Concurrent/Event.cs +++ b/src/wpiutil/Concurrent/Event.cs @@ -1,17 +1,21 @@ +using System.Runtime.InteropServices.Marshalling; +using WPIUtil.Handles; using WPIUtil.Natives; namespace WPIUtil.Concurrent; + + public sealed class Event(bool manualReset = false, bool initialState = false) : IDisposable { - public int Handle { get; private set; } = SynchronizationNative.CreateEvent(manualReset, initialState); + public WpiEventHandle Handle { get; private set; } = SynchronizationNative.CreateEvent(manualReset, initialState); public void Dispose() { - if (Handle != 0) + if (Handle.Handle != 0) { SynchronizationNative.DestroyEvent(Handle); - Handle = 0; + Handle = default; } } diff --git a/src/wpiutil/Concurrent/Synchronization.cs b/src/wpiutil/Concurrent/Synchronization.cs index 08b946f5..f839dccc 100644 --- a/src/wpiutil/Concurrent/Synchronization.cs +++ b/src/wpiutil/Concurrent/Synchronization.cs @@ -1,8 +1,12 @@ +using System.Runtime.InteropServices.Marshalling; using CommunityToolkit.Diagnostics; +using WPIUtil.Handles; using WPIUtil.Natives; namespace WPIUtil.Concurrent; + + public static class Synchronization { public static SynchronizationResult WaitForObject(int handle) diff --git a/src/wpiutil/Handles/SynchronizationHandles.cs b/src/wpiutil/Handles/SynchronizationHandles.cs new file mode 100644 index 00000000..6d98d3b6 --- /dev/null +++ b/src/wpiutil/Handles/SynchronizationHandles.cs @@ -0,0 +1,15 @@ +using System.Runtime.InteropServices.Marshalling; + +namespace WPIUtil.Handles; + +public interface IWpiSynchronizationHandle : IWPIIntHandle; + +[NativeMarshalling(typeof(WPIIntHandleMarshaller))] +public record struct WpiSemaphoreHandle(int Handle) : IWPIIntHandle, IWpiSynchronizationHandle +{ +} + +[NativeMarshalling(typeof(WPIIntHandleMarshaller))] +public record struct WpiEventHandle(int Handle) : IWPIIntHandle, IWpiSynchronizationHandle +{ +} diff --git a/src/wpiutil/Logging/DataLog.cs b/src/wpiutil/Logging/DataLog.cs index 8fd55ed5..6d42f36c 100644 --- a/src/wpiutil/Logging/DataLog.cs +++ b/src/wpiutil/Logging/DataLog.cs @@ -46,9 +46,12 @@ public void Dispose() } } - public void SetFilename(string filename) + public string Filename { - DataLogNative.SetFilename(NativeHandle, filename); + set + { + DataLogNative.SetFilename(NativeHandle, value); + } } public void Flush() diff --git a/src/wpiutil/Natives/SynchronizationNative.cs b/src/wpiutil/Natives/SynchronizationNative.cs index 696837ef..57dcea49 100644 --- a/src/wpiutil/Natives/SynchronizationNative.cs +++ b/src/wpiutil/Natives/SynchronizationNative.cs @@ -4,54 +4,54 @@ namespace WPIUtil.Natives; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using CommunityToolkit.Diagnostics; -using SynchronizationHandle = int; +using WPIUtil.Handles; public static partial class SynchronizationNative { [LibraryImport("wpiutil", EntryPoint = "WPI_CreateEvent")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial SynchronizationHandle CreateEvent([MarshalAs(UnmanagedType.I4)] bool manualReset, [MarshalAs(UnmanagedType.I4)] bool initialState); + public static partial WpiEventHandle CreateEvent([MarshalAs(UnmanagedType.I4)] bool manualReset, [MarshalAs(UnmanagedType.I4)] bool initialState); [LibraryImport("wpiutil", EntryPoint = "WPI_DestroyEvent")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void DestroyEvent(SynchronizationHandle handle); + public static partial void DestroyEvent(WpiEventHandle handle); [LibraryImport("wpiutil", EntryPoint = "WPI_SetEvent")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void SetEvent(SynchronizationHandle handle); + public static partial void SetEvent(WpiEventHandle handle); [LibraryImport("wpiutil", EntryPoint = "WPI_ResetEvent")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void ResetEvent(SynchronizationHandle handle); + public static partial void ResetEvent(WpiEventHandle handle); [LibraryImport("wpiutil", EntryPoint = "WPI_CreateSemaphore")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial SynchronizationHandle CreateSemaphore(int initialCount, int maximumCount); + public static partial WpiSemaphoreHandle CreateSemaphore(int initialCount, int maximumCount); [LibraryImport("wpiutil", EntryPoint = "WPI_DestroySemaphore")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void DestroySemaphore(SynchronizationHandle handle); + public static partial void DestroySemaphore(WpiSemaphoreHandle handle); [LibraryImport("wpiutil", EntryPoint = "WPI_ReleaseSemaphore")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [return: MarshalAs(UnmanagedType.U4)] - public static unsafe partial bool ReleaseSemaphore(SynchronizationHandle handle, int releaseCount, out int prevCount); + public static unsafe partial bool ReleaseSemaphore(WpiSemaphoreHandle handle, int releaseCount, out int prevCount); [LibraryImport("wpiutil", EntryPoint = "WPI_WaitForObject")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [return: MarshalAs(UnmanagedType.U4)] - public static partial bool WaitForObject(SynchronizationHandle handle); + public static partial bool WaitForObject(int handle); [LibraryImport("wpiutil", EntryPoint = "WPI_WaitForObjectTimeout")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [return: MarshalAs(UnmanagedType.U4)] - public static unsafe partial bool WaitForObjectTimeout(SynchronizationHandle handle, double timeout, [MarshalAs(UnmanagedType.I4)] out bool timedOut); + public static unsafe partial bool WaitForObjectTimeout(int handle, double timeout, [MarshalAs(UnmanagedType.I4)] out bool timedOut); [LibraryImport("wpiutil", EntryPoint = "WPI_WaitForObjects")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static unsafe partial int WaitForObjects(ReadOnlySpan handles, int handlesCount, Span signaled); + public static unsafe partial int WaitForObjects(ReadOnlySpan handles, int handlesCount, Span signaled); - public static unsafe ReadOnlySpan WaitForObjects(ReadOnlySpan handles, Span signaled) + public static unsafe ReadOnlySpan WaitForObjects(ReadOnlySpan handles, Span signaled) { if (handles.Length > signaled.Length) { @@ -63,21 +63,21 @@ public static unsafe ReadOnlySpan WaitForObjects(ReadOnlySpan handles, int handlesCount, Span signaled, double timeout, [MarshalAs(UnmanagedType.I4)] out bool timedOut); + public static unsafe partial int WaitForObjectsTimeout(ReadOnlySpan handles, int handlesCount, Span signaled, double timeout, [MarshalAs(UnmanagedType.I4)] out bool timedOut); [LibraryImport("wpiutil", EntryPoint = "WPI_CreateSemaphore")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void CreateSignalObject(SynchronizationHandle handle, [MarshalAs(UnmanagedType.I4)] bool manualReset, [MarshalAs(UnmanagedType.I4)] bool InitialState); + public static partial void CreateSignalObject(int handle, [MarshalAs(UnmanagedType.I4)] bool manualReset, [MarshalAs(UnmanagedType.I4)] bool InitialState); [LibraryImport("wpiutil", EntryPoint = "WPI_SetSignalObject")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void SetSignalObject(SynchronizationHandle handle); + public static partial void SetSignalObject(int handle); [LibraryImport("wpiutil", EntryPoint = "WPI_ResetSignalObject")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void ResetSignalObject(SynchronizationHandle handle); + public static partial void ResetSignalObject(int handle); [LibraryImport("wpiutil", EntryPoint = "WPI_DestroySignalObject")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - public static partial void DestroySignalObject(SynchronizationHandle handle); + public static partial void DestroySignalObject(int handle); }