toolset progress

rusefi · Jul 30, 2023 · da62825 · da62825
1 parent 0dd654f
commit da62825
Show file tree

Hide file tree

Showing 11 changed files with 74 additions and 39 deletions.
diff --git a/src/main/java/com/rusefi/can/analysis/ByteRateOfChange.java b/src/main/java/com/rusefi/can/analysis/ByteRateOfChange.java
@@ -12,17 +12,12 @@
 public class ByteRateOfChange {
 
     static class TraceFileMetaIndex {
-        HashMap<ByteId, ByteStatistics> statistics = new HashMap<>();
+        final HashMap<ByteId, ByteStatistics> statistics = new HashMap<>();
 
-        Set<Integer> SIDs = new HashSet<>();
+        final Set<Integer> SIDs = new HashSet<>();
     }
 
     public static TraceReport process(String reportDestinationFolder, String simpleFileName, List<CANPacket> packets) throws IOException {
-
-        PerSidDump.handle(packets, simpleFileName);
-
-        CounterScanner.scanForCounters(packets);
-
         TraceFileMetaIndex traceFileMetaIndex = calculateByteStatistics(packets);
 
         writeByteReport(reportDestinationFolder, simpleFileName, traceFileMetaIndex);

diff --git a/src/main/java/com/rusefi/can/analysis/ByteRateOfChangeReports.java b/src/main/java/com/rusefi/can/analysis/ByteRateOfChangeReports.java
@@ -4,47 +4,52 @@
 import com.rusefi.can.reader.CANLineReader;
 import com.rusefi.util.FolderUtil;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.io.*;
+import java.util.*;
 
 public class ByteRateOfChangeReports {
     /**
      * sweet baby O(n^2)
      */
-    public static void compareEachReportAgainstAllOthers(List<ByteRateOfChange.TraceReport> reports) {
+    public static void compareEachReportAgainstAllOthers(String reportDestinationFolder, List<ByteRateOfChange.TraceReport> reports) throws FileNotFoundException {
         for (int i = 0; i < reports.size(); i++) {
             for (int j = i + 1; j < reports.size(); j++)
-                compareTwoReports(reports.get(i), reports.get(j));
+                compareTwoReports(reportDestinationFolder, reports.get(i), reports.get(j));
         }
     }
 
-    private static void compareTwoReports(ByteRateOfChange.TraceReport traceReport1, ByteRateOfChange.TraceReport traceReport2) {
-        Set<ByteRateOfChange.ByteId> allKeys = new HashSet<>();
+    private static void compareTwoReports(String reportDestinationFolder, ByteRateOfChange.TraceReport traceReport1, ByteRateOfChange.TraceReport traceReport2) throws FileNotFoundException {
+        Set<ByteRateOfChange.ByteId> allKeys = new TreeSet<>();
         allKeys.addAll(traceReport1.getStatistics().keySet());
         allKeys.addAll(traceReport2.getStatistics().keySet());
 
-        PrintStream report = System.out;
+        String comparingFolder = reportDestinationFolder + File.separator + "comparison";
+        new File(comparingFolder).mkdirs();
+
+        String outputFileName = comparingFolder + File.separator + traceReport1.getSimpleFileName() + "-vs-" + traceReport2.getSimpleFileName() + ".txt";
+        PrintWriter report = new PrintWriter(new FileOutputStream(outputFileName));
 
         report.println("Comparing unique value count per byte " + traceReport1.getSummary() + " and " + traceReport2.getSummary());
 
         List<ByteVariationDifference> differences = new ArrayList<>();
 
+        report.println("******************** Sorted by key ********************");
+
+
         for (ByteRateOfChange.ByteId id : allKeys) {
             ByteRateOfChange.ByteStatistics s1 = traceReport1.getStatistics().computeIfAbsent(id, ByteRateOfChange.ByteStatistics::new);
             ByteRateOfChange.ByteStatistics s2 = traceReport2.getStatistics().computeIfAbsent(id, ByteRateOfChange.ByteStatistics::new);
 
             if (s1.getUniqueValues() != s2.getUniqueValues()) {
                 String msg = id + ": " + s1.getUniqueValues() + " vs " + s2.getUniqueValues();
-                differences.add(new ByteVariationDifference(Math.abs(s1.getUniqueValues() - s2.getUniqueValues()), msg));
+                int deltaCount = Math.abs(s1.getUniqueValues() - s2.getUniqueValues());
+                differences.add(new ByteVariationDifference(deltaCount, msg));
+                report.println(msg + " delta=" + deltaCount);
             }
         }
-        differences.sort((o1, o2) -> o2.deltaCount - o1.deltaCount);
 
+        report.println("******************** Sorted by delta count ********************");
+        differences.sort((o1, o2) -> o2.deltaCount - o1.deltaCount);
         for (ByteVariationDifference difference : differences)
             report.println(difference.msg);
 
@@ -53,7 +58,9 @@ private static void compareTwoReports(ByteRateOfChange.TraceReport traceReport1,
         report.println();
         report.println();
         report.println();
+        report.close();
     }
+
     public static String createOutputFolder(String inputFolderName) {
         String reportDestinationFolder = inputFolderName + File.separator + "processed";
         new File(reportDestinationFolder).mkdirs();
@@ -69,11 +76,16 @@ public static void scanInputFolder(String inputFolderName, String fileNameSuffix
 
             List<CANPacket> logFileContent = CANLineReader.getReader().readFile(fullInputFileName);
 
+            PerSidDump.handle(reportDestinationFolder, simpleFileName, logFileContent);
+            CounterScanner.scanForCounters(reportDestinationFolder, logFileContent);
+
             ByteRateOfChange.TraceReport report = ByteRateOfChange.process(reportDestinationFolder, simpleFileName, logFileContent);
             reports.add(report);
         }, fileNameSuffix);
+
+
         System.out.println("Processing " + reports.size() + " report(s)");
-        compareEachReportAgainstAllOthers(reports);
+        compareEachReportAgainstAllOthers(reportDestinationFolder, reports);
     }
 
     static class ByteVariationDifference {

diff --git a/src/main/java/com/rusefi/can/analysis/CounterAggregator.java b/src/main/java/com/rusefi/can/analysis/CounterAggregator.java
@@ -43,8 +43,6 @@ static class ScanState {
 
         List<CounterWithWidth> counters = new ArrayList<>();
 
-        StringBuffer sb = new StringBuffer();
-
         CounterScanner.BitStateKey start;
         int totalNumberOfBits;
 

diff --git a/src/main/java/com/rusefi/can/analysis/CounterScanner.java b/src/main/java/com/rusefi/can/analysis/CounterScanner.java
@@ -2,12 +2,14 @@
 
 import com.rusefi.can.CANPacket;
 
+import java.io.*;
 import java.util.*;
 
-import static com.rusefi.can.analysis.ByteRateOfChange.dualSid;
-
 public class CounterScanner {
-    public static void scanForCounters(List<CANPacket> packets) {
+    public static void scanForCounters(String reportDestinationFolder, List<CANPacket> packets) throws FileNotFoundException {
+
+        String outputFileName = reportDestinationFolder + File.separator + "counter_report.txt";
+        PrintWriter pw = new PrintWriter(new FileOutputStream(outputFileName));
 
         Map<BitStateKey, BitState> bitStates = new TreeMap<>();
 
@@ -31,24 +33,25 @@ public static void scanForCounters(List<CANPacket> packets) {
 
 
         LinkedHashMap<BitStateKey, Integer> counters = new LinkedHashMap<>();
-
         for (Map.Entry<BitStateKey, BitState> e : bitStates.entrySet()) {
 
             BitState bitState = e.getValue();
             if (bitState.couldBeCounter()) {
                 BitStateKey key = e.getKey();
-                System.out.println("Looks like counter " + key + " " + bitState.cycleLength);
+                pw.println("Working: Looks like counter key=" + key + " cycleLength=" + bitState.cycleLength);
 
                 counters.put(key, bitState.cycleLength);
-
             }
         }
 
+        pw.println("Scanning...");
         List<CounterAggregator.CounterWithWidth> countersWithWidth = CounterAggregator.scan(counters);
 
+        pw.println("Here are the founding:");
         for (CounterAggregator.CounterWithWidth counterWithWidth : countersWithWidth) {
-            System.out.println("Found " + counterWithWidth);
+            pw.println("Found " + counterWithWidth);
         }
+        pw.close();
     }
 
     static class BitStateKey implements Comparable {

diff --git a/src/main/java/com/rusefi/can/analysis/PerSidDump.java b/src/main/java/com/rusefi/can/analysis/PerSidDump.java
@@ -3,6 +3,7 @@
 import com.rusefi.can.CANPacket;
 import com.rusefi.can.writer.SteveWriter;
 
+import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintWriter;
@@ -11,8 +12,14 @@
 
 import static com.rusefi.can.analysis.ByteRateOfChange.dualSid;
 
+/**
+ * Write a separate file for each unique packet ID
+ */
 public class PerSidDump {
-    public static void handle(List<CANPacket> packets, String simpleFileName) throws IOException {
+    public static void handle(String reportDestinationFolder, String simpleFileName, List<CANPacket> packets) throws IOException {
+        String filteredDestinationFolder = reportDestinationFolder + File.separator + "filtered";
+        new File(filteredDestinationFolder).mkdirs();
+
         TreeSet<Integer> sids = new TreeSet<>();
         // todo: one day I will let streams into my heart
         for (CANPacket packet : packets)
@@ -21,13 +28,14 @@ public static void handle(List<CANPacket> packets, String simpleFileName) throws
         // O(n*M) is not so bad
         for (int sid : sids) {
 
-            PrintWriter pw = new PrintWriter(new FileOutputStream(simpleFileName + "_filtered_" + dualSid(sid, "_") + ".txt"));
+            String outputFileName = filteredDestinationFolder + File.separator + simpleFileName + "_filtered_" + dualSid(sid, "_") + ".txt";
+            PrintWriter pw = new PrintWriter(new FileOutputStream(outputFileName));
 
             for (CANPacket packet : packets) {
                 if (packet.getId() != sid)
                     continue;
 
-                // no reason to use SteveWriter just need any human-readable writer here
+                // no specific reason to use SteveWriter just need any human-readable writer here
                 SteveWriter.append(pw, packet);
 
 

diff --git a/src/main/java/com/rusefi/can/reader/CANLineReader.java b/src/main/java/com/rusefi/can/reader/CANLineReader.java
@@ -33,7 +33,7 @@ static CANLineReader getReader() {
                 return CanHackerReader.INSTANCE;
             case PCAN2:
                 return PcanTrcReader2_0.INSTANCE;
-            case PCAN:
+            case PCAN1_1:
             default:
                 return new PcanTrcReader1_1();
         }

diff --git a/src/main/java/com/rusefi/can/reader/ReaderType.java b/src/main/java/com/rusefi/can/reader/ReaderType.java
@@ -1,7 +1,7 @@
 package com.rusefi.can.reader;
 
 public enum ReaderType {
-    PCAN,
+    PCAN1_1,
     PCAN2,
     CANOE,
     CANHACKER,

diff --git a/src/test/java/com/rusefi/can/ByteRateOfChangeNissanSandbox.java b/src/test/java/com/rusefi/can/ByteRateOfChangeNissanSandbox.java
@@ -8,7 +8,7 @@
 
 public class ByteRateOfChangeNissanSandbox {
     public static void main(String[] args) throws IOException {
-        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
+        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN1_1;
 
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Nissan\\2011_Xterra\\CAN-Nov-2022";
 

diff --git a/src/test/java/com/rusefi/can/ByteRateOfChangeVagSandbox.java b/src/test/java/com/rusefi/can/ByteRateOfChangeVagSandbox.java
@@ -8,7 +8,7 @@
 
 public class ByteRateOfChangeVagSandbox {
     public static void main(String[] args) throws IOException {
-        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN;
+        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN1_1;
 
         String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\VAG\\2006-Passat-B6";
 

diff --git a/src/test/java/com/rusefi/can/KiaSandbox.java b/src/test/java/com/rusefi/can/KiaSandbox.java
@@ -0,0 +1,17 @@
+package com.rusefi.can;
+
+import com.rusefi.can.analysis.ByteRateOfChangeReports;
+import com.rusefi.can.reader.ReaderType;
+import com.rusefi.can.reader.ReaderTypeHolder;
+
+import java.io.IOException;
+
+public class KiaSandbox {
+    public static void main(String[] args) throws IOException {
+        ReaderTypeHolder.INSTANCE.type = ReaderType.PCAN2;
+
+        String inputFolderName = "C:\\stuff\\rusefi_documentation\\OEM-Docs\\Kia\\2013-CAN-logs";
+
+        ByteRateOfChangeReports.scanInputFolder(inputFolderName, ".trc");
+    }
+}
diff --git a/...ava/com/rusefi/can/BmwE65DecoderTest.java → ...efi/can/deprecated/BmwE65DecoderTest.java b/...ava/com/rusefi/can/BmwE65DecoderTest.java → ...efi/can/deprecated/BmwE65DecoderTest.java
@@ -1,5 +1,7 @@
-package com.rusefi.can;
+package com.rusefi.can.deprecated;
 
+import com.rusefi.can.CANPacket;
+import com.rusefi.can.SensorValue;
 import com.rusefi.can.deprecated.PacketPayload;
 import com.rusefi.can.deprecated.SensorType;
 import com.rusefi.can.deprecated.decoders.bmw.Bmw0A9;