accounting for context

src/main/java/com/rusefi/can/analysis/ByteRateOfChange.java

@@ -45,7 +45,7 @@ private static void writeByteReport(List<ByteStatistics> allStats, String fileNa
 
         for (ByteStatistics byteStatistics : allStats) {
             ByteId key = byteStatistics.key;
-            ps.println(dualSid(key.sid) + " byte " + key.bitIndex + " has " + byteStatistics.getUniqueValuesCount() + " unique value(s)");
+            ps.println(dualSid(key.sid) + " byte " + key.getByteIndex() + " has " + byteStatistics.getUniqueValuesCount() + " unique value(s)");
         }
 
         ps.close();
@@ -105,36 +105,40 @@ public String toString() {
 
     public static class ByteId implements Comparable<ByteId> {
         final int sid;
-        final int bitIndex;
+        final int byteIndex;
 
-        public ByteId(int sid, int bitIndex) {
+        public ByteId(int sid, int byteIndex) {
             this.sid = sid;
-            this.bitIndex = bitIndex;
+            this.byteIndex = byteIndex;
         }
 
         public String getLogKey() {
-            return dualSid(sid) + "_byte_" + bitIndex + "_bit_" + (bitIndex * 8);
+            return dualSid(sid) + "_byte_" + byteIndex + "_bit_" + (byteIndex * 8);
         }
 
         @Override
         public boolean equals(Object o) {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             ByteId byteId = (ByteId) o;
-            return sid == byteId.sid && bitIndex == byteId.bitIndex;
+            return sid == byteId.sid && byteIndex == byteId.byteIndex;
+        }
+
+        public int getByteIndex() {
+            return byteIndex;
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(sid, bitIndex);
+            return Objects.hash(sid, byteIndex);
         }
 
         @Override
         public int compareTo(ByteId o) {
             ByteId other = o;
             if (other.sid != sid)
                 return sid - other.sid;
-            return bitIndex - other.bitIndex;
+            return byteIndex - other.byteIndex;
         }
 
         @Override

src/main/java/com/rusefi/can/analysis/ByteRateOfChangeReports.java

@@ -11,14 +11,14 @@ public class ByteRateOfChangeReports {
     /**
      * sweet baby O(n^2)
      */
-    public static void compareEachReportAgainstAllOthers(String reportDestinationFolder, List<ByteRateOfChange.TraceReport> reports) throws FileNotFoundException {
+    public static void compareEachReportAgainstAllOthers(String reportDestinationFolder, List<ByteRateOfChange.TraceReport> reports, CanContext context) throws FileNotFoundException {
         for (int i = 0; i < reports.size(); i++) {
             for (int j = i + 1; j < reports.size(); j++)
-                compareTwoReports(reportDestinationFolder, reports.get(i), reports.get(j));
+                compareTwoReports(reportDestinationFolder, reports.get(i), reports.get(j), context);
         }
     }
 
-    private static void compareTwoReports(String reportDestinationFolder, ByteRateOfChange.TraceReport traceReport1, ByteRateOfChange.TraceReport traceReport2) throws FileNotFoundException {
+    private static void compareTwoReports(String reportDestinationFolder, ByteRateOfChange.TraceReport traceReport1, ByteRateOfChange.TraceReport traceReport2, CanContext context) throws FileNotFoundException {
         Set<ByteRateOfChange.ByteId> allKeys = new TreeSet<>();
         allKeys.addAll(traceReport1.getStatistics().keySet());
         allKeys.addAll(traceReport2.getStatistics().keySet());
@@ -37,6 +37,15 @@ private static void compareTwoReports(String reportDestinationFolder, ByteRateOf
 
 
         for (ByteRateOfChange.ByteId id : allKeys) {
+            if (context.counterBytes.contains(id)) {
+                // skipping byte with a known counter
+                continue;
+            }
+            if (id.getByteIndex() == 7 && context.withChecksum.contains(id.sid)) {
+                // skipping known checksum byte
+                continue;
+            }
+
             ByteRateOfChange.ByteStatistics s1 = traceReport1.getStatistics().computeIfAbsent(id, ByteRateOfChange.ByteStatistics::new);
             ByteRateOfChange.ByteStatistics s2 = traceReport2.getStatistics().computeIfAbsent(id, ByteRateOfChange.ByteStatistics::new);
 
@@ -70,14 +79,18 @@ public static String createOutputFolder(String inputFolderName) {
     public static void scanInputFolder(String inputFolderName, String fileNameSuffix) throws IOException {
         String reportDestinationFolder = createOutputFolder(inputFolderName);
 
+        CanContext context = CanContext.read(inputFolderName);
+
         List<ByteRateOfChange.TraceReport> reports = new ArrayList<>();
 
         FolderUtil.handleFolder(inputFolderName, (simpleFileName, fullInputFileName) -> {
 
             List<CANPacket> logFileContent = CANLineReader.getReader().readFile(fullInputFileName);
 
             PerSidDump.handle(reportDestinationFolder, simpleFileName, logFileContent);
+            // at the moment we overwrite counter detection report after we process each file
             CounterScanner.scanForCounters(reportDestinationFolder, logFileContent);
+            ChecksumScanner.scanForChecksums(reportDestinationFolder, logFileContent);
 
             CanToMegaLogViewer.createMegaLogViewer(reportDestinationFolder, logFileContent, simpleFileName);
 
@@ -87,7 +100,7 @@ public static void scanInputFolder(String inputFolderName, String fileNameSuffix
 
 
         System.out.println("Processing " + reports.size() + " report(s)");
-        compareEachReportAgainstAllOthers(reportDestinationFolder, reports);
+        compareEachReportAgainstAllOthers(reportDestinationFolder, reports, context);
     }
 
     static class ByteVariationDifference {

src/main/java/com/rusefi/can/analysis/CanContext.java

@@ -0,0 +1,48 @@
+package com.rusefi.can.analysis;
+
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class CanContext {
+    final Set<Integer> withChecksum = new HashSet<>();
+    final Map<Integer, Map<Integer, Integer>> countersMap;
+
+    Set<ByteRateOfChange.ByteId> counterBytes = new HashSet<>();
+
+    private CanContext(List<Integer> withChecksum, Map<Integer, Map<Integer, Integer>> countersMap) {
+        this.countersMap = countersMap;
+        this.withChecksum.addAll(withChecksum);
+
+        for (Map.Entry<Integer, Map<Integer, Integer>> e : countersMap.entrySet()) {
+            int sid = e.getKey();
+            Map<Integer, Integer> v = e.getValue();
+
+            for (Map.Entry<Integer, Integer> e2 : v.entrySet()) {
+                int bitIndex = e2.getKey();
+                int size = e2.getValue();
+
+                if (size > 4) {
+                    int byteIndex = bitIndex / 8;
+                    counterBytes.add(new ByteRateOfChange.ByteId(sid, byteIndex));
+                }
+            }
+        }
+    }
+
+    public static CanContext read(String inputFolderName) throws FileNotFoundException {
+        Yaml checksum = new Yaml();
+        List<Integer> withChecksum = checksum.load(new FileReader(inputFolderName + File.separator + ChecksumScanner.CHECKSUM_YAML));
+
+        Yaml countersYaml = new Yaml();
+        Map<Integer, Map<Integer, Integer>> countersMap = countersYaml.load(new FileReader(inputFolderName + File.separator + CounterScanner.COUNTERS_YAML));
+
+        return new CanContext(withChecksum, countersMap);
+    }
+}

src/main/java/com/rusefi/can/analysis/ChecksumScanner.java

@@ -0,0 +1,49 @@
+package com.rusefi.can.analysis;
+
+import com.rusefi.can.CANPacket;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ChecksumScanner {
+
+    public static final String CHECKSUM_YAML = "checksum.yaml";
+
+    public static void scanForChecksums(String reportDestinationFolder, List<CANPacket> packets) throws IOException {
+        Map<Integer, AtomicBoolean> isChecksumMap = new HashMap<>();
+
+        J1850_SAE_crc8_Calculator c = new J1850_SAE_crc8_Calculator();
+
+        for (CANPacket packet : packets) {
+            AtomicBoolean isChecksum = isChecksumMap.computeIfAbsent(packet.getId(), integer -> new AtomicBoolean(true));
+            if (!isChecksum.get())
+                continue;
+            byte[] data = packet.getData();
+            if (data.length != 8) {
+                isChecksum.set(false);
+                continue;
+            }
+            byte checksum = c.crc8(data, 7);
+            isChecksum.set(data[7] == checksum);
+        }
+
+
+        List<Integer> withChecksum = new ArrayList<>();
+
+        for (Map.Entry<Integer, AtomicBoolean> e : isChecksumMap.entrySet()) {
+            if (e.getValue().get()) {
+                Integer sid = e.getKey();
+                System.out.println("Ends with checksum " + sid);
+                withChecksum.add(sid);
+            }
+        }
+        Yaml yaml = new Yaml();
+        String yamlCountersReportFileName = reportDestinationFolder + File.separator + CHECKSUM_YAML;
+        System.out.println("Writing report to " + yamlCountersReportFileName);
+        yaml.dump(withChecksum, new FileWriter(yamlCountersReportFileName));
+    }
+}

src/main/java/com/rusefi/can/analysis/CounterScanner.java

@@ -7,6 +7,9 @@
 import java.util.*;
 
 public class CounterScanner {
+
+    public static final String COUNTERS_YAML = "counters.yaml";
+
     public static void scanForCounters(String reportDestinationFolder, List<CANPacket> packets) throws IOException {
 
         String outputFileName = reportDestinationFolder + File.separator + "counter_report.txt";
@@ -59,7 +62,7 @@ public static void scanForCounters(String reportDestinationFolder, List<CANPacke
 
             lengthByStartIndex.put(counterWithWidth.getStart().getTotalBitIndex(), counterWithWidth.getTotalNumberOfBits());
         }
-        String yamlCountersReportFileName = reportDestinationFolder + File.separator + "counters.yaml";
+        String yamlCountersReportFileName = reportDestinationFolder + File.separator + COUNTERS_YAML;
         System.out.println("Writing report to " + yamlCountersReportFileName);
         yaml.dump(map, new FileWriter(yamlCountersReportFileName));
         pw.close();
@@ -75,15 +78,15 @@ public BitStateKey(int sid, int byteIndex, int bitIndex) {
         }
 
         public int getTotalBitIndex() {
-            return byteId.bitIndex * 8 + bitIndex;
+            return byteId.byteIndex * 8 + bitIndex;
         }
 
         public int getSid() {
             return byteId.sid;
         }
 
         public int getByteIndex() {
-            return byteId.bitIndex;
+            return byteId.byteIndex;
         }
 
         public int getBitIndex() {