analysis: add a dedupe pass (#797)

See #796.

src/analysis/pass/dedupe/dedupe

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# dedupe: filter out any cohorts whose inputs have already appeared at least once.
+
+require "json"
+require "set"
+
+STDERR.puts "[+] pass: dedupe"
+
+count = 0
+seen = Set.new
+STDIN.each_line do |line|
+  result = JSON.parse line, symbolize_names: true
+
+  # add? returns nil if the element is already present, saving us
+  # two separate operations (check + add).
+  if seen.add?(result[:input]).nil?
+    count += 1
+    next
+  end
+
+  STDOUT.puts result.to_json
+end
+
+STDERR.puts "[+] pass: dedupe done: #{count} filtered"

src/analysis/pass/dedupe/spec.yml

@@ -0,0 +1,3 @@
+name: dedupe
+desc: Filter out any duplicate cohorts (by input)
+run: dedupe

src/analysis/pass/minimize-input/minimize-input

@@ -15,6 +15,7 @@ STDIN.each_line do |line|
 
   # If the maximum ndecoded is 0, then all are 0 and we should skip
   # this cohort entirely.
+  # In effect, this is probably identical to filter-all-failure.
   if max_ndecoded.zero?
     count += 1
     next

src/analysis/passes.yml

@@ -1,4 +1,5 @@
 default:
+  - dedupe
   - filter-all-failure
   - filter-ndecoded
   - minimize-input
@@ -15,19 +16,22 @@ same-size-different-decodings:
 
 # Find inputs that not all workers either succeed or fail on.
 status-discrepancies:
+  - dedupe
   - filter-all-failure
   - filter-all-success
   - minimize-input
   - normalize
 
 destroy-capstone:
+  - dedupe
   - filter-all-success
   - filter-ndecoded
   - filter-destroy-capstone
   - minimize-input
   - normalize
 
 destroy-bddisasm:
+  - dedupe
   - filter-all-success
   - filter-ndecoded
   - filter-destroy-bddisasm