feat(coinjoin): improve speed by limiting denoms (#226)

* fix(coinjoin): improve detection of used collateral tx for fees

* fix(coinjoin): add date/time to coinjoin-report filename

* fix(coinjoin): add customization of mixing denominations to client options

* fix(wallettool): add checks for empty wallet when mixing

* fix(coinjoin): track more timeout causes for CoinJoinReporter

* fix(coinjoin): track collateral transactions (fees)

* test(coinjoin): Add context to CoinJoinTest since collateral tests need it

* chore: update WalletTool CJ Mix run config to 8 sessions with multi-session

* fix(coinjoin): make standard denominations immutable

* fix(coinjoin): make standard denominations immutable

* fix(coinjoin): do not label a collateral tx as SELF
  * The TX will be ignored by Peer.processTransaction.

.run/WalletTool CJ Mix.run.xml

@@ -1,8 +1,8 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="WalletTool CJ Mix" type="Application" factoryName="Application">
     <option name="MAIN_CLASS_NAME" value="org.bitcoinj.tools.WalletTool" />
-    <module name="dashj-master-three.tools.main" />
-    <option name="PROGRAM_PARAMETERS" value="mix --wallet=coinjoin.testnet.wallet --net=TEST --debuglog --amount=0.007 --sessions=4 --rounds=1" />
+    <module name="dashj-feature-coinjoin.tools.main" />
+    <option name="PROGRAM_PARAMETERS" value="mix --wallet=coinjoin.testnet.wallet --net=TEST --debuglog --amount=0.15 --sessions=8 --rounds=1 --multi-session" />
     <option name="VM_PARAMETERS" value="-Djava.library.path=contrib/dashj-bls/bls/target/cmake" />
     <extension name="coverage">
       <pattern>

core/src/main/java/org/bitcoinj/coinjoin/CoinJoin.java

@@ -32,6 +32,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -42,12 +43,16 @@
 
 public class CoinJoin {
     private static final Logger log = LoggerFactory.getLogger(CoinJoin.class);
-    private static final List<Coin> standardDenominations = Lists.newArrayList(
-            Coin.COIN.multiply(10).add(Coin.valueOf(10000)),
-            Coin.COIN.add(Coin.valueOf(1000)),
-            Coin.COIN.div(10).add(Coin.valueOf(100)),
-            Coin.COIN.div(100).add(Coin.valueOf(10)),
-            Coin.COIN.div(1000).add(Coin.valueOf(1))
+    // this list of standard denominations cannot be modified by DashJ and must remain the same as
+    // CoinJoin::vecStandardDenominations in coinjoin.cpp
+    private static final List<Coin> standardDenominations = Collections.unmodifiableList(
+            Lists.newArrayList(
+                Coin.COIN.multiply(10).add(Coin.valueOf(10000)),
+                Coin.COIN.add(Coin.valueOf(1000)),
+                Coin.COIN.div(10).add(Coin.valueOf(100)),
+                Coin.COIN.div(100).add(Coin.valueOf(10)),
+                Coin.COIN.div(1000).add(Coin.valueOf(1))
+            )
     );
 
     private static final HashMap<Sha256Hash, CoinJoinBroadcastTx> mapDSTX = new HashMap<>();
@@ -185,12 +190,14 @@ public static boolean isCollateralValid(Transaction txCollateral, boolean checkI
         }
 
         // the collateral tx must not have been seen on the network
-        boolean hasBeenSeen = txCollateral.getConfidence().getConfidenceType() == TransactionConfidence.ConfidenceType.UNKNOWN;
+        TransactionConfidence confidence = txCollateral.getConfidence();
+        boolean hasBeenSeen = confidence.getConfidenceType() != TransactionConfidence.ConfidenceType.UNKNOWN ||
+                confidence.numBroadcastPeers() > 0 || confidence.getIXType() != TransactionConfidence.IXType.IX_NONE;
         if (hasBeenSeen) {
             log.info("coinjoin: collateral has been spent, need to recreate txCollateral={}", txCollateral.getTxId());
         }
         log.info("coinjoin: collateral: {}", txCollateral); /* Continued */
-        return hasBeenSeen;
+        return !hasBeenSeen;
     }
     public static Coin getCollateralAmount() { return getSmallestDenomination().div(10); }
     public static Coin getMaxCollateralAmount() { return getCollateralAmount().multiply(4); }

core/src/main/java/org/bitcoinj/coinjoin/CoinJoinClientManager.java

@@ -23,13 +23,15 @@
 import org.bitcoinj.coinjoin.listeners.SessionCompleteListener;
 import org.bitcoinj.coinjoin.listeners.SessionStartedListener;
 import org.bitcoinj.core.AbstractBlockChain;
+import org.bitcoinj.core.Coin;
 import org.bitcoinj.core.Context;
 import org.bitcoinj.core.MasternodeAddress;
 import org.bitcoinj.core.MasternodeSync;
 import org.bitcoinj.core.Message;
 import org.bitcoinj.core.Peer;
 import org.bitcoinj.core.Sha256Hash;
 import org.bitcoinj.core.StoredBlock;
+import org.bitcoinj.core.Transaction;
 import org.bitcoinj.core.VerificationException;
 import org.bitcoinj.core.listeners.NewBestBlockListener;
 import org.bitcoinj.evolution.Masternode;
@@ -39,6 +41,7 @@
 import org.bitcoinj.utils.Threading;
 import org.bitcoinj.wallet.Wallet;
 import org.bitcoinj.wallet.WalletEx;
+import org.bitcoinj.wallet.listeners.WalletCoinsReceivedEventListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,7 +64,7 @@
 import static org.bitcoinj.coinjoin.CoinJoinConstants.COINJOIN_AUTO_TIMEOUT_MAX;
 import static org.bitcoinj.coinjoin.CoinJoinConstants.COINJOIN_AUTO_TIMEOUT_MIN;
 
-public class CoinJoinClientManager {
+public class CoinJoinClientManager implements WalletCoinsReceivedEventListener {
     private static final Logger log = LoggerFactory.getLogger(CoinJoinClientManager.class);
     private static final Random random = new Random();
     // Keep track of the used Masternodes
@@ -122,11 +125,13 @@ public CoinJoinClientManager(Wallet wallet) {
         checkArgument(wallet instanceof WalletEx);
         mixingWallet = (WalletEx) wallet;
         context = wallet.getContext();
+        mixingWallet.addCoinsReceivedEventListener(this);
     }
 
     public CoinJoinClientManager(WalletEx wallet) {
         mixingWallet = wallet;
         context = wallet.getContext();
+        mixingWallet.addCoinsReceivedEventListener(this);
     }
 
     public void processMessage(Peer from, Message message, boolean enable_bip61) {
@@ -409,6 +414,7 @@ public void setBlockChain(AbstractBlockChain blockChain) {
 
     public void close(AbstractBlockChain blockChain) {
         blockChain.removeNewBestBlockListener(newBestBlockListener);
+        mixingWallet.removeCoinsReceivedEventListener(this);
     }
 
     public void updatedSuccessBlock() {
@@ -650,4 +656,13 @@ public EnumSet<PoolStatus> getContinueMixingOnStatus() {
     public void addContinueMixingOnError(PoolStatus error) {
         continueMixingOnStatus.add(error);
     }
+
+    public void processTransaction(Transaction tx) {
+        deqSessions.forEach(coinJoinClientSession -> coinJoinClientSession.processTransaction(tx));
+    }
+
+    @Override
+    public void onCoinsReceived(Wallet wallet, Transaction tx, Coin prevBalance, Coin newBalance) {
+        processTransaction(tx);
+    }
 }

core/src/main/java/org/bitcoinj/coinjoin/CoinJoinClientOptions.java

@@ -18,6 +18,8 @@
 
 import org.bitcoinj.core.Coin;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -58,6 +60,12 @@ public static void setAmount(Coin amount) {
 
     public static boolean isEnabled() { return CoinJoinClientOptions.get().enableCoinJoin.get(); }
     public static boolean isMultiSessionEnabled() { return CoinJoinClientOptions.get().isCoinJoinMultiSession.get(); }
+
+    public static List<Coin> getDenominations() { return CoinJoinClientOptions.get().allowedDenominations.get(); }
+
+    public static boolean removeDenomination(Coin amount) { return CoinJoinClientOptions.get().allowedDenominations.get().remove(amount); }
+
+    public static void resetDenominations() { CoinJoinClientOptions.get().allowedDenominations.set(CoinJoin.getStandardDenominations()); }
     private static CoinJoinClientOptions instance;
     private static boolean onceFlag;
 
@@ -69,6 +77,7 @@ public static void setAmount(Coin amount) {
     private final AtomicInteger coinJoinDenomsHardCap = new AtomicInteger(DEFAULT_COINJOIN_DENOMS_HARDCAP);
     private final AtomicBoolean enableCoinJoin = new AtomicBoolean(false);
     private final AtomicBoolean isCoinJoinMultiSession = new AtomicBoolean(DEFAULT_COINJOIN_MULTISESSION);
+    private final AtomicReference<List<Coin>> allowedDenominations = new AtomicReference<>(new ArrayList<>(CoinJoin.getStandardDenominations()));
 
     private CoinJoinClientOptions() {
         coinJoinSessions.set(DEFAULT_COINJOIN_SESSIONS);
@@ -96,4 +105,11 @@ public static void reset() {
     private static void init() {
         instance = new CoinJoinClientOptions();
     }
+
+    public static String getString() {
+        return "amount: " + getAmount() + "\n" +
+                "rounds: " + getRounds() + "\n" +
+                "multi-session: " + isMultiSessionEnabled() + "\n" +
+                "denoms: " + getDenominations().size() + "\n";
+    }
 }

core/src/main/java/org/bitcoinj/coinjoin/CoinJoinClientSession.java

@@ -33,7 +33,9 @@
 import org.bitcoinj.core.Message;
 import org.bitcoinj.core.NetworkParameters;
 import org.bitcoinj.core.Peer;
+import org.bitcoinj.core.Sha256Hash;
 import org.bitcoinj.core.Transaction;
+import org.bitcoinj.core.TransactionConfidence;
 import org.bitcoinj.core.TransactionDestination;
 import org.bitcoinj.core.TransactionInput;
 import org.bitcoinj.core.TransactionOutPoint;
@@ -110,6 +112,7 @@ public class CoinJoinClientSession extends CoinJoinBaseSession {
 
     private final AtomicBoolean hasNothingToDo = new AtomicBoolean(false); // is mixing finished?
 
+    private final HashMap<Sha256Hash, Integer> collateralSessionMap = new HashMap<>();
     private final CopyOnWriteArrayList<ListenerRegistration<SessionStartedListener>> sessionStartedListeners
             = new CopyOnWriteArrayList<>();
     private final CopyOnWriteArrayList<ListenerRegistration<SessionCompleteListener>> sessionCompleteListeners
@@ -155,6 +158,13 @@ public int compare(CompactTallyItem o, CompactTallyItem t1) {
         return false;
     }
 
+    public void processTransaction(Transaction tx) {
+        log.info("processing tx of {}: {}", tx.getValue(mixingWallet), tx.getTxId());
+        if (collateralSessionMap.containsKey(tx.getTxId())) {
+            queueTransactionListeners(tx, collateralSessionMap.get(tx.getTxId()), CoinJoinTransactionType.MIXING_FEE);
+        }
+    }
+
     static class Result<T> {
         private T result;
 
@@ -203,7 +213,7 @@ private boolean createDenominated(Coin balanceToDenominate, CompactTallyItem tal
         // ****** Add outputs for denoms ************ /
 
         final Result<Boolean> addFinal = new Result<>(true);
-        List<Coin> denoms = CoinJoin.getStandardDenominations();
+        List<Coin> denoms = CoinJoinClientOptions.getDenominations();
 
         HashMap<Coin, Integer> mapDenomCount = new HashMap<>();
         for (Coin denomValue : denoms) {
@@ -368,7 +378,7 @@ public int process(Coin amount) {
 
         log.info("coinjoin: txid: {}", strResult);
 
-        queueCreateDenominationListeners(txBuilder.getTransaction(), CoinJoinTransactionType.CREATE_DENOMINATION);
+        queueTransactionListeners(txBuilder.getTransaction(), CoinJoinTransactionType.CREATE_DENOMINATION);
 
         return true;
     }
@@ -499,6 +509,7 @@ private boolean makeCollateralAmounts(CompactTallyItem tallyItem, boolean fTryDe
         mixingWallet.getContext().coinJoinManager.coinJoinClientManagers.get(mixingWallet.getDescription()).updatedSuccessBlock();
 
         log.info("coinjoin: txid: {}", strResult);
+        queueTransactionListeners(txBuilder.getTransaction(), CoinJoinTransactionType.CREATE_COLLATERAL);
 
         return true;
     }
@@ -541,6 +552,9 @@ private boolean createCollateralTransaction(Transaction txCollateral, StringBuil
         }
 
         try {
+            // don't set the source to TransactionConfidence.Source.SELF on these collateral transactions
+            // otherwise when the masternodes do submit these transactions to the network, they will be
+            // ignored by DashJ and when they are received
             SendRequest req = SendRequest.forTx(txCollateral);
             req.aesKey = context.coinJoinManager.requestKeyParameter(mixingWallet);
             mixingWallet.signTransaction(req);
@@ -549,6 +563,9 @@ private boolean createCollateralTransaction(Transaction txCollateral, StringBuil
             return false;
         }
         isMyCollateralValid = true;
+        if (!collateralSessionMap.containsKey(txCollateral.getTxId())) {
+            collateralSessionMap.put(txCollateral.getTxId(), 0);
+        }
         return true;
     }
 
@@ -877,6 +894,7 @@ private void processPoolStateUpdate(Peer peer, CoinJoinStatusUpdate statusUpdate
                 if (state.get() == statusUpdate.getState() && statusUpdate.getState() == POOL_STATE_QUEUE && sessionID.get() == 0 && statusUpdate.getSessionID() != 0) {
                     // new session id should be set only in POOL_STATE_QUEUE state
                     sessionID.set(statusUpdate.getSessionID());
+                    collateralSessionMap.put(txMyCollateral.getTxId(), statusUpdate.getSessionID());
                     timeLastSuccessfulStep.set(Utils.currentTimeSeconds());
                     strMessageTmp = strMessageTmp + " Set nSessionID to " + sessionID.get();
                     queueSessionStartedListeners(MSG_SUCCESS);
@@ -1309,7 +1327,7 @@ public boolean doAutomaticDenominating(boolean fDryRun) {
 
             // adjust balanceNeedsAnonymized to consume final denom
             if (nBalanceDenominated.subtract(nBalanceAnonymized).isGreaterThan(balanceNeedsAnonymized)) {
-                List<Coin> denoms = CoinJoin.getStandardDenominations();
+                List<Coin> denoms = CoinJoinClientOptions.getDenominations();
                 Coin nAdditionalDenom = Coin.ZERO;
                 for (Coin denom :denoms){
                     if (balanceNeedsAnonymized.isLessThan(denom)) {
@@ -1675,15 +1693,24 @@ public boolean removeTransactionListener(CoinJoinTransactionListener listener) {
         return ListenerRegistration.removeFromList(listener, transactionListeners);
     }
 
-    protected void queueCreateDenominationListeners(Transaction denominationTransaction, CoinJoinTransactionType type) {
+    protected void queueTransactionListeners(Transaction tx, CoinJoinTransactionType type) {
         //checkState(lock.isHeldByCurrentThread());
         for (final ListenerRegistration<CoinJoinTransactionListener> registration : transactionListeners) {
-            registration.executor.execute(new Runnable() {
-                @Override
-                public void run() {
-                    registration.listener.onTransactionProcessed(denominationTransaction, type);
-                }
-            });
+            registration.executor.execute(() -> registration.listener.onTransactionProcessed(tx, type, getSessionID()));
+        }
+    }
+
+    /**
+     *
+     * @param tx Transaction used for some purpose
+     * @param sessionId the session ID when the tx was used
+     * @param type CoinJoinTransactionType of tx
+     */
+
+    protected void queueTransactionListeners(Transaction tx, int sessionId, CoinJoinTransactionType type) {
+        //checkState(lock.isHeldByCurrentThread());
+        for (final ListenerRegistration<CoinJoinTransactionListener> registration : transactionListeners) {
+            registration.executor.execute(() -> registration.listener.onTransactionProcessed(tx, type, sessionId));
         }
     }
 }

core/src/main/java/org/bitcoinj/coinjoin/listeners/CoinJoinTransactionListener.java

@@ -18,5 +18,5 @@
 import org.bitcoinj.core.Transaction;
 
 public interface CoinJoinTransactionListener {
-    void onTransactionProcessed(Transaction denominationTx, CoinJoinTransactionType type);
+    void onTransactionProcessed(Transaction denominationTx, CoinJoinTransactionType type, int sessionId);
 }

core/src/main/java/org/bitcoinj/coinjoin/progress/MixingProgressTracker.java

@@ -43,6 +43,9 @@ public class MixingProgressTracker implements SessionStartedListener, SessionCom
     protected int completedSessions = 0;
     protected int timedOutSessions = 0;
     protected int timedOutConnections = 0;
+    protected int timedOutQueues = 0;
+    protected int timedOutWhileSigning = 0;
+    protected int timedOutWhileAccepting = 0;
     private double lastPercent = 0;
 
     private final SettableFuture<PoolMessage> future = SettableFuture.create();
@@ -60,6 +63,17 @@ public void onSessionComplete(WalletEx wallet, int sessionId, int denomination,
         } else if (message == PoolMessage.ERR_CONNECTION_TIMEOUT) {
             timedOutConnections++;
         } else {
+            switch (state) {
+                case POOL_STATE_QUEUE:
+                    ++timedOutQueues;
+                    break;
+                case POOL_STATE_SIGNING:
+                    ++timedOutWhileSigning;
+                    break;
+                case POOL_STATE_ACCEPTING_ENTRIES:
+                    ++timedOutWhileAccepting;
+                    break;
+            }
             timedOutSessions++;
         }
     }
@@ -109,7 +123,7 @@ public void notifyNewBestBlock(StoredBlock block) throws VerificationException {
     }
 
     @Override
-    public void onTransactionProcessed(Transaction denominationTx, CoinJoinTransactionType type) {
+    public void onTransactionProcessed(Transaction tx, CoinJoinTransactionType type, int sessionId) {
 
     }
 }