Merge pull request #59 from FigureTechnologies/pb_bug_fee_adjustment

Re-calculate fees within the block range of the pb release of 1.11
FigureTechnologies · Oct 24, 2022 · 8aba27c · 8aba27c
2 parents 7463b03 + 1663b39
commit 8aba27c
Show file tree

Hide file tree

Showing 15 changed files with 108 additions and 32 deletions.
diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/Config.kt b/common/src/main/kotlin/tech/figure/aggregate/common/Config.kt
@@ -34,7 +34,8 @@ data class Config (
     val hrp: String,
     val upload: UploadConfig = UploadConfig.empty(),
     val dbConfig: DBConfig,
-    val apiHost: String
+    val apiHost: String,
+    val badBlockRange: List<Long>
 )
 
 data class DBConfig(

diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/Extensions.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/Extensions.kt
@@ -1,6 +1,7 @@
 package tech.figure.aggregate.common.models
 
 import com.google.common.io.BaseEncoding
+import com.google.gson.Gson
 import cosmos.crypto.secp256k1.Keys
 import cosmos.tx.v1beta1.TxOuterClass
 import tech.figure.aggregate.common.decodeBase64
@@ -20,6 +21,10 @@ import io.provenance.eventstream.stream.clients.BlockData
 import io.provenance.eventstream.stream.models.Event
 import io.provenance.hdwallet.bech32.toBech32
 import io.provenance.hdwallet.common.hashing.sha256hash160
+import tech.figure.aggregate.common.models.fee.Fee
+import tech.figure.aggregate.common.models.fee.SignerInfo
+import tech.figure.aggregate.common.repeatDecodeBase64
+import java.math.BigDecimal
 import java.security.MessageDigest
 import java.security.NoSuchAlgorithmException
 import tendermint.types.Types.Header as GrpcHeader
@@ -75,7 +80,7 @@ fun Block.txData(index: Int, hrp: String): TxInfo? {
 
         return TxInfo(
             this.data?.txs?.get(index)?.hash(),
-            Fee(amount, denom, signerAddr, feeIncurringAddr)
+            Fee(amount, denom, SignerInfo(signerAddr, feeIncurringAddr))
         )
     }
     return null
@@ -88,18 +93,61 @@ fun Block.dateTime() = this.header?.dateTime()
 fun BlockHeader.dateTime(): OffsetDateTime? =
     runCatching { OffsetDateTime.parse(this.time, DateTimeFormatter.ISO_DATE_TIME) }.getOrNull()
 
-fun BlockResultsResponseResult.txEvents(blockDateTime: OffsetDateTime?, txHash: (Int) -> TxInfo?): List<TxEvent> =
+fun BlockResultsResponseResult.txEvents(blockDateTime: OffsetDateTime?, badBlockRange: Pair<Long, Long>, txHash: (Int) -> TxInfo?): List<TxEvent> =
     txsResults?.flatMapIndexed { index: Int, tx: BlockResultsResponseResultTxsResults ->
         val txHashData = txHash(index)
+
+        // 1.11 bug - fees were not properly swept [0] - lower, [1] - higher
+        val fee = if(height >= badBlockRange.first && height <= badBlockRange.second ) {
+            tx.toBugFee(txHashData?.fee?.signerInfo)
+        } else {
+            txHashData?.fee
+        }
+
         if(tx.code?.toInt() != 0) {
-           mutableListOf(toTxEvent(height, blockDateTime, txHashData?.txHash, txHashData?.fee ?: Fee()))
+           mutableListOf(toTxEvent(height, blockDateTime, txHashData?.txHash, fee ?: Fee()))
         } else {
             tx.events?.map {
-                it.toTxEvent(height, blockDateTime, txHashData?.txHash, fee = txHashData?.fee ?: Fee())
+                it.toTxEvent(height, blockDateTime, txHashData?.txHash, fee = fee ?: Fee())
             } ?: emptyList()
         }
     } ?: emptyList()
 
+fun BlockResultsResponseResultTxsResults.toBugFee(signerInfo: SignerInfo?): Fee {
+    var msgBaseFeeList = listOf<MsgFeeAttribute>()
+    this.events?.map { event ->
+        if(event.type?.repeatDecodeBase64() == "provenance.msgfees.v1.EventMsgFees") {
+            event.attributes?.toDecodedMap()?.map { attribute ->
+                msgBaseFeeList = Gson().fromJson(attribute.value?.decodeBase64(), Array<MsgFeeAttribute>::class.java).toList()
+            }
+        }
+    }
+
+    val msgBaseFee = if(msgBaseFeeList.isNotEmpty()) msgBaseFeeList[0].total.toAmountDenom().amount.toLong() else 0
+
+    // manually calc the proper fees
+    val gasPrice = 1905
+    val baseFee = BigDecimal(this.gasUsed!!.toLong() * gasPrice.toLong()).toLong()
+    val overageFee = BigDecimal(this.gasWanted!!.toLong() * gasPrice.toLong()).toLong() - baseFee
+    val totalFee = baseFee + msgBaseFee + overageFee
+
+    return Fee(fee = totalFee, "nhash", signerInfo)
+}
+
+fun String.toAmountDenom(): AmountDenom {
+    val amount = StringBuilder(this)
+    val denom = StringBuilder()
+    for (i in this.length - 1 downTo 0) {
+        val ch = this[i]
+        if (!ch.isDigit()) {
+            amount.deleteCharAt(i)
+            denom.insert(0, ch)
+        } else {
+            break
+        }
+    }
+    return AmountDenom(amount.toString(), denom.toString())
+}
 
 fun String.toSignerAddr(hrp: String): List<String> {
     val tx = TxOuterClass.Tx.parseFrom(BaseEncoding.base64().decode(this)) ?: return mutableListOf()
@@ -108,14 +156,22 @@ fun String.toSignerAddr(hrp: String): List<String> {
     }
 }
 
-fun BlockResultsResponseResult.txErroredEvents(blockDateTime: OffsetDateTime?, txHash: (Int) -> TxInfo?): List<TxError> =
+fun BlockResultsResponseResult.txErroredEvents(blockDateTime: OffsetDateTime?, gasPriceUpdateBlockHeight: Long, badBlockRange: Pair<Long, Long>, txHash: (Int) -> TxInfo?): List<TxError> =
     txsResults?.filter {
-            it.code?.toInt() ?: 0 != 0
+        (it.code?.toInt() ?: 0) != 0
         }?.mapIndexed { index: Int, tx: BlockResultsResponseResultTxsResults ->
-            tx.toBlockError(height, blockDateTime, txHash(index)?.txHash, txHash(index)?.fee ?: Fee())
-    }?.filterNotNull() ?: emptyList()
 
-fun BlockResultsResponseResultTxsResults.toBlockError(blockHeight: Long, blockDateTime: OffsetDateTime?, txHash: String?, fee: Fee): TxError? =
+        // 1.11 bug - fees were not properly swept [0] - lower, [1] - higher
+        val fee = if(height >= badBlockRange.first && height <= badBlockRange.second ) {
+            tx.toBugFee(txHash(index)?.fee?.signerInfo)
+        } else {
+            txHash(index)?.fee
+        }
+
+        tx.toBlockError(height, blockDateTime, txHash(index)?.txHash, fee = fee ?: Fee())
+    } ?: emptyList()
+
+fun BlockResultsResponseResultTxsResults.toBlockError(blockHeight: Long, blockDateTime: OffsetDateTime?, txHash: String?, fee: Fee): TxError =
     TxError(
         blockHeight = blockHeight,
         blockDateTime = blockDateTime,
@@ -151,8 +207,8 @@ fun BlockResultsResponseResultEvents.toTxEvent(
         fee = fee
     )
 
-fun BlockResultsResponse.txEvents(blockDate: OffsetDateTime, txHash: (index: Int) -> TxInfo): List<TxEvent> =
-    this.result.txEvents(blockDate, txHash)
+fun BlockResultsResponse.txEvents(blockDate: OffsetDateTime, badBlockRange: Pair<Long, Long>, txHash: (index: Int) -> TxInfo): List<TxEvent> =
+    this.result.txEvents(blockDate, badBlockRange, txHash)
 
 fun BlockResultsResponseResult.blockEvents(blockDateTime: OffsetDateTime?): List<BlockEvent> =
     beginBlockEvents?.map { e: BlockResultsResponseResultEvents ->
@@ -164,11 +220,11 @@ fun BlockResultsResponseResult.blockEvents(blockDateTime: OffsetDateTime?): List
         )
     } ?: emptyList()
 
-fun BlockData.toStreamBlock(hrp: String): StreamBlockImpl {
+fun BlockData.toStreamBlock(hrp: String, badBlockRange: Pair<Long, Long>): StreamBlockImpl {
     val blockDatetime = block.header?.dateTime()
     val blockEvents = blockResult.blockEvents(blockDatetime)
     val blockTxResults = blockResult.txsResults
-    val txEvents = blockResult.txEvents(blockDatetime) { index: Int -> block.txData(index, hrp) }
+    val txEvents = blockResult.txEvents(blockDatetime, badBlockRange) { index: Int -> block.txData(index, hrp) }
     return StreamBlockImpl(block, blockEvents, blockTxResults, txEvents)
 }
 

diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/Fee.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/Fee.kt
diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/MsgFeeAttribute.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/MsgFeeAttribute.kt
@@ -0,0 +1,8 @@
+package tech.figure.aggregate.common.models
+
+data class MsgFeeAttribute(
+    val msgType: String,
+    val count: Int,
+    val total: String,
+    val recipient: String
+)
diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/fee/Fee.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/fee/Fee.kt
@@ -0,0 +1,7 @@
+package tech.figure.aggregate.common.models.fee
+
+data class Fee(
+    val fee: Long? = 0L,
+    val denom: String? = "",
+    val signerInfo: SignerInfo? = SignerInfo()
+)
diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/fee/SignerInfo.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/fee/SignerInfo.kt
@@ -0,0 +1,6 @@
+package tech.figure.aggregate.common.models.fee
+
+data class SignerInfo(
+    val signerAddrs: List<String>? = mutableListOf(),
+    val incurrAddr: String? = ""
+)
diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxError.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxError.kt
@@ -1,7 +1,7 @@
 package tech.figure.aggregate.common.models.tx
 
 import com.squareup.moshi.JsonClass
-import tech.figure.aggregate.common.models.Fee
+import tech.figure.aggregate.common.models.fee.Fee
 import java.time.OffsetDateTime
 
 /**

diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxEvent.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxEvent.kt
@@ -1,7 +1,7 @@
 package tech.figure.aggregate.common.models.tx
 
 import com.squareup.moshi.JsonClass
-import tech.figure.aggregate.common.models.Fee
+import tech.figure.aggregate.common.models.fee.Fee
 import tech.figure.aggregate.common.models.block.EncodedBlockchainEvent
 import io.provenance.eventstream.stream.models.Event
 import java.time.OffsetDateTime

diff --git a/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxInfo.kt b/common/src/main/kotlin/tech/figure/aggregate/common/models/tx/TxInfo.kt
@@ -1,6 +1,6 @@
 package tech.figure.aggregate.common.models.tx
 
-import tech.figure.aggregate.common.models.Fee
+import tech.figure.aggregate.common.models.fee.Fee
 
 data class TxInfo(
     val txHash: String? = "",

diff --git a/src/main/kotlin/tech/figure/aggregate/service/Main.kt b/src/main/kotlin/tech/figure/aggregate/service/Main.kt
@@ -263,7 +263,7 @@ fun main(args: Array<String>) {
         val blockFlow: Flow<BlockData> = blockDataFlow(netAdapter, moshiDecoderAdapter(), from = options.fromHeight, to = options.toHeight)
         if (observe) {
             blockFlow
-                .transform { emit(it.toStreamBlock(config.hrp)) }
+                .transform { emit(it.toStreamBlock(config.hrp, Pair(config.badBlockRange[0], config.badBlockRange[1]))) }
                 .onEach {
                     val text = "Block: ${it.block.header?.height ?: "--"}:${it.block.header?.dateTime()?.toLocalDate()}"
                     println(green(text))
@@ -281,7 +281,7 @@ fun main(args: Array<String>) {
                             }
                         }
                     }
-            }
+                }
         } else {
             if (config.upload.extractors.isNotEmpty()) {
                 log.info("upload: adding extractors")
@@ -295,7 +295,8 @@ fun main(args: Array<String>) {
                 aws,
                 repository,
                 options,
-                config.hrp
+                config.hrp,
+                Pair(config.badBlockRange[0], config.badBlockRange[1])
             )
                 .addExtractor(config.upload.extractors)
                 .upload()

diff --git a/src/main/kotlin/tech/figure/aggregate/service/stream/consumers/EventStreamUploader.kt b/src/main/kotlin/tech/figure/aggregate/service/stream/consumers/EventStreamUploader.kt
@@ -48,6 +48,7 @@ class EventStreamUploader(
     private val repository: RepositoryBase?,
     private val options: BlockStreamOptions,
     private val hrp: String,
+    private val badBlockRange: Pair<Long, Long>,
     private val dispatchers: DispatcherProvider = DefaultDispatcherProvider(),
 ) {
 
@@ -151,7 +152,7 @@ class EventStreamUploader(
             }
 
         return blockFlow
-            .transform { emit(it.toStreamBlock(hrp)) }
+            .transform { emit(it.toStreamBlock(hrp, badBlockRange)) }
             .filter { streamBlock ->
                 !streamBlock.blockResult.isNullOrEmpty().also {
                     log.info(

diff --git a/src/main/kotlin/tech/figure/aggregate/service/stream/extractors/csv/impl/TxFees.kt b/src/main/kotlin/tech/figure/aggregate/service/stream/extractors/csv/impl/TxFees.kt
@@ -25,7 +25,7 @@ class TxFees: CSVFileExtractor(
                     event.blockDateTime,
                     event.fee.fee,
                     event.fee.denom,
-                    event.fee.incurrAddr, // wallet addr that is paying the fee
+                    event.fee.signerInfo?.incurrAddr, // wallet addr that is paying the fee
                     includeHash = true
                 )
             }

diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
@@ -9,6 +9,7 @@ aws:
 
 ws_node: ${WS_NODE}
 api_host: ${API_HOST}
+bad_block_range: ${ONE_DOT_ELEVEN_BAD_BLOCK_RANGE}
 
 dbConfig:
   addr: ${DB_URI}

diff --git a/src/main/resources/local.env.properties b/src/main/resources/local.env.properties
@@ -13,3 +13,4 @@ DB_TYPE=RAVENDB
 DB_MAX_CONNECTIONS=100
 API_HOST=http://localhost:8081
 HRP=tp
+ONE_DOT_ELEVEN_BAD_BLOCK_RANGE=8721085,10046412
diff --git a/src/test/kotlin/tech/figure/aggregate/service/EventStreamUploaderTests.kt b/src/test/kotlin/tech/figure/aggregate/service/EventStreamUploaderTests.kt
@@ -128,7 +128,8 @@ class EventStreamUploaderTests {
                         aws,
                         repository,
                         options,
-                        "tp"
+                        "tp",
+                        Pair(config.badBlockRange[0], config.badBlockRange[1])
                     )
                         .addExtractor(config.upload.extractors)
                         .upload { inspected1 = true }
@@ -169,7 +170,8 @@ class EventStreamUploaderTests {
                         aws,
                         repository,
                         options,
-                        "tp"
+                        "tp",
+                        Pair(config.badBlockRange[0], config.badBlockRange[1])
                     )
                         .addExtractor(config.upload.extractors)
                         .upload { inspected1 = true }