Add tracing by block logic for geth

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ### Features
 
+- [#9072](https://github.com/blockscout/blockscout/pull/9072) - Add tracing by block logic for geth
 - [#6721](https://github.com/blockscout/blockscout/pull/6721) - Implement fetching internal transactions from callTracer
 - [#5561](https://github.com/blockscout/blockscout/pull/5561), [#6523](https://github.com/blockscout/blockscout/pull/6523) - Improve working with contracts implementations
 - [#6401](https://github.com/blockscout/blockscout/pull/6401) - Add Sol2Uml contract visualization

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc.ex

@@ -25,6 +25,8 @@ defmodule EthereumJSONRPC do
   documentation for `EthereumJSONRPC.RequestCoordinator`.
   """
 
+  require Logger
+
   alias EthereumJSONRPC.{
     Block,
     Blocks,
@@ -380,6 +382,29 @@ defmodule EthereumJSONRPC do
     |> Enum.into(%{}, fn {params, id} -> {id, params} end)
   end
 
+  @doc """
+  Assigns not matched ids between requests and responses to responses with incorrect ids
+  """
+  def sanitize_responses(responses, id_to_params) do
+    responses
+    |> Enum.reduce(
+      {[], Map.keys(id_to_params) -- Enum.map(responses, & &1.id)},
+      fn
+        %{id: nil} = res, {result_res, [id | rest]} ->
+          Logger.error(
+            "Empty id in response: #{inspect(res)}, stacktrace: #{inspect(Process.info(self(), :current_stacktrace))}"
+          )
+
+          {[%{res | id: id} | result_res], rest}
+
+        res, {result_res, non_matched} ->
+          {[res | result_res], non_matched}
+      end
+    )
+    |> elem(0)
+    |> Enum.reverse()
+  end
+
   @doc """
     1. POSTs JSON `payload` to `url`
     2. Decodes the response
@@ -404,7 +429,7 @@ defmodule EthereumJSONRPC do
   @doc """
   Converts `t:quantity/0` to `t:non_neg_integer/0`.
   """
-  @spec quantity_to_integer(quantity) :: non_neg_integer() | :error
+  @spec quantity_to_integer(quantity) :: non_neg_integer() | nil
   def quantity_to_integer("0x" <> hexadecimal_digits) do
     String.to_integer(hexadecimal_digits, 16)
   end
@@ -414,10 +439,12 @@ defmodule EthereumJSONRPC do
   def quantity_to_integer(string) when is_binary(string) do
     case Integer.parse(string) do
       {integer, ""} -> integer
-      _ -> :error
+      _ -> nil
     end
   end
 
+  def quantity_to_integer(_), do: nil
+
   @doc """
   Converts `t:non_neg_integer/0` to `t:quantity/0`
   """
@@ -489,7 +516,7 @@ defmodule EthereumJSONRPC do
   """
   def timestamp_to_datetime(timestamp) do
     case quantity_to_integer(timestamp) do
-      :error ->
+      nil ->
         nil
 
       quantity ->

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/blocks.ex

@@ -32,6 +32,7 @@ defmodule EthereumJSONRPC.Blocks do
   def from_responses(responses, id_to_params) when is_list(responses) and is_map(id_to_params) do
     %{errors: errors, blocks: blocks} =
       responses
+      |> EthereumJSONRPC.sanitize_responses(id_to_params)
       |> Enum.map(&Block.from_response(&1, id_to_params))
       |> Enum.reduce(%{errors: [], blocks: []}, fn
         {:ok, block}, %{blocks: blocks} = acc ->

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/fetched_balances.ex

@@ -19,6 +19,7 @@ defmodule EthereumJSONRPC.FetchedBalances do
   """
   def from_responses(responses, id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&FetchedBalance.from_response(&1, id_to_params))
     |> Enum.reduce(
       %__MODULE__{},

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/fetched_beneficiaries.ex

@@ -77,6 +77,7 @@ defmodule EthereumJSONRPC.FetchedBeneficiaries do
   """
   def from_responses(responses, id_to_params) when is_list(responses) and is_map(id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&response_to_params_set(&1, id_to_params))
     |> Enum.reduce(
       %EthereumJSONRPC.FetchedBeneficiaries{},

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/fetched_codes.ex

@@ -34,6 +34,7 @@ defmodule EthereumJSONRPC.FetchedCodes do
   """
   def from_responses(responses, id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&FetchedCode.from_response(&1, id_to_params))
     |> Enum.reduce(
       %__MODULE__{},

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/geth.ex

@@ -46,12 +46,65 @@ defmodule EthereumJSONRPC.Geth do
   def fetch_first_trace(_transactions_params, _json_rpc_named_arguments), do: :ignore
 
   @doc """
-  Internal transaction fetching for entire blocks is not currently supported for Geth.
-
-  To signal to the caller that fetching is not supported, `:ignore` is returned.
+  Fetches the `t:Explorer.Chain.InternalTransaction.changeset/2` params from the Geth trace URL.
   """
   @impl EthereumJSONRPC.Variant
-  def fetch_block_internal_transactions(_block_range, _json_rpc_named_arguments), do: :ignore
+  def fetch_block_internal_transactions(block_numbers, json_rpc_named_arguments) do
+    id_to_params = id_to_params(block_numbers)
+
+    with {:ok, blocks_responses} <-
+           id_to_params
+           |> debug_trace_block_by_number_requests()
+           |> json_rpc(json_rpc_named_arguments),
+         :ok <- check_errors_exist(blocks_responses, id_to_params) do
+      transactions_params = to_transactions_params(blocks_responses, id_to_params)
+
+      {transactions_id_to_params, transactions_responses} =
+        Enum.reduce(transactions_params, {%{}, []}, fn {params, calls}, {id_to_params_acc, calls_acc} ->
+          {Map.put(id_to_params_acc, params[:id], params), [calls | calls_acc]}
+        end)
+
+      debug_trace_transaction_responses_to_internal_transactions_params(
+        transactions_responses,
+        transactions_id_to_params,
+        json_rpc_named_arguments
+      )
+    end
+  end
+
+  defp check_errors_exist(blocks_responses, id_to_params) do
+    blocks_responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
+    |> Enum.reduce([], fn
+      %{result: _result}, acc -> acc
+      %{error: error}, acc -> [error | acc]
+    end)
+    |> case do
+      [] -> :ok
+      errors -> {:error, errors}
+    end
+  end
+
+  defp to_transactions_params(blocks_responses, id_to_params) do
+    Enum.reduce(blocks_responses, [], fn %{id: id, result: tx_result}, blocks_acc ->
+      extract_transactions_params(Map.fetch!(id_to_params, id), tx_result) ++ blocks_acc
+    end)
+  end
+
+  defp extract_transactions_params(block_number, tx_result) do
+    tx_result
+    |> Enum.reduce({[], 0}, fn %{"txHash" => tx_hash, "result" => calls_result}, {tx_acc, counter} ->
+      {
+        [
+          {%{block_number: block_number, hash_data: tx_hash, transaction_index: counter, id: counter},
+           %{id: counter, result: calls_result}}
+          | tx_acc
+        ],
+        counter + 1
+      }
+    end)
+    |> elem(0)
+  end
 
   @doc """
   Fetches the pending transactions from the Geth node.
@@ -73,29 +126,63 @@ defmodule EthereumJSONRPC.Geth do
     end)
   end
 
+  defp debug_trace_block_by_number_requests(id_to_params) do
+    Enum.map(id_to_params, &debug_trace_block_by_number_request/1)
+  end
+
   @tracer_path "priv/js/ethereum_jsonrpc/geth/debug_traceTransaction/tracer.js"
   @external_resource @tracer_path
   @tracer File.read!(@tracer_path)
 
   defp debug_trace_transaction_request(%{id: id, hash_data: hash_data}) do
     timeout = Application.get_env(:ethereum_jsonrpc, :internal_transaction_timeout)
 
-    tracer =
-      case Application.get_env(:ethereum_jsonrpc, __MODULE__)[:tracer] do
-        "js" -> @tracer
-        "call_tracer" -> "callTracer"
-      end
-
     request(%{
       id: id,
       method: "debug_traceTransaction",
       params: [
         hash_data,
-        %{tracer: tracer, disableStack: true, disableMemory: true, disableStorage: true, timeout: timeout}
+        %{timeout: timeout} |> Map.merge(tracer_params())
       ]
     })
   end
 
+  defp debug_trace_block_by_number_request({id, block_number}) do
+    timeout = Application.get_env(:ethereum_jsonrpc, :internal_transaction_timeout)
+
+    request(%{
+      id: id,
+      method: "debug_traceBlockByNumber",
+      params: [
+        integer_to_quantity(block_number),
+        %{timeout: timeout} |> Map.merge(tracer_params())
+      ]
+    })
+  end
+
+  defp tracer_params do
+    cond do
+      tracer_type() == "js" ->
+        %{"tracer" => @tracer}
+
+      tracer_type() in ~w(opcode polygon_edge) ->
+        %{
+          "enableMemory" => true,
+          "disableStack" => false,
+          "disableStorage" => true,
+          "enableReturnData" => false
+        }
+
+      true ->
+        %{
+          "tracer" => "callTracer",
+          "disableStack" => true,
+          "disableMemory" => true,
+          "disableStorage" => true
+        }
+    end
+  end
+
   defp debug_trace_transaction_responses_to_internal_transactions_params(
          [%{result: %{"structLogs" => _}} | _] = responses,
          id_to_params,
@@ -136,6 +223,7 @@ defmodule EthereumJSONRPC.Geth do
        )
        when is_list(responses) and is_map(id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&debug_trace_transaction_response_to_internal_transactions_params(&1, id_to_params))
     |> reduce_internal_transactions_params()
   end
@@ -328,4 +416,8 @@ defmodule EthereumJSONRPC.Geth do
   defp finalize_internal_transactions_params({:error, acc_reasons}) do
     {:error, Enum.reverse(acc_reasons)}
   end
+
+  defp tracer_type do
+    Application.get_env(:ethereum_jsonrpc, __MODULE__)[:tracer]
+  end
 end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/http.ex

@@ -159,15 +159,18 @@ defmodule EthereumJSONRPC.HTTP do
 
     standardized = %{jsonrpc: jsonrpc, id: id}
 
-    case unstandardized do
-      %{"result" => _, "error" => _} ->
+    case {id, unstandardized} do
+      {_id, %{"result" => _, "error" => _}} ->
         raise ArgumentError,
               "result and error keys are mutually exclusive in JSONRPC 2.0 response objects, but got #{inspect(unstandardized)}"
 
-      %{"result" => result} ->
+      {nil, %{"result" => error}} ->
+        Map.put(standardized, :error, standardize_error(error))
+
+      {_id, %{"result" => result}} ->
         Map.put(standardized, :result, result)
 
-      %{"error" => error} ->
+      {_id, %{"error" => error}} ->
         Map.put(standardized, :error, standardize_error(error))
     end
   end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/receipts.ex

@@ -265,6 +265,7 @@ defmodule EthereumJSONRPC.Receipts do
   defp reduce_responses(responses, id_to_transaction_params)
        when is_list(responses) and is_map(id_to_transaction_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_transaction_params)
     |> Stream.map(&response_to_receipt(&1, id_to_transaction_params))
     |> Enum.reduce({:ok, []}, &reduce_receipt(&1, &2))
   end

apps/ethereum_jsonrpc/lib/ethereum_jsonrpc/trace_replay_block_transactions.ex

@@ -66,6 +66,7 @@ defmodule EthereumJSONRPC.TraceReplayBlockTransactions do
   defp trace_replay_block_transactions_responses_to_traces(responses, id_to_params)
        when is_list(responses) and is_map(id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&trace_replay_block_transactions_response_to_traces(&1, id_to_params))
     |> Enum.reduce(
       {:ok, []},
@@ -158,6 +159,7 @@ defmodule EthereumJSONRPC.TraceReplayBlockTransactions do
   defp trace_replay_transaction_responses_to_first_trace(responses, id_to_params)
        when is_list(responses) and is_map(id_to_params) do
     responses
+    |> EthereumJSONRPC.sanitize_responses(id_to_params)
     |> Enum.map(&trace_replay_transaction_response_to_first_trace(&1, id_to_params))
     |> Enum.reduce(
       {:ok, []},