From 4f5d992193fa95af61f5c268ea1d9719b0b50073 Mon Sep 17 00:00:00 2001 From: Mihail Dobrev Date: Tue, 29 Oct 2024 10:17:15 +0200 Subject: [PATCH] feat: add top miners endpoint --- lib/ae_mdw/db/int_transfer.ex | 5 +- lib/ae_mdw/db/model.ex | 12 +- .../db/mutations/top_miner_stats_mutation.ex | 70 ++++++ lib/ae_mdw/stats.ex | 90 ++++++++ .../controllers/stats_controller.ex | 10 + lib/ae_mdw_web/router.ex | 1 + .../20241025101739_generate_top_miners.ex | 43 ++++ test/ae_mdw/db/sync/block_test.exs | 2 + .../controllers/stats_controller_test.exs | 207 ++++++++++++++++++ 9 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 lib/ae_mdw/db/mutations/top_miner_stats_mutation.ex create mode 100644 priv/migrations/20241025101739_generate_top_miners.ex diff --git a/lib/ae_mdw/db/int_transfer.ex b/lib/ae_mdw/db/int_transfer.ex index b80a7e762..712910747 100644 --- a/lib/ae_mdw/db/int_transfer.ex +++ b/lib/ae_mdw/db/int_transfer.ex @@ -8,6 +8,7 @@ defmodule AeMdw.Db.IntTransfer do alias AeMdw.Db.Model alias AeMdw.Db.MinerRewardsMutation alias AeMdw.Db.Mutation + alias AeMdw.Db.TopMinerStatsMutation alias AeMdw.Db.State alias AeMdw.Collection @@ -40,6 +41,7 @@ defmodule AeMdw.Db.IntTransfer do def block_rewards_mutations(key_block) do height = :aec_blocks.height(key_block) delay = :aec_governance.beneficiary_reward_delay() + time = :aec_blocks.time_in_msecs(key_block) dev_benefs = for {protocol, _height} <- :aec_hard_forks.protocols(), @@ -72,7 +74,8 @@ defmodule AeMdw.Db.IntTransfer do [ IntTransfersMutation.new(height, miners_transfers ++ devs_transfers), - MinerRewardsMutation.new(miners_rewards) + MinerRewardsMutation.new(miners_rewards), + TopMinerStatsMutation.new(miners_rewards, time) ] end diff --git a/lib/ae_mdw/db/model.ex b/lib/ae_mdw/db/model.ex index 2494fdafb..dfbeb51a8 100644 --- a/lib/ae_mdw/db/model.ex +++ b/lib/ae_mdw/db/model.ex @@ -1236,6 +1236,14 @@ defmodule AeMdw.Db.Model do @type miner_index() :: pubkey() @type miner() :: record(:miner, index: miner_index(), total_reward: non_neg_integer()) + @type top_miner_stats_index() :: + {Stats.interval_by(), Stats.interval_start(), pos_integer(), pubkey()} + @type top_miner_stats() :: + record(:top_miner_stats, index: top_miner_stats_index()) + + @top_miner_stats_defaults [:index] + defrecord :top_miner_stats, @top_miner_stats_defaults + ################################################################################ # starts with only chain_tables and add them progressively by groups @@ -1373,7 +1381,8 @@ defmodule AeMdw.Db.Model do AeMdw.Db.Model.DeltaStat, AeMdw.Db.Model.TotalStat, AeMdw.Db.Model.Stat, - AeMdw.Db.Model.Statistic + AeMdw.Db.Model.Statistic, + AeMdw.Db.Model.TopMinerStats ] end @@ -1486,4 +1495,5 @@ defmodule AeMdw.Db.Model do def record(AeMdw.Db.Model.Statistic), do: :statistic def record(AeMdw.Db.Model.Miner), do: :miner def record(AeMdw.Db.Model.AccountNamesCount), do: :account_names_count + def record(AeMdw.Db.Model.TopMinerStats), do: :top_miner_stats end diff --git a/lib/ae_mdw/db/mutations/top_miner_stats_mutation.ex b/lib/ae_mdw/db/mutations/top_miner_stats_mutation.ex new file mode 100644 index 000000000..8fcdc6f7c --- /dev/null +++ b/lib/ae_mdw/db/mutations/top_miner_stats_mutation.ex @@ -0,0 +1,70 @@ +defmodule AeMdw.Db.TopMinerStatsMutation do + @moduledoc """ + Increments the top miners stats. + """ + + alias AeMdw.Collection + alias AeMdw.Db.IntTransfer + alias AeMdw.Db.State + alias AeMdw.Db.Model + alias AeMdw.Db.Sync.Stats + + require Model + + @derive AeMdw.Db.Mutation + defstruct [:rewards, :time] + + @opaque t() :: %__MODULE__{rewards: IntTransfer.rewards(), time: non_neg_integer()} + + @spec new(IntTransfer.rewards(), non_neg_integer()) :: t() + def new(rewards, time), do: %__MODULE__{rewards: rewards, time: time} + + @spec execute(t(), State.t()) :: State.t() + def execute(%__MODULE__{rewards: rewards, time: time}, state) do + Enum.reduce(rewards, state, fn {beneficiary_pk, _reward}, state -> + increment_top_miners(state, time, beneficiary_pk) + end) + end + + defp increment_top_miners(state, time, beneficiary_pk) do + time + |> Stats.time_intervals() + |> Enum.reduce(state, fn {interval_by, interval_start}, state -> + kb = + Collection.generate_key_boundary( + {interval_by, interval_start, Collection.integer(), Collection.binary()} + ) + + state + |> Collection.stream(Model.TopMinerStats, :backward, kb, nil) + |> Stream.filter(fn {_interval_by, _interval_start, _count, bpk} -> + bpk == beneficiary_pk + end) + |> tap(&IO.inspect(Enum.count(&1))) + |> Enum.at(0, :none) + |> case do + {^interval_by, ^interval_start, count, ^beneficiary_pk} -> + IO.inspect("updating") + + state + |> State.delete( + Model.TopMinerStats, + {interval_by, interval_start, count, beneficiary_pk} + ) + |> State.put( + Model.TopMinerStats, + Model.top_miner_stats(index: {interval_by, interval_start, count + 1, beneficiary_pk}) + ) + + :none -> + IO.inspect("inserting missing") + + State.put( + state, + Model.TopMinerStats, + Model.top_miner_stats(index: {interval_by, interval_start, 1, beneficiary_pk}) + ) + end + end) + end +end diff --git a/lib/ae_mdw/stats.ex b/lib/ae_mdw/stats.ex index eb653e4e3..58daa327d 100644 --- a/lib/ae_mdw/stats.ex +++ b/lib/ae_mdw/stats.ex @@ -280,6 +280,24 @@ defmodule AeMdw.Stats do end end + @spec fetch_top_miners_stats(State.t(), pagination(), query(), range(), cursor()) :: + {:ok, {pagination_cursor(), [statistic()], pagination_cursor()}} | {:error, reason()} + def fetch_top_miners_stats(state, pagination, query, range, cursor) do + with {:ok, filters} <- Util.convert_params(query, &convert_param/1), + {:ok, cursor} <- deserialize_top_miners_cursor(cursor) do + paginated_top_miners = + state + |> build_top_miners_streamer(filters, range, cursor) + |> Collection.paginate( + pagination, + &render_top_miner_statistic(state, &1), + &serialize_top_miners_cursor/1 + ) + + {:ok, paginated_top_miners} + end + end + defp fetch_statistics(state, pagination, filters, range, cursor, tag) do with {:ok, cursor} <- deserialize_statistic_cursor(cursor) do paginated_statistics = @@ -315,6 +333,21 @@ defmodule AeMdw.Stats do end end + defp build_top_miners_streamer(state, filters, _scope, cursor) do + interval_by = Map.get(filters, :interval_by, :day) + {start_network_date, end_network_date} = DbUtil.network_date_interval(state) + min_date = filters |> Map.get(:min_start_date, start_network_date) |> to_interval(interval_by) + max_date = filters |> Map.get(:max_start_date, end_network_date) |> to_interval(interval_by) + + key_boundary = + {{interval_by, min_date, 0, Util.min_bin()}, + {interval_by, max_date, Util.max_int(), Util.max_256bit_bin()}} + + fn direction -> + Collection.stream(state, Model.TopMinerStats, direction, key_boundary, cursor) + end + end + defp fill_missing_dates(stream, tag, interval_by, :backward, cursor, min_date, max_date) do max_date = case cursor do @@ -401,6 +434,42 @@ defmodule AeMdw.Stats do render_statistic(state, {:virtual, statistic_key, count}) end + defp render_top_miner_statistic( + _state, + {:month, interval_start, count, beneficiary_id} + ) do + %{ + start_date: months_to_iso(interval_start), + end_date: months_to_iso(interval_start + 1), + miner: :aeapi.format_account_pubkey(beneficiary_id), + count: count + } + end + + defp render_top_miner_statistic( + _state, + {:week, interval_start, count, beneficiary_id} + ) do + %{ + start_date: days_to_iso(interval_start * @days_per_week), + end_date: days_to_iso((interval_start + 1) * @days_per_week), + miner: :aeapi.format_account_pubkey(beneficiary_id), + count: count + } + end + + defp render_top_miner_statistic( + _state, + {:day, interval_start, count, beneficiary_id} + ) do + %{ + start_date: days_to_iso(interval_start), + end_date: days_to_iso(interval_start + 1), + miner: :aeapi.format_account_pubkey(beneficiary_id), + count: count + } + end + defp convert_blocks_param({"type", "key"}), do: {:ok, {:block_type, :key}} defp convert_blocks_param({"type", "micro"}), do: {:ok, {:block_type, :micro}} defp convert_blocks_param(param), do: convert_param(param) @@ -440,6 +509,12 @@ defmodule AeMdw.Stats do defp serialize_statistics_cursor({:virtual, {_tag, _interval_by, interval_start}, _count}), do: "#{interval_start}" + defp serialize_top_miners_cursor({_interval_by, _interval_start, _count, _ben} = cursor) do + cursor + |> :erlang.term_to_binary() + |> Base.encode64() + end + defp deserialize_statistic_cursor(nil), do: {:ok, nil} defp deserialize_statistic_cursor(cursor_bin) do @@ -449,6 +524,21 @@ defmodule AeMdw.Stats do end end + defp deserialize_top_miners_cursor(nil), do: {:ok, nil} + + defp deserialize_top_miners_cursor(cursor_bin) do + case Base.decode64(cursor_bin) do + {:ok, bin} -> + case :erlang.binary_to_term(bin) do + cursor when is_tuple(cursor) -> {:ok, cursor} + _ -> {:error, ErrInput.Cursor.exception(value: cursor_bin)} + end + + :error -> + {:error, ErrInput.Cursor.exception(value: cursor_bin)} + end + end + defp render_delta_stats(state, gens), do: Enum.map(gens, &fetch_delta_stat!(state, &1)) defp render_total_stats(state, gens), do: Enum.map(gens, &fetch_total_stat!(state, &1)) diff --git a/lib/ae_mdw_web/controllers/stats_controller.ex b/lib/ae_mdw_web/controllers/stats_controller.ex index 02a543ee3..ac5834780 100644 --- a/lib/ae_mdw_web/controllers/stats_controller.ex +++ b/lib/ae_mdw_web/controllers/stats_controller.ex @@ -134,4 +134,14 @@ defmodule AeMdwWeb.StatsController do Util.render(conn, paginated_stats) end end + + @spec top_miners_stats(Conn.t(), map()) :: Conn.t() + def top_miners_stats(%Conn{assigns: assigns} = conn, _params) do + %{state: state, pagination: pagination, query: query, scope: scope, cursor: cursor} = assigns + + with {:ok, paginated_stats} <- + Stats.fetch_top_miners_stats(state, pagination, query, scope, cursor) do + Util.render(conn, paginated_stats) + end + end end diff --git a/lib/ae_mdw_web/router.ex b/lib/ae_mdw_web/router.ex index 8d1a3c347..a3874ce68 100644 --- a/lib/ae_mdw_web/router.ex +++ b/lib/ae_mdw_web/router.ex @@ -73,6 +73,7 @@ defmodule AeMdwWeb.Router do get "/stats/total", StatsController, :total_stats get "/stats/delta", StatsController, :delta_stats get "/stats/miners", StatsController, :miners_stats + get "/stats/miners/top", StatsController, :top_miners_stats get "/stats/contracts", StatsController, :contracts_stats get "/stats/aex9-transfers", StatsController, :aex9_transfers_stats get "/stats", StatsController, :stats diff --git a/priv/migrations/20241025101739_generate_top_miners.ex b/priv/migrations/20241025101739_generate_top_miners.ex new file mode 100644 index 000000000..3af127b02 --- /dev/null +++ b/priv/migrations/20241025101739_generate_top_miners.ex @@ -0,0 +1,43 @@ +defmodule AeMdw.Migrations.GenerateTopMiners do + alias AeMdw.Db.TopMinerStatsMutation + alias AeMdw.Db.Model + alias AeMdw.Db.RocksDbCF + alias AeMdw.Db.State + + require Model + + @spec run(State.t(), boolean()) :: {:ok, non_neg_integer()} + def run(state, _from_start?) do + # dev_benefs = + # for {protocol, _height} <- :aec_hard_forks.protocols(), + # {pk, _share} <- :aec_dev_reward.beneficiaries(protocol) do + # pk + # end + + # delay = :aec_governance.beneficiary_reward_delay() + + {_state, count} = + Model.Block + |> RocksDbCF.stream() + |> Stream.filter(fn Model.block(index: {_key_index, micro_index}) -> micro_index == -1 end) + |> Stream.map(fn Model.block() = block -> + Model.block(hash: hash) = block + {:ok, key_block} = :aec_chain.get_block(hash) + time = :aec_blocks.time_in_msecs(key_block) + + miner = + key_block + |> :aec_blocks.to_header() + |> :aec_headers.miner() + + TopMinerStatsMutation.new([{miner, 0}], time) + end) + |> Stream.chunk_every(1000) + |> Enum.reduce({state, 0}, fn mutations, {state, count} -> + len = length(mutations) + {State.commit_db(state, mutations), count + len} + end) + + {:ok, count} + end +end diff --git a/test/ae_mdw/db/sync/block_test.exs b/test/ae_mdw/db/sync/block_test.exs index 201ba7b54..f399c8184 100644 --- a/test/ae_mdw/db/sync/block_test.exs +++ b/test/ae_mdw/db/sync/block_test.exs @@ -1,7 +1,9 @@ defmodule AeMdw.Db.Sync.BlockTest do use ExUnit.Case + alias AeMdw.Collection alias AeMdw.Db.Model + alias AeMdw.Db.State alias AeMdw.Db.Sync.Block import AeMdwWeb.BlockchainSim, diff --git a/test/ae_mdw_web/controllers/stats_controller_test.exs b/test/ae_mdw_web/controllers/stats_controller_test.exs index 62b806b04..79bb6f9c9 100644 --- a/test/ae_mdw_web/controllers/stats_controller_test.exs +++ b/test/ae_mdw_web/controllers/stats_controller_test.exs @@ -907,6 +907,213 @@ defmodule AeMdwWeb.StatsControllerTest do end end + describe "top miners stats" do + setup %{store: store, conn: conn} do + miner1 = <<1::256>> + miner2 = <<2::256>> + miner3 = <<3::256>> + miner4 = <<4::256>> + {network_start_time, network_end_time} = network_time_interval() + + store = + [ + Model.top_miner_stats(index: {:day, 0, 7, miner1}), + Model.top_miner_stats(index: {:day, 0, 6, miner2}), + Model.top_miner_stats(index: {:day, 0, 5, miner3}), + Model.top_miner_stats(index: {:day, 0, 4, miner4}), + Model.top_miner_stats(index: {:day, 1, 1, miner1}), + Model.top_miner_stats(index: {:day, 1, 2, miner2}), + Model.top_miner_stats(index: {:day, 1, 3, miner3}), + Model.top_miner_stats(index: {:day, 1, 4, miner4}), + Model.top_miner_stats(index: {:week, 0, 8, miner1}), + Model.top_miner_stats(index: {:week, 0, 8, miner2}), + Model.top_miner_stats(index: {:week, 0, 8, miner3}), + Model.top_miner_stats(index: {:week, 0, 8, miner4}), + Model.top_miner_stats(index: {:week, 1, 8, miner1}), + Model.top_miner_stats(index: {:week, 1, 8, miner2}), + Model.top_miner_stats(index: {:week, 1, 8, miner3}), + Model.top_miner_stats(index: {:week, 1, 8, miner4}), + Model.top_miner_stats(index: {:month, 0, 16, miner1}), + Model.top_miner_stats(index: {:month, 0, 16, miner2}), + Model.top_miner_stats(index: {:month, 0, 16, miner3}), + Model.top_miner_stats(index: {:month, 0, 16, miner4}) + ] + |> Enum.reduce(store, fn mutation, store -> + Store.put(store, Model.TopMinerStats, mutation) + end) + |> Store.put(Model.Time, Model.time(index: {network_start_time, 0})) + |> Store.put(Model.Time, Model.time(index: {network_end_time, 200})) + + miners = + [ + miner1, + miner2, + miner3, + miner4 + ] + |> Enum.map(&:aeapi.format_account_pubkey/1) + + conn = with_store(conn, store) + + {:ok, %{store: store, miners: miners, conn: conn}} + end + + test "it returns the top miners for specific date", %{conn: conn, miners: miners} do + assert %{"prev" => nil, "data" => [st1, st2] = statistics, "next" => next_url} = + conn + |> get("/v3/stats/miners/top", + limit: 2, + min_start_date: "1970-01-01", + max_start_date: "1970-01-01" + ) + |> json_response(200) + + [miner1, miner2, miner3, miner4] = miners + assert %{"miner" => ^miner1, "count" => 7} = st1 + assert %{"miner" => ^miner2, "count" => 6} = st2 + + assert %{"prev" => prev_url, "data" => [st3, st4], "next" => nil} = + conn + |> get(next_url) + |> json_response(200) + + assert %{"miner" => ^miner3, "count" => 5} = st3 + assert %{"miner" => ^miner4, "count" => 4} = st4 + + assert %{"data" => ^statistics} = + conn + |> get(prev_url) + |> json_response(200) + end + + test "it returns the top miners for each day", %{conn: conn, miners: miners} do + assert %{"prev" => nil, "data" => [st1, st2, st3, st4] = statistics, "next" => next_url} = + conn + |> get("/v3/stats/miners/top", + limit: 4, + min_start_date: "1970-01-01", + max_start_date: "1970-01-02" + ) + |> json_response(200) + + [miner1, miner2, miner3, miner4] = miners + + assert %{"miner" => ^miner4, "count" => 4} = st1 + assert %{"miner" => ^miner3, "count" => 3} = st2 + assert %{"miner" => ^miner2, "count" => 2} = st3 + assert %{"miner" => ^miner1, "count" => 1} = st4 + + assert %{"prev" => prev_url, "data" => [st5, st6, st7, st8], "next" => nil} = + conn + |> get(next_url) + |> json_response(200) + + assert %{"miner" => ^miner1, "count" => 7} = st5 + assert %{"miner" => ^miner2, "count" => 6} = st6 + assert %{"miner" => ^miner3, "count" => 5} = st7 + assert %{"miner" => ^miner4, "count" => 4} = st8 + + assert %{"data" => ^statistics} = + conn + |> get(prev_url) + |> json_response(200) + end + + test "it returns top miners for a week", %{conn: conn, miners: miners} do + [miner1, miner2, miner3, miner4] = miners + + assert %{"prev" => nil, "data" => [st1, st2] = statistics, "next" => next_url} = + conn + |> get("/v3/stats/miners/top", + limit: 2, + interval_by: "week", + min_start_date: "1970-01-01", + max_start_date: "1970-01-07" + ) + |> json_response(200) + + assert %{"miner" => ^miner4, "count" => 8} = st1 + assert %{"miner" => ^miner3, "count" => 8} = st2 + + assert %{"prev" => prev_url, "data" => [st3, st4], "next" => nil} = + conn + |> get(next_url) + |> json_response(200) + + assert %{"miner" => ^miner2, "count" => 8} = st3 + assert %{"miner" => ^miner1, "count" => 8} = st4 + + assert %{"data" => ^statistics} = + conn + |> get(prev_url) + |> json_response(200) + end + + test "it returns top miners for multiple weeks", %{conn: conn, miners: miners} do + [miner1, miner2, miner3, miner4] = miners + + assert %{"prev" => nil, "data" => [st1, st2, st3, st4] = statistics, "next" => next_url} = + conn + |> get("/v3/stats/miners/top", + limit: 4, + interval_by: "week", + min_start_date: "1970-01-01", + max_start_date: "1970-01-13" + ) + |> json_response(200) + + assert %{"miner" => ^miner4, "count" => 8, "start_date" => "1970-01-08"} = st1 + assert %{"miner" => ^miner3, "count" => 8, "start_date" => "1970-01-08"} = st2 + assert %{"miner" => ^miner2, "count" => 8, "start_date" => "1970-01-08"} = st3 + assert %{"miner" => ^miner1, "count" => 8, "start_date" => "1970-01-08"} = st4 + + assert %{"prev" => prev_url, "data" => [st5, st6, st7, st8], "next" => nil} = + conn + |> get(next_url) + |> json_response(200) + + assert %{"miner" => ^miner4, "count" => 8, "start_date" => "1970-01-01"} = st5 + assert %{"miner" => ^miner3, "count" => 8, "start_date" => "1970-01-01"} = st6 + assert %{"miner" => ^miner2, "count" => 8, "start_date" => "1970-01-01"} = st7 + assert %{"miner" => ^miner1, "count" => 8, "start_date" => "1970-01-01"} = st8 + + assert %{"data" => ^statistics} = + conn + |> get(prev_url) + |> json_response(200) + end + + test "it returns top miners for a month", %{conn: conn, miners: miners} do + [miner1, miner2, miner3, miner4] = miners + + assert %{"prev" => nil, "data" => [st1, st2] = statistics, "next" => next_url} = + conn + |> get("/v3/stats/miners/top", + limit: 2, + interval_by: "month", + min_start_date: "1970-01-01", + max_start_date: "1970-01-31" + ) + |> json_response(200) + + assert %{"miner" => ^miner4, "count" => 16} = st1 + assert %{"miner" => ^miner3, "count" => 16} = st2 + + assert %{"prev" => prev_url, "data" => [st3, st4], "next" => nil} = + conn + |> get(next_url) + |> json_response(200) + + assert %{"miner" => ^miner2, "count" => 16} = st3 + assert %{"miner" => ^miner1, "count" => 16} = st4 + + assert %{"data" => ^statistics} = + conn + |> get(prev_url) + |> json_response(200) + end + end + defp add_transactions(store, start_txi, end_txi) do start_txi..end_txi |> Enum.reduce({store, 1}, fn txi, {store, i} ->