Merge pull request #7 from maestro-org/perf/api-performance-improvement

perf/api-performance-improvement

Author: stivens13

apps/block_scout_web/lib/block_scout_web/controllers/api/v2/token_transfer_controller.ex

@@ -14,7 +14,8 @@ defmodule BlockScoutWeb.API.V2.TokenTransferController do
   import BlockScoutWeb.PagingHelper,
     only: [
       delete_parameters_from_next_page_params: 1,
-      token_transfers_types_options: 1
+      token_transfers_types_options: 1,
+      token_transfers_activity_options: 1
     ]
 
   import Explorer.MicroserviceInterfaces.BENS, only: [maybe_preload_ens: 1]
@@ -41,6 +42,7 @@ defmodule BlockScoutWeb.API.V2.TokenTransferController do
         %PagingOptions{paging_options | page_size: page_size + 1}
       end)
       |> Keyword.merge(token_transfers_types_options(params))
+      |> Keyword.merge(token_transfers_activity_options(params))
       |> Keyword.merge(@api_true)
 
     result =

apps/block_scout_web/lib/block_scout_web/paging_helper.ex

@@ -37,6 +37,7 @@ defmodule BlockScoutWeb.PagingHelper do
   end
 
   @allowed_token_transfer_type_labels ["ERC-20", "ERC-721", "ERC-1155", "ERC-404"]
+  @allowed_token_transfer_activity_labels ["MINTING", "BURNING", "TRANSFER", "SPAWNING"]
   @allowed_nft_type_labels ["ERC-721", "ERC-1155", "ERC-404"]
   @allowed_chain_id [1, 56, 99]
   @allowed_stability_validators_states ["active", "probation", "inactive"]
@@ -81,6 +82,19 @@ defmodule BlockScoutWeb.PagingHelper do
 
   def token_transfers_types_options(_), do: [token_type: []]
 
+  @doc """
+    Parse 'activity' query parameter from request option map for token transfers.
+    Accepts: minting, burning, transfer, spawning (case-insensitive, comma-separated)
+  """
+  @spec token_transfers_activity_options(map()) :: [{:activity, list}]
+  def token_transfers_activity_options(%{"activity" => filters}) do
+    [
+      activity: filters_to_list(filters, @allowed_token_transfer_activity_labels)
+    ]
+  end
+
+  def token_transfers_activity_options(_), do: [activity: []]
+
   @doc """
     Parse 'type' query parameter from request option map
   """

apps/block_scout_web/lib/block_scout_web/views/api/v2/block_view.ex

@@ -33,6 +33,7 @@ defmodule BlockScoutWeb.API.V2.BlockView do
     priority_fee = block.base_fee_per_gas && BlockPriorityFeeCounter.fetch(block.hash)
 
     transaction_fees = Block.transaction_fees(block.transactions)
+    total_value = Block.total_value(block.transactions)
 
     %{
       "height" => block.number,
@@ -59,6 +60,7 @@ defmodule BlockScoutWeb.API.V2.BlockView do
       "burnt_fees_percentage" => burnt_fees_percentage(burnt_fees, transaction_fees),
       "type" => block |> BlockView.block_type() |> String.downcase(),
       "transaction_fees" => transaction_fees,
+      "total_value" => total_value,
       "withdrawals_count" => count_withdrawals(block)
     }
     |> chain_type_fields(block, single_block?)

apps/explorer/lib/explorer/chain.ex

@@ -575,41 +575,42 @@ defmodule Explorer.Chain do
     necessity_by_association = Keyword.get(options, :necessity_by_association, %{})
     type_filter = Keyword.get(options, :type)
 
-    options
-    |> Keyword.get(:paging_options, @default_paging_options)
-    |> fetch_transactions_in_ascending_order_by_index()
-    |> join(:inner, [transaction], block in assoc(transaction, :block))
-    |> where([_, block], block.hash == ^block_hash)
-    |> apply_filter_by_type_to_transactions(type_filter)
-    |> join_associations(necessity_by_association)
-    |> Transaction.put_has_token_transfers_to_transaction(old_ui?)
-    |> (&if(old_ui?, do: preload(&1, [{:token_transfers, [:token, :from_address, :to_address]}]), else: &1)).()
-    |> select_repo(options).all()
-    |> (&if(old_ui?,
-          do: &1,
-          else:
-            Enum.map(&1, fn transaction ->
-              preload_token_transfers(transaction, @token_transfer_necessity_by_association, options)
-            end)
-        )).()
+    transactions =
+      options
+      |> Keyword.get(:paging_options, @default_paging_options)
+      |> fetch_transactions_in_ascending_order_by_index()
+      |> join(:inner, [transaction], block in assoc(transaction, :block))
+      |> where([_, block], block.hash == ^block_hash)
+      |> apply_filter_by_type_to_transactions(type_filter)
+      |> join_associations(necessity_by_association)
+      |> Transaction.put_has_token_transfers_to_transaction(old_ui?)
+      |> (&if(old_ui?, do: preload(&1, [{:token_transfers, [:token, :from_address, :to_address]}]), else: &1)).()
+      |> select_repo(options).all()
+
+    # Use batch preloading instead of N+1 queries for new UI
+    if old_ui? do
+      transactions
+    else
+      batch_preload_token_transfers(transactions, @token_transfer_necessity_by_association, options)
+    end
   end
 
   @spec execution_node_to_transactions(Hash.Address.t(), [paging_options | necessity_by_association_option | api?()]) ::
           [Transaction.t()]
   def execution_node_to_transactions(execution_node_hash, options \\ []) when is_list(options) do
     necessity_by_association = Keyword.get(options, :necessity_by_association, %{})
 
-    options
-    |> Keyword.get(:paging_options, @default_paging_options)
-    |> fetch_transactions_in_descending_order_by_block_and_index()
-    |> where(execution_node_hash: ^execution_node_hash)
-    |> join_associations(necessity_by_association)
-    |> Transaction.put_has_token_transfers_to_transaction(false)
-    |> (& &1).()
-    |> select_repo(options).all()
-    |> (&Enum.map(&1, fn transaction ->
-          preload_token_transfers(transaction, @token_transfer_necessity_by_association, options)
-        end)).()
+    transactions =
+      options
+      |> Keyword.get(:paging_options, @default_paging_options)
+      |> fetch_transactions_in_descending_order_by_block_and_index()
+      |> where(execution_node_hash: ^execution_node_hash)
+      |> join_associations(necessity_by_association)
+      |> Transaction.put_has_token_transfers_to_transaction(false)
+      |> select_repo(options).all()
+
+    # Use batch preloading instead of N+1 queries
+    batch_preload_token_transfers(transactions, @token_transfer_necessity_by_association, options)
   end
 
   @spec block_to_withdrawals(
@@ -1368,6 +1369,62 @@ defmodule Explorer.Chain do
 
   def get_token_transfers_per_transaction_preview_count, do: @token_transfers_per_transaction_preview
 
+  @doc """
+  Batch preloads token transfers for multiple transactions in a single query.
+  This eliminates the N+1 query problem when loading token transfers for transaction lists.
+
+  For each transaction, loads up to `@token_transfers_per_transaction_preview` token transfers
+  with their associated addresses and tokens.
+
+  Token transfers are filtered by block_hash in Elixir to ensure data correctness during reorgs,
+  matching the original behavior where transfers are filtered by both transaction_hash AND block_hash.
+  """
+  @spec batch_preload_token_transfers([Transaction.t()], map(), Keyword.t()) :: [Transaction.t()]
+  def batch_preload_token_transfers([], _necessity_by_association, _options), do: []
+
+  def batch_preload_token_transfers(transactions, necessity_by_association, options) do
+    # Build a map of transaction_hash -> block_hash for filtering
+    tx_block_hash_map =
+      transactions
+      |> Enum.map(fn tx -> {tx.hash, tx.block_hash} end)
+      |> Map.new()
+
+    transaction_hashes = Map.keys(tx_block_hash_map)
+
+    # Query all token transfers for all transactions in one query
+    # We fetch more than needed and limit per-transaction in Elixir
+    token_transfers =
+      from(tt in TokenTransfer,
+        where: tt.transaction_hash in ^transaction_hashes,
+        order_by: [asc: tt.transaction_hash, asc: tt.log_index]
+      )
+      |> join_associations(necessity_by_association)
+      |> select_repo(options).all()
+      |> flat_1155_batch_token_transfers()
+      # Filter by block_hash to ensure data correctness (matches original behavior)
+      # If tx.block_hash is nil (pending tx), accept any token transfer for that tx
+      # Otherwise, only accept transfers from the same block
+      |> Enum.filter(fn tt ->
+        tx_block_hash = Map.get(tx_block_hash_map, tt.transaction_hash)
+        is_nil(tx_block_hash) or tt.block_hash == tx_block_hash
+      end)
+
+    # Group token transfers by transaction hash
+    transfers_by_tx_hash =
+      token_transfers
+      |> Enum.group_by(& &1.transaction_hash)
+
+    # Attach token transfers to each transaction, limiting to preview count
+    Enum.map(transactions, fn tx ->
+      tx_transfers =
+        transfers_by_tx_hash
+        |> Map.get(tx.hash, [])
+        |> Enum.take(@token_transfers_per_transaction_preview)
+
+      %Transaction{tx | token_transfers: tx_transfers}
+    end)
+  end
+
   @doc """
   Converts list of `t:Explorer.Chain.Transaction.t/0` `hashes` to the list of `t:Explorer.Chain.Transaction.t/0`s for
   those `hashes`.
@@ -2685,22 +2742,23 @@ defmodule Explorer.Chain do
         []
 
       _ ->
-        paging_options
-        |> Transaction.fetch_transactions()
-        |> where([transaction], not is_nil(transaction.block_number) and not is_nil(transaction.index))
-        |> apply_filter_by_method_id_to_transactions(method_id_filter)
-        |> apply_filter_by_type_to_transactions(type_filter)
-        |> join_associations(necessity_by_association)
-        |> Transaction.put_has_token_transfers_to_transaction(old_ui?)
-        |> (&if(old_ui?, do: preload(&1, [{:token_transfers, [:token, :from_address, :to_address]}]), else: &1)).()
-        |> select_repo(options).all()
-        |> (&if(old_ui?,
-              do: &1,
-              else:
-                Enum.map(&1, fn transaction ->
-                  preload_token_transfers(transaction, @token_transfer_necessity_by_association, options)
-                end)
-            )).()
+        transactions =
+          paging_options
+          |> Transaction.fetch_transactions()
+          |> where([transaction], not is_nil(transaction.block_number) and not is_nil(transaction.index))
+          |> apply_filter_by_method_id_to_transactions(method_id_filter)
+          |> apply_filter_by_type_to_transactions(type_filter)
+          |> join_associations(necessity_by_association)
+          |> Transaction.put_has_token_transfers_to_transaction(old_ui?)
+          |> (&if(old_ui?, do: preload(&1, [{:token_transfers, [:token, :from_address, :to_address]}]), else: &1)).()
+          |> select_repo(options).all()
+
+        # Use batch preloading instead of N+1 queries for new UI
+        if old_ui? do
+          transactions
+        else
+          batch_preload_token_transfers(transactions, @token_transfer_necessity_by_association, options)
+        end
     end
   end
 

apps/explorer/lib/explorer/chain/block.ex

@@ -358,6 +358,21 @@ defmodule Explorer.Chain.Block do
     end)
   end
 
+  @doc """
+  Calculates total value transferred for the list of transactions (from a single block)
+  """
+  @spec total_value([Transaction.t()]) :: Decimal.t()
+  def total_value(transactions) do
+    Enum.reduce(transactions, Decimal.new(0), fn %{value: value}, acc ->
+      if value do
+        value.value
+        |> Decimal.add(acc)
+      else
+        acc
+      end
+    end)
+  end
+
   @doc """
   Finds blob transaction gas price for the list of transactions (from a single block)
   """

apps/explorer/lib/explorer/chain/token_transfer.ex

@@ -140,7 +140,7 @@ defmodule Explorer.Chain.TokenTransfer do
   import Ecto.Changeset
 
   alias Explorer.{Chain, Helper}
-  alias Explorer.Chain.{DenormalizationHelper, Hash, Log, TokenTransfer}
+  alias Explorer.Chain.{DenormalizationHelper, Hash, Log, SmartContract, TokenTransfer}
   alias Explorer.Chain.SmartContract.Proxy.Models.Implementation
   alias Explorer.{PagingOptions, Repo}
 
@@ -292,6 +292,7 @@ defmodule Explorer.Chain.TokenTransfer do
   def fetch(options) do
     paging_options = Keyword.get(options, :paging_options, @default_paging_options)
     token_type = Keyword.get(options, :token_type)
+    activity = Keyword.get(options, :activity, [])
 
     case paging_options do
       %PagingOptions{key: {0, 0}} ->
@@ -310,6 +311,7 @@ defmodule Explorer.Chain.TokenTransfer do
         |> preload(^preloads)
         |> order_by([tt], desc: tt.block_number, desc: tt.log_index)
         |> maybe_filter_by_token_type(token_type)
+        |> maybe_filter_by_activity(activity)
         |> page_token_transfer(paging_options)
         |> limit(^paging_options.page_size)
         |> Chain.select_repo(options).all()
@@ -331,6 +333,51 @@ defmodule Explorer.Chain.TokenTransfer do
     end
   end
 
+  @doc """
+  Filters token transfers by activity type (minting, burning, transfer, spawning).
+  Activity is determined by comparing from/to addresses with the burn address (0x0...0).
+  """
+  defp maybe_filter_by_activity(query, activities) when is_list(activities) do
+    if Enum.empty?(activities) do
+      query
+    else
+      {:ok, burn_address_hash} = Chain.string_to_address_hash(SmartContract.burn_address_hash_string())
+
+      conditions =
+        activities
+        |> Enum.map(&activity_to_condition(&1, burn_address_hash))
+        |> Enum.reject(&is_nil/1)
+
+      if Enum.empty?(conditions) do
+        query
+      else
+        combined_condition = Enum.reduce(conditions, fn condition, acc ->
+          dynamic([tt], ^acc or ^condition)
+        end)
+
+        where(query, ^combined_condition)
+      end
+    end
+  end
+
+  defp activity_to_condition("MINTING", burn_address_hash) do
+    dynamic([tt], tt.from_address_hash == ^burn_address_hash and tt.to_address_hash != ^burn_address_hash)
+  end
+
+  defp activity_to_condition("BURNING", burn_address_hash) do
+    dynamic([tt], tt.to_address_hash == ^burn_address_hash and tt.from_address_hash != ^burn_address_hash)
+  end
+
+  defp activity_to_condition("SPAWNING", burn_address_hash) do
+    dynamic([tt], tt.from_address_hash == ^burn_address_hash and tt.to_address_hash == ^burn_address_hash)
+  end
+
+  defp activity_to_condition("TRANSFER", burn_address_hash) do
+    dynamic([tt], tt.from_address_hash != ^burn_address_hash and tt.to_address_hash != ^burn_address_hash)
+  end
+
+  defp activity_to_condition(_, _burn_address_hash), do: nil
+
   @spec count_token_transfers_from_token_hash(Hash.t()) :: non_neg_integer()
   def count_token_transfers_from_token_hash(token_address_hash) do
     query =