scripts: copy prod scripts from 0.14.1 to 0.14.0 (#9763)

Author: guyf-starkware

scripts/prod/restart_all_nodes_together.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+from typing import Optional
+
+import urllib.error
+import urllib.parse
+import urllib.request
+from update_config_and_restart_nodes_lib import (
+    ApolloArgsParserBuilder,
+    RestartStrategy,
+    Service,
+    get_configmap,
+    get_context_list_from_args,
+    get_current_block_number,
+    get_logs_explorer_url,
+    get_namespace_list_from_args,
+    parse_config_from_yaml,
+    print_colored,
+    update_config_and_restart_nodes,
+)
+
+
+# TODO(guy.f): Remove this once we have metrics we use to decide based on.
+def get_logs_explorer_url_for_proposal(
+    namespace: str,
+    validator_id: str,
+    min_block_number: int,
+    project_name: str,
+) -> str:
+    # Remove the 0x prefix from the validator id to get the number.
+    validator_id = validator_id[2:]
+
+    query = (
+        f'resource.labels.namespace_name:"{urllib.parse.quote(namespace)}"\n'
+        f'resource.labels.container_name="sequencer-core"\n'
+        f'textPayload =~ "DECISION_REACHED:.*proposer 0x0*{validator_id}"\n'
+        f'CAST(REGEXP_EXTRACT(textPayload, "height: (\\\\d+)"), "INT64") > {min_block_number}'
+    )
+    return get_logs_explorer_url(query, project_name)
+
+
+def get_validator_id(namespace: str, context: Optional[str]) -> str:
+    # Get current config and normalize it (e.g. " vs ') to ensure not showing bogus diffs.
+    original_config = get_configmap(namespace, context, Service.Core)
+    _, config_data = parse_config_from_yaml(original_config)
+
+    return config_data["validator_id"]
+
+
+def main():
+    usage_example = """
+Examples:
+  # Restart all nodes at once.
+  %(prog)s --namespace-prefix apollo-sepolia-integration --num-nodes 3 --feeder-url feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  %(prog)s -n apollo-sepolia-integration -m 3 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  
+  # Restart nodes with cluster prefix
+  %(prog)s -n apollo-sepolia-integration -m 3 -c my-cluster -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  
+  # Restart nodes starting from specific node index
+  %(prog)s -n apollo-sepolia-integration -m 3 -s 5 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  
+  # Use different feeder URL
+  %(prog)s -n apollo-sepolia-integration -m 3 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  
+  # Use namespace list instead of prefix (restart specific namespaces)
+  %(prog)s --namespace-list apollo-sepolia-integration-0 apollo-sepolia-integration-2 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  %(prog)s -N apollo-sepolia-integration-0 apollo-sepolia-integration-2 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  
+  # Use cluster list for multiple clusters (only works with namespace-list, not namespace-prefix)
+  %(prog)s -N apollo-sepolia-integration-0 apollo-sepolia-integration-1 -C cluster1 cluster2 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+  %(prog)s --namespace-list apollo-sepolia-integration-0 apollo-sepolia-integration-1 --cluster-list cluster1 cluster2 -f feeder.integration-sepolia.starknet.io --project-name my-gcp-project
+        """
+
+    args_builder = ApolloArgsParserBuilder(
+        "Restart all nodes using the value from the feeder URL",
+        usage_example,
+        include_restart_strategy=False,
+    )
+
+    args_builder.add_argument(
+        "-f",
+        "--feeder-url",
+        required=True,
+        type=str,
+        help="The feeder URL to get the current block from",
+    )
+
+    # TODO(guy.f): Remove this when we rely on metrics for restarting.
+    args_builder.add_argument(
+        "--project-name",
+        required=True,
+        help="The name of the project to get logs from.",
+    )
+
+    args = args_builder.build()
+
+    # Get current block number from feeder URL
+    current_block_number = get_current_block_number(args.feeder_url)
+    next_block_number = current_block_number + 1
+
+    print_colored(f"Current block number: {current_block_number}")
+    print_colored(f"Next block number: {next_block_number}")
+
+    config_overrides = {
+        "consensus_manager_config.cende_config.skip_write_height": next_block_number,
+        "consensus_manager_config.immediate_active_height": next_block_number,
+    }
+
+    namespace_list = get_namespace_list_from_args(args)
+    context_list = get_context_list_from_args(args)
+    if context_list is not None:
+        assert len(namespace_list) == len(
+            context_list
+        ), "namespace_list and context_list must have the same length"
+
+    # Generate logs explorer URLs if needed
+    post_restart_instructions = []
+
+    for namespace, context in zip(namespace_list, context_list or [None] * len(namespace_list)):
+        url = get_logs_explorer_url_for_proposal(
+            namespace,
+            get_validator_id(namespace, context),
+            # Feeder could be behind by up to 10 blocks, so we add 10 to the current block number.
+            current_block_number + 10,
+            args.project_name,
+        )
+        post_restart_instructions.append(
+            f"Please check logs and verify that the node has proposed a block that was accepted. Logs URL: {url}"
+        )
+
+    update_config_and_restart_nodes(
+        config_overrides,
+        namespace_list,
+        Service.Core,
+        context_list,
+        RestartStrategy.ALL_AT_ONCE,
+        post_restart_instructions,
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/prod/set_node_revert_mode.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+
+import sys
+from typing import Optional
+
+import urllib.parse
+from update_config_and_restart_nodes_lib import (
+    ApolloArgsParserBuilder,
+    RestartStrategy,
+    Service,
+    get_context_list_from_args,
+    get_current_block_number,
+    get_logs_explorer_url,
+    get_namespace_list_from_args,
+    print_colored,
+    print_error,
+    update_config_and_restart_nodes,
+)
+
+
+# TODO(guy.f): Remove this once we have metrics we use to decide based on.
+def get_logs_explorer_url_for_revert(
+    namespace: str,
+    block_number: int,
+    project_name: str,
+) -> str:
+    query = (
+        f'resource.labels.namespace_name:"{urllib.parse.quote(namespace)}"\n'
+        f'resource.labels.container_name="sequencer-core"\n'
+        f'textPayload =~ "Done reverting.*storage up to height {block_number}"'
+    )
+    return get_logs_explorer_url(query, project_name)
+
+
+def set_revert_mode(
+    namespace_list: list[str],
+    context_list: Optional[list[str]],
+    project_name: str,
+    should_revert: bool,
+    revert_up_to_block: int,
+    restart_strategy: RestartStrategy,
+):
+    config_overrides = {
+        "revert_config.should_revert": should_revert,
+        "revert_config.revert_up_to_and_including": revert_up_to_block,
+    }
+
+    post_restart_instructions = []
+    for namespace, context in zip(namespace_list, context_list):
+        url = get_logs_explorer_url_for_revert(namespace, revert_up_to_block, project_name)
+
+        post_restart_instructions.append(
+            f"Please check logs and verify that revert has completed (both in the batcher and for sync). Logs URL: {url}"
+        )
+
+    update_config_and_restart_nodes(
+        config_overrides,
+        namespace_list,
+        Service.Core,
+        context_list,
+        restart_strategy,
+        post_restart_instructions,
+    )
+
+
+def main():
+    usage_example = """
+Examples:
+  # Set revert mode up to a specific block
+  %(prog)s --namespace apollo-sepolia-integration --num-nodes 3 -t all_at_once --revert-only --revert_up_to_block 12345
+  %(prog)s -n apollo-sepolia-integration -N 3 -t one_by_one --revert-only -b 12345
+  
+  # Set revert mode using feeder URL to get current block
+  %(prog)s --namespace apollo-sepolia-integration --num-nodes 3 -t all_at_once --revert-only --feeder-url feeder.integration-sepolia.starknet.io   
+  %(prog)s -n apollo-sepolia-integration -N 3 -t one_by_one --revert-only -f feeder.integration-sepolia.starknet.io
+  
+  # Disable revert mode
+  %(prog)s --namespace apollo-sepolia-integration --num-nodes 3 -t all_at_once --disable-revert-only
+  %(prog)s -n apollo-sepolia-integration -N 3 -t one_by_one --disable-revert-only
+  
+  # Set revert mode with cluster prefix
+  %(prog)s -n apollo-sepolia-integration -N 3 -c my-cluster -t all_at_once --revert-only -b 12345
+  
+  # Set revert mode with feeder URL and cluster prefix
+  %(prog)s -n apollo-sepolia-integration -N 3 -c my-cluster -t one_by_one --revert-only -f feeder.integration-sepolia.starknet.io
+  
+  # Disable revert mode without restarting nodes
+  %(prog)s -n apollo-sepolia-integration -N 3 -t no_restart --disable-revert-only
+  
+  # Set revert mode with explicit restart
+  %(prog)s -n apollo-sepolia-integration -N 3 -t all_at_once --revert-only -b 12345
+  
+  # Set revert mode with feeder URL and explicit restart
+  %(prog)s -n apollo-sepolia-integration -N 3 -t one_by_one --revert-only -f feeder.integration-sepolia.starknet.io
+  
+  # Set revert mode starting from specific node index
+  %(prog)s -n apollo-sepolia-integration -N 3 -i 5 -t all_at_once --revert-only -b 12345
+  
+  # Set revert mode with feeder URL starting from specific node index
+  %(prog)s -n apollo-sepolia-integration -N 3 -i 5 -t one_by_one --revert-only -f feeder.integration-sepolia.starknet.io
+        """
+
+    args_builder = ApolloArgsParserBuilder(
+        "Sets or unsets the revert mode for the sequencer nodes", usage_example
+    )
+
+    revert_group = args_builder.parser.add_mutually_exclusive_group()
+    revert_group.add_argument("--revert-only", action="store_true", help="Enable revert mode")
+    revert_group.add_argument(
+        "--disable-revert-only", action="store_true", help="Disable revert mode"
+    )
+
+    args_builder.add_argument(
+        "-b",
+        "--revert-up-to-block",
+        type=int,
+        help="Block number up to which to revert (inclusive). Must be a positive integer.",
+    )
+
+    args_builder.add_argument(
+        "-f",
+        "--feeder-url",
+        type=str,
+        help="The feeder URL to get the current block from. We will revert all blocks above it.",
+    )
+
+    # TODO(guy.f): Remove this when we rely on metrics for restarting.
+    args_builder.add_argument(
+        "--project-name",
+        help="The name of the project to get logs from. If One_By_One strategy is used, this is required.",
+    )
+
+    args = args_builder.build()
+
+    if args.restart_strategy == RestartStrategy.ONE_BY_ONE and args.project_name is None:
+        print_error("Error: --project-name is required when using One_By_One strategy")
+        sys.exit(1)
+
+    should_revert = not args.disable_revert_only
+    if should_revert:
+        if args.feeder_url is None and args.revert_up_to_block is None:
+            print_error(
+                "Error: Either --feeder-url or --revert_up_to_block (-b) are required when reverting is requested."
+            )
+            sys.exit(1)
+        if args.feeder_url is not None and args.revert_up_to_block is not None:
+            print_error("Error: Cannot specify both --feeder-url and --revert_up_to_block (-b).")
+            sys.exit(1)
+
+    if args.disable_revert_only:
+        if args.feeder_url is not None:
+            print_error("Error: --feeder-url cannot be set when using --disable-revert-only")
+            sys.exit(1)
+        if args.revert_up_to_block is not None:
+            print_error("Error: --revert-up-to-block (-b) cannot be set when disabling revert.")
+            sys.exit(1)
+
+    namespace_list = get_namespace_list_from_args(args)
+    context_list = get_context_list_from_args(args)
+
+    should_disable_revert = not args.revert_only
+    if should_revert:
+        revert_up_to_block = (
+            args.revert_up_to_block
+            if args.revert_up_to_block is not None
+            else get_current_block_number(args.feeder_url)
+        )
+        f"\nEnabling revert mode up to (and including) block {revert_up_to_block}"
+        set_revert_mode(
+            namespace_list,
+            context_list,
+            args.project_name,
+            True,
+            revert_up_to_block,
+            args.restart_strategy,
+        )
+    if should_disable_revert:
+        print_colored(f"\nDisabling revert mode")
+        # Setting to max block to max u64.
+        set_revert_mode(
+            namespace_list,
+            context_list,
+            args.project_name,
+            False,
+            18446744073709551615,
+            args.restart_strategy,
+        )
+
+
+if __name__ == "__main__":
+    main()