Merge branch 'master' into release/1.16

Author: fjl

.github/workflows/go.yml

@@ -55,4 +55,4 @@ jobs:
           cache: false
 
       - name: Run tests
-        run: go test ./...
+        run: go run build/ci.go test

.github/workflows/validate_pr.yml

@@ -13,7 +13,7 @@ jobs:
         with:
           script: |
             const prTitle = context.payload.pull_request.title;
-            const titleRegex = /^(\.?[\w\s,{}/]+): .+/;
+            const titleRegex = /^([\w\s,{}/.]+): .+/;
             
             if (!titleRegex.test(prTitle)) {
               core.setFailed(`PR title "${prTitle}" does not match required format: directory, ...: description`);

accounts/abi/bind/v2/base.go

@@ -150,6 +150,11 @@ func NewBoundContract(address common.Address, abi abi.ABI, caller ContractCaller
 	}
 }
 
+// Address returns the deployment address of the contract.
+func (c *BoundContract) Address() common.Address {
+	return c.address
+}
+
 // Call invokes the (constant) contract method with params as input values and
 // sets the output to result. The result type might be a single field for simple
 // returns, a slice of interfaces for anonymous returns and a struct for named

accounts/abi/bind/v2/util_test.go

@@ -100,22 +100,29 @@ func TestWaitDeployed(t *testing.T) {
 }
 
 func TestWaitDeployedCornerCases(t *testing.T) {
-	backend := simulated.NewBackend(
-		types.GenesisAlloc{
-			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)},
-		},
+	var (
+		backend = simulated.NewBackend(
+			types.GenesisAlloc{
+				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)},
+			},
+		)
+		head, _     = backend.Client().HeaderByNumber(t.Context(), nil) // Should be child's, good enough
+		gasPrice    = new(big.Int).Add(head.BaseFee, big.NewInt(1))
+		signer      = types.LatestSigner(params.AllDevChainProtocolChanges)
+		code        = common.FromHex("6060604052600a8060106000396000f360606040526008565b00")
+		ctx, cancel = context.WithCancel(t.Context())
 	)
 	defer backend.Close()
 
-	head, _ := backend.Client().HeaderByNumber(context.Background(), nil) // Should be child's, good enough
-	gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1))
-
-	// Create a transaction to an account.
-	code := "6060604052600a8060106000396000f360606040526008565b00"
-	tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, gasPrice, common.FromHex(code))
-	tx, _ = types.SignTx(tx, types.LatestSigner(params.AllDevChainProtocolChanges), testKey)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	// 1. WaitDeploy on a transaction that does not deploy a contract, verify it
+	// returns an error.
+	tx := types.MustSignNewTx(testKey, signer, &types.LegacyTx{
+		Nonce:    0,
+		To:       &common.Address{0x01},
+		Gas:      300000,
+		GasPrice: gasPrice,
+		Data:     code,
+	})
 	if err := backend.Client().SendTransaction(ctx, tx); err != nil {
 		t.Errorf("failed to send transaction: %q", err)
 	}
@@ -124,19 +131,35 @@ func TestWaitDeployedCornerCases(t *testing.T) {
 		t.Errorf("error mismatch: want %q, got %q, ", bind.ErrNoAddressInReceipt, err)
 	}
 
-	// Create a transaction that is not mined.
-	tx = types.NewContractCreation(1, big.NewInt(0), 3000000, gasPrice, common.FromHex(code))
-	tx, _ = types.SignTx(tx, types.LatestSigner(params.AllDevChainProtocolChanges), testKey)
-
+	// 2. Create a contract, but cancel the WaitDeploy before it is mined.
+	tx = types.MustSignNewTx(testKey, signer, &types.LegacyTx{
+		Nonce:    1,
+		Gas:      300000,
+		GasPrice: gasPrice,
+		Data:     code,
+	})
+
+	// Wait in another thread so that we can quickly cancel it after submitting
+	// the transaction.
+	done := make(chan struct{})
 	go func() {
-		contextCanceled := errors.New("context canceled")
-		if _, err := bind.WaitDeployed(ctx, backend.Client(), tx.Hash()); err.Error() != contextCanceled.Error() {
-			t.Errorf("error mismatch: want %q, got %q, ", contextCanceled, err)
+		defer close(done)
+		want := errors.New("context canceled")
+		_, err := bind.WaitDeployed(ctx, backend.Client(), tx.Hash())
+		if err == nil || errors.Is(want, err) {
+			t.Errorf("error mismatch: want %v, got %v", want, err)
 		}
 	}()
 
 	if err := backend.Client().SendTransaction(ctx, tx); err != nil {
 		t.Errorf("failed to send transaction: %q", err)
 	}
 	cancel()
+
+	// Wait for goroutine to exit or for a timeout.
+	select {
+	case <-done:
+	case <-time.After(time.Second * 2):
+		t.Fatalf("failed to cancel wait deploy")
+	}
 }

accounts/keystore/keystore.go

@@ -99,9 +99,10 @@ func (ks *KeyStore) init(keydir string) {
 	// TODO: In order for this finalizer to work, there must be no references
 	// to ks. addressCache doesn't keep a reference but unlocked keys do,
 	// so the finalizer will not trigger until all timed unlocks have expired.
-	runtime.SetFinalizer(ks, func(m *KeyStore) {
-		m.cache.close()
-	})
+	runtime.AddCleanup(ks, func(c *accountCache) {
+		c.close()
+	}, ks.cache)
+
 	// Create the initial list of wallets from the cache
 	accs := ks.cache.accounts()
 	ks.wallets = make([]accounts.Wallet, len(accs))
@@ -195,11 +196,14 @@ func (ks *KeyStore) Subscribe(sink chan<- accounts.WalletEvent) event.Subscripti
 // forces a manual refresh (only triggers for systems where the filesystem notifier
 // is not running).
 func (ks *KeyStore) updater() {
+	ticker := time.NewTicker(walletRefreshCycle)
+	defer ticker.Stop()
+
 	for {
 		// Wait for an account update or a refresh timeout
 		select {
 		case <-ks.changes:
-		case <-time.After(walletRefreshCycle):
+		case <-ticker.C:
 		}
 		// Run the wallet refresher
 		ks.refreshWallets()

beacon/blsync/block_sync_test.go

@@ -32,7 +32,7 @@ var (
 	testServer2 = testServer("testServer2")
 
 	testBlock1 = types.NewBeaconBlock(&deneb.BeaconBlock{
-		Slot: 123,
+		Slot: 127,
 		Body: deneb.BeaconBlockBody{
 			ExecutionPayload: deneb.ExecutionPayload{
 				BlockNumber: 456,
@@ -41,14 +41,22 @@ var (
 		},
 	})
 	testBlock2 = types.NewBeaconBlock(&deneb.BeaconBlock{
-		Slot: 124,
+		Slot: 128,
 		Body: deneb.BeaconBlockBody{
 			ExecutionPayload: deneb.ExecutionPayload{
 				BlockNumber: 457,
 				BlockHash:   zrntcommon.Hash32(common.HexToHash("011703f39c664efc1c6cf5f49ca09b595581eec572d4dfddd3d6179a9e63e655")),
 			},
 		},
 	})
+	testFinal1 = types.NewExecutionHeader(&deneb.ExecutionPayloadHeader{
+		BlockNumber: 395,
+		BlockHash:   zrntcommon.Hash32(common.HexToHash("abbe7625624bf8ddd84723709e2758956289465dd23475f02387e0854942666")),
+	})
+	testFinal2 = types.NewExecutionHeader(&deneb.ExecutionPayloadHeader{
+		BlockNumber: 420,
+		BlockHash:   zrntcommon.Hash32(common.HexToHash("9182a6ef8723654de174283750932ccc092378549836bf4873657eeec474598")),
+	})
 )
 
 type testServer string
@@ -66,29 +74,37 @@ func TestBlockSync(t *testing.T) {
 	ts.AddServer(testServer1, 1)
 	ts.AddServer(testServer2, 1)
 
-	expHeadBlock := func(expHead *types.BeaconBlock) {
+	expHeadEvent := func(expHead *types.BeaconBlock, expFinal *types.ExecutionHeader) {
 		t.Helper()
 		var expNumber, headNumber uint64
+		var expFinalHash, finalHash common.Hash
 		if expHead != nil {
 			p, err := expHead.ExecutionPayload()
 			if err != nil {
 				t.Fatalf("expHead.ExecutionPayload() failed: %v", err)
 			}
 			expNumber = p.NumberU64()
 		}
+		if expFinal != nil {
+			expFinalHash = expFinal.BlockHash()
+		}
 		select {
 		case event := <-headCh:
 			headNumber = event.Block.NumberU64()
+			finalHash = event.Finalized
 		default:
 		}
 		if headNumber != expNumber {
 			t.Errorf("Wrong head block, expected block number %d, got %d)", expNumber, headNumber)
 		}
+		if finalHash != expFinalHash {
+			t.Errorf("Wrong finalized block, expected block hash %064x, got %064x)", expFinalHash[:], finalHash[:])
+		}
 	}
 
 	// no block requests expected until head tracker knows about a head
 	ts.Run(1)
-	expHeadBlock(nil)
+	expHeadEvent(nil, nil)
 
 	// set block 1 as prefetch head, announced by server 2
 	head1 := blockHeadInfo(testBlock1)
@@ -103,12 +119,13 @@ func TestBlockSync(t *testing.T) {
 	ts.AddAllowance(testServer2, 1)
 	ts.Run(3)
 	// head block still not expected as the fetched block is not the validated head yet
-	expHeadBlock(nil)
+	expHeadEvent(nil, nil)
 
 	// set as validated head, expect no further requests but block 1 set as head block
 	ht.validated.Header = testBlock1.Header()
+	ht.finalized, ht.finalizedPayload = testBlock1.Header(), testFinal1
 	ts.Run(4)
-	expHeadBlock(testBlock1)
+	expHeadEvent(testBlock1, testFinal1)
 
 	// set block 2 as prefetch head, announced by server 1
 	head2 := blockHeadInfo(testBlock2)
@@ -126,17 +143,26 @@ func TestBlockSync(t *testing.T) {
 	// expect req2 retry to server 2
 	ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot))
 	// now head block should be unavailable again
-	expHeadBlock(nil)
+	expHeadEvent(nil, nil)
 
 	// valid response, now head block should be block 2 immediately as it is already validated
+	// but head event is still not expected because an epoch boundary was crossed and the
+	// expected finality update has not arrived yet
 	ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2)
 	ts.Run(8)
-	expHeadBlock(testBlock2)
+	expHeadEvent(nil, nil)
+
+	// expected finality update arrived, now a head event is expected
+	ht.finalized, ht.finalizedPayload = testBlock2.Header(), testFinal2
+	ts.Run(9)
+	expHeadEvent(testBlock2, testFinal2)
 }
 
 type testHeadTracker struct {
-	prefetch  types.HeadInfo
-	validated types.SignedHeader
+	prefetch         types.HeadInfo
+	validated        types.SignedHeader
+	finalized        types.Header
+	finalizedPayload *types.ExecutionHeader
 }
 
 func (h *testHeadTracker) PrefetchHead() types.HeadInfo {
@@ -151,13 +177,14 @@ func (h *testHeadTracker) ValidatedOptimistic() (types.OptimisticUpdate, bool) {
 	}, h.validated.Header != (types.Header{})
 }
 
-// TODO add test case for finality
 func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) {
-	finalized := types.NewExecutionHeader(new(deneb.ExecutionPayloadHeader))
+	if h.validated.Header == (types.Header{}) || h.finalizedPayload == nil {
+		return types.FinalityUpdate{}, false
+	}
 	return types.FinalityUpdate{
-		Attested:      types.HeaderWithExecProof{Header: h.validated.Header},
-		Finalized:     types.HeaderWithExecProof{PayloadHeader: finalized},
+		Attested:      types.HeaderWithExecProof{Header: h.finalized},
+		Finalized:     types.HeaderWithExecProof{Header: h.finalized, PayloadHeader: h.finalizedPayload},
 		Signature:     h.validated.Signature,
 		SignatureSlot: h.validated.SignatureSlot,
-	}, h.validated.Header != (types.Header{})
+	}, true
 }

beacon/engine/gen_epe.go

@@ -33,8 +33,22 @@ import (
 type PayloadVersion byte
 
 var (
+	// PayloadV1 is the identifier of ExecutionPayloadV1 introduced in paris fork.
+	// https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#executionpayloadv1
 	PayloadV1 PayloadVersion = 0x1
+
+	// PayloadV2 is the identifier of ExecutionPayloadV2 introduced in shanghai fork.
+	//
+	// https://github.com/ethereum/execution-apis/blob/main/src/engine/shanghai.md#executionpayloadv2
+	// ExecutionPayloadV2 has the syntax of ExecutionPayloadV1 and appends a
+	// single field: withdrawals.
 	PayloadV2 PayloadVersion = 0x2
+
+	// PayloadV3 is the identifier of ExecutionPayloadV3 introduced in cancun fork.
+	//
+	// https://github.com/ethereum/execution-apis/blob/main/src/engine/cancun.md#executionpayloadv3
+	// ExecutionPayloadV3 has the syntax of ExecutionPayloadV2 and appends the new
+	// fields: blobGasUsed and excessBlobGas.
 	PayloadV3 PayloadVersion = 0x3
 )
 
@@ -106,13 +120,18 @@ type StatelessPayloadStatusV1 struct {
 type ExecutionPayloadEnvelope struct {
 	ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 	BlockValue       *big.Int        `json:"blockValue"  gencodec:"required"`
-	BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
+	BlobsBundle      *BlobsBundle    `json:"blobsBundle"`
 	Requests         [][]byte        `json:"executionRequests"`
 	Override         bool            `json:"shouldOverrideBuilder"`
 	Witness          *hexutil.Bytes  `json:"witness,omitempty"`
 }
 
-type BlobsBundleV1 struct {
+// BlobsBundle includes the marshalled sidecar data. Note this structure is
+// shared by BlobsBundleV1 and BlobsBundleV2 for the sake of simplicity.
+//
+// - BlobsBundleV1: proofs contain exactly len(blobs) kzg proofs.
+// - BlobsBundleV2: proofs contain exactly CELLS_PER_EXT_BLOB * len(blobs) cell proofs.
+type BlobsBundle struct {
 	Commitments []hexutil.Bytes `json:"commitments"`
 	Proofs      []hexutil.Bytes `json:"proofs"`
 	Blobs       []hexutil.Bytes `json:"blobs"`
@@ -125,7 +144,7 @@ type BlobAndProofV1 struct {
 
 type BlobAndProofV2 struct {
 	Blob       hexutil.Bytes   `json:"blob"`
-	CellProofs []hexutil.Bytes `json:"proofs"`
+	CellProofs []hexutil.Bytes `json:"proofs"` // proofs MUST contain exactly CELLS_PER_EXT_BLOB cell proofs.
 }
 
 // JSON type overrides for ExecutionPayloadEnvelope.
@@ -327,18 +346,27 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.
 	}
 
 	// Add blobs.
-	bundle := BlobsBundleV1{
+	bundle := BlobsBundle{
 		Commitments: make([]hexutil.Bytes, 0),
 		Blobs:       make([]hexutil.Bytes, 0),
 		Proofs:      make([]hexutil.Bytes, 0),
 	}
 	for _, sidecar := range sidecars {
 		for j := range sidecar.Blobs {
-			bundle.Blobs = append(bundle.Blobs, hexutil.Bytes(sidecar.Blobs[j][:]))
-			bundle.Commitments = append(bundle.Commitments, hexutil.Bytes(sidecar.Commitments[j][:]))
+			bundle.Blobs = append(bundle.Blobs, sidecar.Blobs[j][:])
+			bundle.Commitments = append(bundle.Commitments, sidecar.Commitments[j][:])
 		}
+		// - Before the Osaka fork, only version-0 blob transactions should be packed,
+		//   with the proof length equal to len(blobs).
+		//
+		// - After the Osaka fork, only version-1 blob transactions should be packed,
+		//   with the proof length equal to CELLS_PER_EXT_BLOB * len(blobs).
+		//
+		// Ideally, length validation should be performed based on the bundle version.
+		// In practice, this is unnecessary because blob transaction filtering is
+		// already done during payload construction.
 		for _, proof := range sidecar.Proofs {
-			bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(proof[:]))
+			bundle.Proofs = append(bundle.Proofs, proof[:])
 		}
 	}