Skip to content

Commit

Permalink
Merge branch 'master' into backup-blob-client
Browse files Browse the repository at this point in the history
  • Loading branch information
@ganeshvanahalli
ganeshvanahalli authored Mar 25, 2024
2 parents 8239168 + 5b1871c commit 23aae50
Show file tree
Hide file tree
Showing 28 changed files with 780 additions and 413 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,13 @@ jobs:
if: matrix.test-mode == 'defaults'
run: |
packages=`go list ./...`
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/...
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m
- name: run tests with race detection
if: matrix.test-mode == 'race'
run: |
packages=`go list ./...`
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race -timeout 30m
- name: run redis tests
if: matrix.test-mode == 'defaults'
Expand Down
145 changes: 79 additions & 66 deletions arbnode/batch_poster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,21 @@ type BatchPosterConfig struct {
// Batch post polling interval.
PollInterval time.Duration `koanf:"poll-interval" reload:"hot"`
// Batch posting error delay.
ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"`
CompressionLevel int `koanf:"compression-level" reload:"hot"`
DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"`
GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"`
DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"`
RedisUrl string `koanf:"redis-url"`
RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"`
ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"`
Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"`
IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"`
ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"`
L1BlockBound string `koanf:"l1-block-bound" reload:"hot"`
L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"`
UseAccessLists bool `koanf:"use-access-lists" reload:"hot"`
ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"`
CompressionLevel int `koanf:"compression-level" reload:"hot"`
DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"`
GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"`
DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"`
RedisUrl string `koanf:"redis-url"`
RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"`
ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"`
Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"`
IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"`
ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"`
L1BlockBound string `koanf:"l1-block-bound" reload:"hot"`
L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"`
UseAccessLists bool `koanf:"use-access-lists" reload:"hot"`
GasEstimateBaseFeeMultipleBips arbmath.Bips `koanf:"gas-estimate-base-fee-multiple-bips"`

gasRefunder common.Address
l1BlockBound l1BlockBound
Expand Down Expand Up @@ -194,6 +195,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
f.String(prefix+".l1-block-bound", DefaultBatchPosterConfig.L1BlockBound, "only post messages to batches when they're within the max future block/timestamp as of this L1 block tag (\"safe\", \"finalized\", \"latest\", or \"ignore\" to ignore this check)")
f.Duration(prefix+".l1-block-bound-bypass", DefaultBatchPosterConfig.L1BlockBoundBypass, "post batches even if not within the layer 1 future bounds if we're within this margin of the max delay")
f.Bool(prefix+".use-access-lists", DefaultBatchPosterConfig.UseAccessLists, "post batches with access lists to reduce gas usage (disabled for L3s)")
f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas")
redislock.AddConfigOptions(prefix+".redis-lock", f)
dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig)
genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname)
Expand All @@ -205,23 +207,24 @@ var DefaultBatchPosterConfig = BatchPosterConfig{
// This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go
MaxSize: 100000,
// TODO: is 1000 bytes an appropriate margin for error vs blob space efficiency?
Max4844BatchSize: blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
PollInterval: time.Second * 10,
ErrorDelay: time.Second * 10,
MaxDelay: time.Hour,
WaitForMaxDelay: false,
CompressionLevel: brotli.BestCompression,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 50_000,
Post4844Blobs: false,
IgnoreBlobPrice: false,
DataPoster: dataposter.DefaultDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
RedisLock: redislock.DefaultCfg,
Max4844BatchSize: blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
PollInterval: time.Second * 10,
ErrorDelay: time.Second * 10,
MaxDelay: time.Hour,
WaitForMaxDelay: false,
CompressionLevel: brotli.BestCompression,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 50_000,
Post4844Blobs: false,
IgnoreBlobPrice: false,
DataPoster: dataposter.DefaultDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
RedisLock: redislock.DefaultCfg,
GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2,
}

var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{
Expand All @@ -233,24 +236,25 @@ var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{
}

var TestBatchPosterConfig = BatchPosterConfig{
Enable: true,
MaxSize: 100000,
Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize,
PollInterval: time.Millisecond * 10,
ErrorDelay: time.Millisecond * 10,
MaxDelay: 0,
WaitForMaxDelay: false,
CompressionLevel: 2,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 10_000,
Post4844Blobs: true,
IgnoreBlobPrice: false,
DataPoster: dataposter.TestDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
Enable: true,
MaxSize: 100000,
Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize,
PollInterval: time.Millisecond * 10,
ErrorDelay: time.Millisecond * 10,
MaxDelay: 0,
WaitForMaxDelay: false,
CompressionLevel: 2,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 10_000,
Post4844Blobs: true,
IgnoreBlobPrice: false,
DataPoster: dataposter.TestDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2,
}

type BatchPosterOpts struct {
Expand Down Expand Up @@ -846,11 +850,12 @@ func (b *BatchPoster) encodeAddBatch(
var ErrNormalGasEstimationFailed = errors.New("normal gas estimation failed")

type estimateGasParams struct {
From common.Address `json:"from"`
To *common.Address `json:"to"`
Data hexutil.Bytes `json:"data"`
AccessList types.AccessList `json:"accessList"`
BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"`
From common.Address `json:"from"`
To *common.Address `json:"to"`
Data hexutil.Bytes `json:"data"`
MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"`
AccessList types.AccessList `json:"accessList"`
BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"`
}

func estimateGas(client rpc.ClientInterface, ctx context.Context, params estimateGasParams) (uint64, error) {
Expand All @@ -861,28 +866,35 @@ func estimateGas(client rpc.ClientInterface, ctx context.Context, params estimat

func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, delayedMessages uint64, realData []byte, realBlobs []kzg4844.Blob, realNonce uint64, realAccessList types.AccessList) (uint64, error) {
config := b.config()
rpcClient := b.l1Reader.Client()
rawRpcClient := rpcClient.Client()
useNormalEstimation := b.dataPoster.MaxMempoolTransactions() == 1
if !useNormalEstimation {
// Check if we can use normal estimation anyways because we're at the latest nonce
latestNonce, err := b.l1Reader.Client().NonceAt(ctx, b.dataPoster.Sender(), nil)
latestNonce, err := rpcClient.NonceAt(ctx, b.dataPoster.Sender(), nil)
if err != nil {
return 0, err
}
useNormalEstimation = latestNonce == realNonce
}
rawRpcClient := b.l1Reader.Client().Client()
latestHeader, err := rpcClient.HeaderByNumber(ctx, nil)
if err != nil {
return 0, err
}
maxFeePerGas := arbmath.BigMulByBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips)
if useNormalEstimation {
_, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs)
if err != nil {
return 0, fmt.Errorf("failed to compute real blob commitments: %w", err)
}
// If we're at the latest nonce, we can skip the special future tx estimate stuff
gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: realData,
BlobHashes: realBlobHashes,
AccessList: realAccessList,
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: realData,
MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
BlobHashes: realBlobHashes,
AccessList: realAccessList,
})
if err != nil {
return 0, fmt.Errorf("%w: %w", ErrNormalGasEstimationFailed, err)
Expand All @@ -903,10 +915,11 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte,
return 0, fmt.Errorf("failed to compute blob commitments: %w", err)
}
gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: data,
BlobHashes: blobHashes,
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: data,
MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
BlobHashes: blobHashes,
// This isn't perfect because we're probably estimating the batch at a different sequence number,
// but it should overestimate rather than underestimate which is fine.
AccessList: realAccessList,
Expand Down Expand Up @@ -959,7 +972,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
}
var use4844 bool
config := b.config()
if config.Post4844Blobs && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil {
if config.Post4844Blobs && b.daWriter == nil && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil {
arbOSVersion, err := b.arbOSVersionGetter.ArbOSVersionForMessageNumber(arbutil.MessageIndex(arbmath.SaturatingUSub(uint64(batchPosition.MessageCount), 1)))
if err != nil {
return false, err
Expand Down
23 changes: 21 additions & 2 deletions arbnode/dataposter/data_poster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"github.com/offchainlabs/nitro/util/arbmath"
"github.com/offchainlabs/nitro/util/blobs"
"github.com/offchainlabs/nitro/util/headerreader"
"github.com/offchainlabs/nitro/util/rpcclient"
"github.com/offchainlabs/nitro/util/signature"
"github.com/offchainlabs/nitro/util/stopwaiter"
"github.com/spf13/pflag"
Expand Down Expand Up @@ -364,7 +365,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi

weightDiff := arbmath.MinInt(newCumulativeWeight-confirmedWeight, (nextNonce-unconfirmedNonce)*params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
if weightDiff > cfg.MaxMempoolWeight {
return fmt.Errorf("%w: transaction nonce: %d, transaction cumulative weight: %d, unconfirmed nonce: %d, confirmed weight: %d, new mempool weight: %d, max mempool weight: %d", ErrExceedsMaxMempoolSize, nextNonce, newCumulativeWeight, unconfirmedNonce, confirmedWeight, weightDiff, cfg.MaxMempoolTransactions)
return fmt.Errorf("%w: transaction nonce: %d, transaction cumulative weight: %d, unconfirmed nonce: %d, confirmed weight: %d, new mempool weight: %d, max mempool weight: %d", ErrExceedsMaxMempoolSize, nextNonce, newCumulativeWeight, unconfirmedNonce, confirmedWeight, weightDiff, cfg.MaxMempoolWeight)
}
}
return nil
Expand Down Expand Up @@ -600,6 +601,20 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u
newBlobFeeCap = arbmath.BigDivByUint(newBlobCost, blobGasUsed)
}

if config.MaxFeeBidMultipleBips > 0 {
// Limit the fee caps to be no greater than max(MaxFeeBidMultipleBips, minRbf)
maxNonBlobFee := arbmath.BigMulByBips(currentNonBlobFee, config.MaxFeeBidMultipleBips)
if lastTx != nil {
maxNonBlobFee = arbmath.BigMax(maxNonBlobFee, arbmath.BigMulByBips(lastTx.GasFeeCap(), minRbfIncrease))
}
maxBlobFee := arbmath.BigMulByBips(currentBlobFee, config.MaxFeeBidMultipleBips)
if lastTx != nil && lastTx.BlobGasFeeCap() != nil {
maxBlobFee = arbmath.BigMax(maxBlobFee, arbmath.BigMulByBips(lastTx.BlobGasFeeCap(), minRbfIncrease))
}
newBaseFeeCap = arbmath.BigMin(newBaseFeeCap, maxNonBlobFee)
newBlobFeeCap = arbmath.BigMin(newBlobFeeCap, maxBlobFee)
}

if arbmath.BigGreaterThan(newTipCap, newBaseFeeCap) {
log.Info(
"reducing new tip cap to new basefee cap",
Expand Down Expand Up @@ -812,7 +827,7 @@ func (p *DataPoster) sendTx(ctx context.Context, prevTx *storage.QueuedTransacti
return err
}
if err := p.client.SendTransaction(ctx, newTx.FullTx); err != nil {
if !strings.Contains(err.Error(), "already known") && !strings.Contains(err.Error(), "nonce too low") {
if !rpcclient.IsAlreadyKnownError(err) && !strings.Contains(err.Error(), "nonce too low") {
log.Warn("DataPoster failed to send transaction", "err", err, "nonce", newTx.FullTx.Nonce(), "feeCap", newTx.FullTx.GasFeeCap(), "tipCap", newTx.FullTx.GasTipCap(), "blobFeeCap", newTx.FullTx.BlobGasFeeCap(), "gas", newTx.FullTx.Gas())
return err
}
Expand Down Expand Up @@ -1116,6 +1131,7 @@ type DataPosterConfig struct {
MinBlobTxTipCapGwei float64 `koanf:"min-blob-tx-tip-cap-gwei" reload:"hot"`
MaxTipCapGwei float64 `koanf:"max-tip-cap-gwei" reload:"hot"`
MaxBlobTxTipCapGwei float64 `koanf:"max-blob-tx-tip-cap-gwei" reload:"hot"`
MaxFeeBidMultipleBips arbmath.Bips `koanf:"max-fee-bid-multiple-bips" reload:"hot"`
NonceRbfSoftConfs uint64 `koanf:"nonce-rbf-soft-confs" reload:"hot"`
AllocateMempoolBalance bool `koanf:"allocate-mempool-balance" reload:"hot"`
UseDBStorage bool `koanf:"use-db-storage"`
Expand Down Expand Up @@ -1170,6 +1186,7 @@ func DataPosterConfigAddOptions(prefix string, f *pflag.FlagSet, defaultDataPost
f.Float64(prefix+".min-blob-tx-tip-cap-gwei", defaultDataPosterConfig.MinBlobTxTipCapGwei, "the minimum tip cap to post EIP-4844 blob carrying transactions at")
f.Float64(prefix+".max-tip-cap-gwei", defaultDataPosterConfig.MaxTipCapGwei, "the maximum tip cap to post transactions at")
f.Float64(prefix+".max-blob-tx-tip-cap-gwei", defaultDataPosterConfig.MaxBlobTxTipCapGwei, "the maximum tip cap to post EIP-4844 blob carrying transactions at")
f.Uint64(prefix+".max-fee-bid-multiple-bips", uint64(defaultDataPosterConfig.MaxFeeBidMultipleBips), "the maximum multiple of the current price to bid for a transaction's fees (may be exceeded due to min rbf increase, 0 = unlimited)")
f.Uint64(prefix+".nonce-rbf-soft-confs", defaultDataPosterConfig.NonceRbfSoftConfs, "the maximum probable reorg depth, used to determine when a transaction will no longer likely need replaced-by-fee")
f.Bool(prefix+".allocate-mempool-balance", defaultDataPosterConfig.AllocateMempoolBalance, "if true, don't put transactions in the mempool that spend a total greater than the batch poster's balance")
f.Bool(prefix+".use-db-storage", defaultDataPosterConfig.UseDBStorage, "uses database storage when enabled")
Expand Down Expand Up @@ -1211,6 +1228,7 @@ var DefaultDataPosterConfig = DataPosterConfig{
MinBlobTxTipCapGwei: 1, // default geth minimum, and relays aren't likely to accept lower values given propagation time
MaxTipCapGwei: 5,
MaxBlobTxTipCapGwei: 1, // lower than normal because 4844 rbf is a minimum of a 2x
MaxFeeBidMultipleBips: arbmath.OneInBips * 10,
NonceRbfSoftConfs: 1,
AllocateMempoolBalance: true,
UseDBStorage: true,
Expand Down Expand Up @@ -1244,6 +1262,7 @@ var TestDataPosterConfig = DataPosterConfig{
MinBlobTxTipCapGwei: 1,
MaxTipCapGwei: 5,
MaxBlobTxTipCapGwei: 1,
MaxFeeBidMultipleBips: arbmath.OneInBips * 10,
NonceRbfSoftConfs: 1,
AllocateMempoolBalance: true,
UseDBStorage: false,
Expand Down
1 change: 1 addition & 0 deletions arbnode/dataposter/externalsigner/externalsigner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ func (a *SignTxArgs) ToTransaction() *types.Transaction {
Commitments: a.Commitments,
Proofs: a.Proofs,
},
ChainID: uint256.NewInt(a.ChainID.ToInt().Uint64()),
})
}

Expand Down
3 changes: 2 additions & 1 deletion arbnode/dataposter/externalsigner/externalsigner_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ var (
)
dynamicFeeTx = types.NewTx(
&types.DynamicFeeTx{
ChainID: big.NewInt(1337),
Nonce: 13,
GasTipCap: big.NewInt(1),
GasFeeCap: big.NewInt(1),
Expand Down Expand Up @@ -63,7 +64,7 @@ func TestToTranssaction(t *testing.T) {
t.Fatalf("TxToSignTxArgs() unexpected error: %v", err)
}
got := signTxArgs.ToTransaction()
hasher := types.LatestSignerForChainID(tc.tx.ChainId())
hasher := types.LatestSignerForChainID(nil)
if h, g := hasher.Hash(tc.tx), hasher.Hash(got); h != g {
t.Errorf("ToTransaction() got hash: %v want: %v", g, h)
}
Expand Down
Loading

0 comments on commit 23aae50

Please sign in to comment.