Skip to content

Commit

Permalink
Merge branch 'master' into postfix-signatures
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshuacolvin0
joshuacolvin0 committed Mar 25, 2024
2 parents 301bd84 + 0399e76 commit 88cf37e
Show file tree
Hide file tree
Showing 19 changed files with 952 additions and 251 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,13 +130,13 @@ jobs:
if: matrix.test-mode == 'defaults'
run: |
packages=`go list ./...`
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/...
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m
- name: run tests with race detection
if: matrix.test-mode == 'race'
run: |
packages=`go list ./...`
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race
gotestsum --format short-verbose --packages="$packages" --rerun-fails=1 -- -race -timeout 30m
- name: run redis tests
if: matrix.test-mode == 'defaults'
Expand Down
145 changes: 79 additions & 66 deletions arbnode/batch_poster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,21 @@ type BatchPosterConfig struct {
// Batch post polling interval.
PollInterval time.Duration `koanf:"poll-interval" reload:"hot"`
// Batch posting error delay.
ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"`
CompressionLevel int `koanf:"compression-level" reload:"hot"`
DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"`
GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"`
DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"`
RedisUrl string `koanf:"redis-url"`
RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"`
ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"`
Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"`
IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"`
ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"`
L1BlockBound string `koanf:"l1-block-bound" reload:"hot"`
L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"`
UseAccessLists bool `koanf:"use-access-lists" reload:"hot"`
ErrorDelay time.Duration `koanf:"error-delay" reload:"hot"`
CompressionLevel int `koanf:"compression-level" reload:"hot"`
DASRetentionPeriod time.Duration `koanf:"das-retention-period" reload:"hot"`
GasRefunderAddress string `koanf:"gas-refunder-address" reload:"hot"`
DataPoster dataposter.DataPosterConfig `koanf:"data-poster" reload:"hot"`
RedisUrl string `koanf:"redis-url"`
RedisLock redislock.SimpleCfg `koanf:"redis-lock" reload:"hot"`
ExtraBatchGas uint64 `koanf:"extra-batch-gas" reload:"hot"`
Post4844Blobs bool `koanf:"post-4844-blobs" reload:"hot"`
IgnoreBlobPrice bool `koanf:"ignore-blob-price" reload:"hot"`
ParentChainWallet genericconf.WalletConfig `koanf:"parent-chain-wallet"`
L1BlockBound string `koanf:"l1-block-bound" reload:"hot"`
L1BlockBoundBypass time.Duration `koanf:"l1-block-bound-bypass" reload:"hot"`
UseAccessLists bool `koanf:"use-access-lists" reload:"hot"`
GasEstimateBaseFeeMultipleBips arbmath.Bips `koanf:"gas-estimate-base-fee-multiple-bips"`

gasRefunder common.Address
l1BlockBound l1BlockBound
Expand Down Expand Up @@ -194,6 +195,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
f.String(prefix+".l1-block-bound", DefaultBatchPosterConfig.L1BlockBound, "only post messages to batches when they're within the max future block/timestamp as of this L1 block tag (\"safe\", \"finalized\", \"latest\", or \"ignore\" to ignore this check)")
f.Duration(prefix+".l1-block-bound-bypass", DefaultBatchPosterConfig.L1BlockBoundBypass, "post batches even if not within the layer 1 future bounds if we're within this margin of the max delay")
f.Bool(prefix+".use-access-lists", DefaultBatchPosterConfig.UseAccessLists, "post batches with access lists to reduce gas usage (disabled for L3s)")
f.Uint64(prefix+".gas-estimate-base-fee-multiple-bips", uint64(DefaultBatchPosterConfig.GasEstimateBaseFeeMultipleBips), "for gas estimation, use this multiple of the basefee (measured in basis points) as the max fee per gas")
redislock.AddConfigOptions(prefix+".redis-lock", f)
dataposter.DataPosterConfigAddOptions(prefix+".data-poster", f, dataposter.DefaultDataPosterConfig)
genericconf.WalletConfigAddOptions(prefix+".parent-chain-wallet", f, DefaultBatchPosterConfig.ParentChainWallet.Pathname)
Expand All @@ -205,23 +207,24 @@ var DefaultBatchPosterConfig = BatchPosterConfig{
// This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go
MaxSize: 100000,
// TODO: is 1000 bytes an appropriate margin for error vs blob space efficiency?
Max4844BatchSize: blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
PollInterval: time.Second * 10,
ErrorDelay: time.Second * 10,
MaxDelay: time.Hour,
WaitForMaxDelay: false,
CompressionLevel: brotli.BestCompression,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 50_000,
Post4844Blobs: false,
IgnoreBlobPrice: false,
DataPoster: dataposter.DefaultDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
RedisLock: redislock.DefaultCfg,
Max4844BatchSize: blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
PollInterval: time.Second * 10,
ErrorDelay: time.Second * 10,
MaxDelay: time.Hour,
WaitForMaxDelay: false,
CompressionLevel: brotli.BestCompression,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 50_000,
Post4844Blobs: false,
IgnoreBlobPrice: false,
DataPoster: dataposter.DefaultDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
RedisLock: redislock.DefaultCfg,
GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2,
}

var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{
Expand All @@ -233,24 +236,25 @@ var DefaultBatchPosterL1WalletConfig = genericconf.WalletConfig{
}

var TestBatchPosterConfig = BatchPosterConfig{
Enable: true,
MaxSize: 100000,
Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize,
PollInterval: time.Millisecond * 10,
ErrorDelay: time.Millisecond * 10,
MaxDelay: 0,
WaitForMaxDelay: false,
CompressionLevel: 2,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 10_000,
Post4844Blobs: true,
IgnoreBlobPrice: false,
DataPoster: dataposter.TestDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
Enable: true,
MaxSize: 100000,
Max4844BatchSize: DefaultBatchPosterConfig.Max4844BatchSize,
PollInterval: time.Millisecond * 10,
ErrorDelay: time.Millisecond * 10,
MaxDelay: 0,
WaitForMaxDelay: false,
CompressionLevel: 2,
DASRetentionPeriod: time.Hour * 24 * 15,
GasRefunderAddress: "",
ExtraBatchGas: 10_000,
Post4844Blobs: true,
IgnoreBlobPrice: false,
DataPoster: dataposter.TestDataPosterConfig,
ParentChainWallet: DefaultBatchPosterL1WalletConfig,
L1BlockBound: "",
L1BlockBoundBypass: time.Hour,
UseAccessLists: true,
GasEstimateBaseFeeMultipleBips: arbmath.OneInBips * 3 / 2,
}

type BatchPosterOpts struct {
Expand Down Expand Up @@ -846,11 +850,12 @@ func (b *BatchPoster) encodeAddBatch(
var ErrNormalGasEstimationFailed = errors.New("normal gas estimation failed")

type estimateGasParams struct {
From common.Address `json:"from"`
To *common.Address `json:"to"`
Data hexutil.Bytes `json:"data"`
AccessList types.AccessList `json:"accessList"`
BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"`
From common.Address `json:"from"`
To *common.Address `json:"to"`
Data hexutil.Bytes `json:"data"`
MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"`
AccessList types.AccessList `json:"accessList"`
BlobHashes []common.Hash `json:"blobVersionedHashes,omitempty"`
}

func estimateGas(client rpc.ClientInterface, ctx context.Context, params estimateGasParams) (uint64, error) {
Expand All @@ -861,28 +866,35 @@ func estimateGas(client rpc.ClientInterface, ctx context.Context, params estimat

func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte, delayedMessages uint64, realData []byte, realBlobs []kzg4844.Blob, realNonce uint64, realAccessList types.AccessList) (uint64, error) {
config := b.config()
rpcClient := b.l1Reader.Client()
rawRpcClient := rpcClient.Client()
useNormalEstimation := b.dataPoster.MaxMempoolTransactions() == 1
if !useNormalEstimation {
// Check if we can use normal estimation anyways because we're at the latest nonce
latestNonce, err := b.l1Reader.Client().NonceAt(ctx, b.dataPoster.Sender(), nil)
latestNonce, err := rpcClient.NonceAt(ctx, b.dataPoster.Sender(), nil)
if err != nil {
return 0, err
}
useNormalEstimation = latestNonce == realNonce
}
rawRpcClient := b.l1Reader.Client().Client()
latestHeader, err := rpcClient.HeaderByNumber(ctx, nil)
if err != nil {
return 0, err
}
maxFeePerGas := arbmath.BigMulByBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips)
if useNormalEstimation {
_, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs)
if err != nil {
return 0, fmt.Errorf("failed to compute real blob commitments: %w", err)
}
// If we're at the latest nonce, we can skip the special future tx estimate stuff
gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: realData,
BlobHashes: realBlobHashes,
AccessList: realAccessList,
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: realData,
MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
BlobHashes: realBlobHashes,
AccessList: realAccessList,
})
if err != nil {
return 0, fmt.Errorf("%w: %w", ErrNormalGasEstimationFailed, err)
Expand All @@ -903,10 +915,11 @@ func (b *BatchPoster) estimateGas(ctx context.Context, sequencerMessage []byte,
return 0, fmt.Errorf("failed to compute blob commitments: %w", err)
}
gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: data,
BlobHashes: blobHashes,
From: b.dataPoster.Sender(),
To: &b.seqInboxAddr,
Data: data,
MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
BlobHashes: blobHashes,
// This isn't perfect because we're probably estimating the batch at a different sequence number,
// but it should overestimate rather than underestimate which is fine.
AccessList: realAccessList,
Expand Down Expand Up @@ -959,7 +972,7 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
}
var use4844 bool
config := b.config()
if config.Post4844Blobs && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil {
if config.Post4844Blobs && b.daWriter == nil && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil {
arbOSVersion, err := b.arbOSVersionGetter.ArbOSVersionForMessageNumber(arbutil.MessageIndex(arbmath.SaturatingUSub(uint64(batchPosition.MessageCount), 1)))
if err != nil {
return false, err
Expand Down
41 changes: 33 additions & 8 deletions arbnode/dataposter/data_poster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ func (p *DataPoster) canPostWithNonce(ctx context.Context, nextNonce uint64, thi

weightDiff := arbmath.MinInt(newCumulativeWeight-confirmedWeight, (nextNonce-unconfirmedNonce)*params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
if weightDiff > cfg.MaxMempoolWeight {
return fmt.Errorf("%w: transaction nonce: %d, transaction cumulative weight: %d, unconfirmed nonce: %d, confirmed weight: %d, new mempool weight: %d, max mempool weight: %d", ErrExceedsMaxMempoolSize, nextNonce, newCumulativeWeight, unconfirmedNonce, confirmedWeight, weightDiff, cfg.MaxMempoolTransactions)
return fmt.Errorf("%w: transaction nonce: %d, transaction cumulative weight: %d, unconfirmed nonce: %d, confirmed weight: %d, new mempool weight: %d, max mempool weight: %d", ErrExceedsMaxMempoolSize, nextNonce, newCumulativeWeight, unconfirmedNonce, confirmedWeight, weightDiff, cfg.MaxMempoolWeight)
}
}
return nil
Expand Down Expand Up @@ -469,13 +469,10 @@ func (p *DataPoster) evalMaxFeeCapExpr(backlogOfBatches uint64, elapsed time.Dur
var big4 = big.NewInt(4)

// The dataPosterBacklog argument should *not* include extraBacklog (it's added in in this function)
func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit uint64, numBlobs uint64, lastTx *types.Transaction, dataCreatedAt time.Time, dataPosterBacklog uint64) (*big.Int, *big.Int, *big.Int, error) {
func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit uint64, numBlobs uint64, lastTx *types.Transaction, dataCreatedAt time.Time, dataPosterBacklog uint64, latestHeader *types.Header) (*big.Int, *big.Int, *big.Int, error) {
config := p.config()
dataPosterBacklog += p.extraBacklog()
latestHeader, err := p.headerReader.LastHeader(ctx)
if err != nil {
return nil, nil, nil, err
}

if latestHeader.BaseFee == nil {
return nil, nil, nil, fmt.Errorf("latest parent chain block %v missing BaseFee (either the parent chain does not have EIP-1559 or the parent chain node is not synced)", latestHeader.Number)
}
Expand Down Expand Up @@ -601,6 +598,20 @@ func (p *DataPoster) feeAndTipCaps(ctx context.Context, nonce uint64, gasLimit u
newBlobFeeCap = arbmath.BigDivByUint(newBlobCost, blobGasUsed)
}

if config.MaxFeeBidMultipleBips > 0 {
// Limit the fee caps to be no greater than max(MaxFeeBidMultipleBips, minRbf)
maxNonBlobFee := arbmath.BigMulByBips(currentNonBlobFee, config.MaxFeeBidMultipleBips)
if lastTx != nil {
maxNonBlobFee = arbmath.BigMax(maxNonBlobFee, arbmath.BigMulByBips(lastTx.GasFeeCap(), minRbfIncrease))
}
maxBlobFee := arbmath.BigMulByBips(currentBlobFee, config.MaxFeeBidMultipleBips)
if lastTx != nil && lastTx.BlobGasFeeCap() != nil {
maxBlobFee = arbmath.BigMax(maxBlobFee, arbmath.BigMulByBips(lastTx.BlobGasFeeCap(), minRbfIncrease))
}
newBaseFeeCap = arbmath.BigMin(newBaseFeeCap, maxNonBlobFee)
newBlobFeeCap = arbmath.BigMin(newBlobFeeCap, maxBlobFee)
}

if arbmath.BigGreaterThan(newTipCap, newBaseFeeCap) {
log.Info(
"reducing new tip cap to new basefee cap",
Expand Down Expand Up @@ -678,7 +689,12 @@ func (p *DataPoster) PostTransaction(ctx context.Context, dataCreatedAt time.Tim
return nil, fmt.Errorf("failed to update data poster balance: %w", err)
}

feeCap, tipCap, blobFeeCap, err := p.feeAndTipCaps(ctx, nonce, gasLimit, uint64(len(kzgBlobs)), nil, dataCreatedAt, 0)
latestHeader, err := p.headerReader.LastHeader(ctx)
if err != nil {
return nil, err
}

feeCap, tipCap, blobFeeCap, err := p.feeAndTipCaps(ctx, nonce, gasLimit, uint64(len(kzgBlobs)), nil, dataCreatedAt, 0, latestHeader)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -863,7 +879,12 @@ func updateGasCaps(tx *types.Transaction, newFeeCap, newTipCap, newBlobFeeCap *b

// The mutex must be held by the caller.
func (p *DataPoster) replaceTx(ctx context.Context, prevTx *storage.QueuedTransaction, backlogWeight uint64) error {
newFeeCap, newTipCap, newBlobFeeCap, err := p.feeAndTipCaps(ctx, prevTx.FullTx.Nonce(), prevTx.FullTx.Gas(), uint64(len(prevTx.FullTx.BlobHashes())), prevTx.FullTx, prevTx.Created, backlogWeight)
latestHeader, err := p.headerReader.LastHeader(ctx)
if err != nil {
return err
}

newFeeCap, newTipCap, newBlobFeeCap, err := p.feeAndTipCaps(ctx, prevTx.FullTx.Nonce(), prevTx.FullTx.Gas(), uint64(len(prevTx.FullTx.BlobHashes())), prevTx.FullTx, prevTx.Created, backlogWeight, latestHeader)
if err != nil {
return err
}
Expand Down Expand Up @@ -1117,6 +1138,7 @@ type DataPosterConfig struct {
MinBlobTxTipCapGwei float64 `koanf:"min-blob-tx-tip-cap-gwei" reload:"hot"`
MaxTipCapGwei float64 `koanf:"max-tip-cap-gwei" reload:"hot"`
MaxBlobTxTipCapGwei float64 `koanf:"max-blob-tx-tip-cap-gwei" reload:"hot"`
MaxFeeBidMultipleBips arbmath.Bips `koanf:"max-fee-bid-multiple-bips" reload:"hot"`
NonceRbfSoftConfs uint64 `koanf:"nonce-rbf-soft-confs" reload:"hot"`
AllocateMempoolBalance bool `koanf:"allocate-mempool-balance" reload:"hot"`
UseDBStorage bool `koanf:"use-db-storage"`
Expand Down Expand Up @@ -1171,6 +1193,7 @@ func DataPosterConfigAddOptions(prefix string, f *pflag.FlagSet, defaultDataPost
f.Float64(prefix+".min-blob-tx-tip-cap-gwei", defaultDataPosterConfig.MinBlobTxTipCapGwei, "the minimum tip cap to post EIP-4844 blob carrying transactions at")
f.Float64(prefix+".max-tip-cap-gwei", defaultDataPosterConfig.MaxTipCapGwei, "the maximum tip cap to post transactions at")
f.Float64(prefix+".max-blob-tx-tip-cap-gwei", defaultDataPosterConfig.MaxBlobTxTipCapGwei, "the maximum tip cap to post EIP-4844 blob carrying transactions at")
f.Uint64(prefix+".max-fee-bid-multiple-bips", uint64(defaultDataPosterConfig.MaxFeeBidMultipleBips), "the maximum multiple of the current price to bid for a transaction's fees (may be exceeded due to min rbf increase, 0 = unlimited)")
f.Uint64(prefix+".nonce-rbf-soft-confs", defaultDataPosterConfig.NonceRbfSoftConfs, "the maximum probable reorg depth, used to determine when a transaction will no longer likely need replaced-by-fee")
f.Bool(prefix+".allocate-mempool-balance", defaultDataPosterConfig.AllocateMempoolBalance, "if true, don't put transactions in the mempool that spend a total greater than the batch poster's balance")
f.Bool(prefix+".use-db-storage", defaultDataPosterConfig.UseDBStorage, "uses database storage when enabled")
Expand Down Expand Up @@ -1212,6 +1235,7 @@ var DefaultDataPosterConfig = DataPosterConfig{
MinBlobTxTipCapGwei: 1, // default geth minimum, and relays aren't likely to accept lower values given propagation time
MaxTipCapGwei: 5,
MaxBlobTxTipCapGwei: 1, // lower than normal because 4844 rbf is a minimum of a 2x
MaxFeeBidMultipleBips: arbmath.OneInBips * 10,
NonceRbfSoftConfs: 1,
AllocateMempoolBalance: true,
UseDBStorage: true,
Expand Down Expand Up @@ -1245,6 +1269,7 @@ var TestDataPosterConfig = DataPosterConfig{
MinBlobTxTipCapGwei: 1,
MaxTipCapGwei: 5,
MaxBlobTxTipCapGwei: 1,
MaxFeeBidMultipleBips: arbmath.OneInBips * 10,
NonceRbfSoftConfs: 1,
AllocateMempoolBalance: true,
UseDBStorage: false,
Expand Down
Loading

0 comments on commit 88cf37e

Please sign in to comment.