From 005db43aff93095a6408e93deff870aa627602d3 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 23 Jan 2023 14:19:00 +0100 Subject: [PATCH] Fix `ReaderConfig`'s default values and a few other fixes (#189) * Don't assume anything * Validate parameters and normalize default values * Add start offsets enum to the types info * Update JS API docs * Remove opinionated default value for MaxWait * Use kafka-go's internal constants * Add OffsetOutOfRangeError for extreme cases - won't be documented, because it is deprecated * Remove commented code * Add example script for consuming from a consumer group * Run prettier to fix formatting * Mention test_consumer_group.js in the README * Add emitted metrics to README * Fix error message --- README.md | 46 ++++++- api-docs/docs/README.md | 1 + api-docs/docs/classes/Connection.md | 10 +- api-docs/docs/classes/Reader.md | 6 +- api-docs/docs/classes/SchemaRegistry.md | 12 +- api-docs/docs/classes/Writer.md | 6 +- api-docs/docs/enums/BALANCERS.md | 10 +- api-docs/docs/enums/GROUP_BALANCERS.md | 6 +- api-docs/docs/enums/SCHEMA_TYPES.md | 10 +- api-docs/docs/enums/START_OFFSETS.md | 28 ++++ api-docs/docs/enums/SUBJECT_NAME_STRATEGY.md | 6 +- api-docs/docs/interfaces/BasicAuth.md | 4 +- api-docs/docs/interfaces/ConfigEntry.md | 4 +- api-docs/docs/interfaces/ConnectionConfig.md | 6 +- api-docs/docs/interfaces/ConsumeConfig.md | 2 +- api-docs/docs/interfaces/Container.md | 6 +- api-docs/docs/interfaces/Message.md | 16 +-- api-docs/docs/interfaces/ProduceConfig.md | 2 +- api-docs/docs/interfaces/ReaderConfig.md | 60 ++++----- api-docs/docs/interfaces/ReplicaAssignment.md | 4 +- api-docs/docs/interfaces/SASLConfig.md | 6 +- api-docs/docs/interfaces/Schema.md | 6 +- .../docs/interfaces/SchemaRegistryConfig.md | 6 +- api-docs/docs/interfaces/SubjectNameConfig.md | 8 +- api-docs/docs/interfaces/TLSConfig.md | 12 +- api-docs/docs/interfaces/TopicConfig.md | 10 +- api-docs/docs/interfaces/WriterConfig.md | 30 ++--- api-docs/index.d.ts | 8 +- error_codes.go | 15 ++- module.go | 10 ++ reader.go | 85 +++++++++---- scripts/test_consumer_group.js | 120 ++++++++++++++++++ scripts/test_json.js | 2 - 33 files changed, 410 insertions(+), 153 deletions(-) create mode 100644 api-docs/docs/enums/START_OFFSETS.md create mode 100644 scripts/test_consumer_group.js diff --git a/README.md b/README.md index 6531d71..12312d0 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ If you want to learn more about the extension, read the [article](https://k6.io/ - Create, list and delete [topics](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_topics.js) - Support for loading Avro schemas from [Schema Registry](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_avro_with_schema_registry.js) - Support for [byte array](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_bytes.js) for binary data (from binary protocols) -- Support consumption from all partitions with a group ID +- Support consumption from all partitions with a [group ID](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_consumer_group.js) - Support Kafka message compression: Gzip, [Snappy](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_json.js), Lz4 & Zstd - Support for sending messages with [no key](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_avro_no_schema_registry.js) - Support for k6 [thresholds](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_json.js) on custom Kafka metrics @@ -330,6 +330,46 @@ The example scripts are available as `test_.js` with more code a vus_max........................: 50 min=50 max=50 ``` +### Emitted Metrics + +| Metric | Type | Description | +| ---------------------------- | ------- | ----------------------------------------------------------------------- | +| kafka_reader_dial_count | Counter | Total number of times the reader tries to connect. | +| kafka_reader_fetches_count | Counter | Total number of times the reader fetches batches of messages. | +| kafka_reader_message_count | Counter | Total number of messages consumed. | +| kafka_reader_message_bytes | Counter | Total bytes consumed. | +| kafka_reader_rebalance_count | Counter | Total number of rebalances of a topic in a consumer group (deprecated). | +| kafka_reader_timeouts_count | Counter | Total number of timeouts occurred when reading. | +| kafka_reader_error_count | Counter | Total number of errors occurred when reading. | +| kafka_reader_dial_seconds | Trend | The time it takes to connect to the leader in a Kafka cluster. | +| kafka_reader_read_seconds | Trend | The time it takes to read a batch of message. | +| kafka_reader_wait_seconds | Trend | Waiting time before read a batch of messages. | +| kafka_reader_fetch_size | Counter | Total messages fetched. | +| kafka_reader_fetch_bytes | Counter | Total bytes fetched. | +| kafka_reader_offset | Gauge | Number of messages read after the given offset in a batch. | +| kafka_reader_lag | Gauge | The lag between the last message offset and the current read offset. | +| kafka_reader_fetch_bytes_min | Gauge | Minimum number of bytes fetched. | +| kafka_reader_fetch_bytes_max | Gauge | Maximum number of bytes fetched. | +| kafka_reader_fetch_wait_max | Gauge | The maximum time it takes to fetch a batch of messages. | +| kafka_reader_queue_length | Gauge | The queue length while reading batch of messages. | +| kafka_reader_queue_capacity | Gauge | The queue capacity while reading batch of messages. | +| kafka_writer_write_count | Counter | Total number of times the writer writes batches of messages. | +| kafka_writer_message_count | Counter | Total number of messages produced. | +| kafka_writer_message_bytes | Counter | Total bytes produced. | +| kafka_writer_error_count | Counter | Total number of errors occurred when writing. | +| kafka_writer_write_seconds | Trend | The time it takes writing messages. | +| kafka_writer_wait_seconds | Trend | Waiting time before writing messages. | +| kafka_writer_retries_count | Counter | Total number of attempts at writing messages. | +| kafka_writer_batch_size | Counter | Total batch size. | +| kafka_writer_batch_bytes | Counter | Total number of bytes in a batch of messages. | +| kafka_writer_attempts_max | Gauge | Maximum number of attempts at writing messages. | +| kafka_writer_batch_max | Gauge | Maximum batch size. | +| kafka_writer_batch_timeout | Gauge | Batch timeout. | +| kafka_writer_read_timeout | Gauge | Batch read timeout. | +| kafka_writer_write_timeout | Gauge | Batch write timeout. | +| kafka_writer_acks_required | Gauge | Required Acks. | +| kafka_writer_async | Rate | Async writer. | + ### FAQ 1. Why do I receive `Error writing messages`? @@ -372,6 +412,10 @@ The example scripts are available as `test_.js` with more code a For performance testing reasons, the `maxWait` of the `Reader` is set to 200ms. If you keep receiving this error, consider increasing it to a larger value. +8. How can I consume from multiple partitions on a single topic? + + You can configure your reader to consume from a (list of) topic(s) and its partitions using a consumer group. This can be achieve by setting `groupTopics`, `groupID` and a few other options for timeouts, intervals and lags. Have a look at the [`test_consumer_group.js`](https://github.com/mostafa/xk6-kafka/blob/main/scripts/test_consumer_group.js) example script. + ## Contributions, Issues and Feedback I'd be thrilled to receive contributions and feedback on this project. You're always welcome to create an issue if you find one (or many). I would do my best to address the issues. Also, feel free to contribute by opening a PR with changes, and I'll do my best to review and merge it as soon as I can. diff --git a/api-docs/docs/README.md b/api-docs/docs/README.md index a06959d..8af7f19 100644 --- a/api-docs/docs/README.md +++ b/api-docs/docs/README.md @@ -17,6 +17,7 @@ This documentation refers to the development version of the xk6-kafka project, w - [ISOLATION_LEVEL](enums/ISOLATION_LEVEL.md) - [SASL_MECHANISMS](enums/SASL_MECHANISMS.md) - [SCHEMA_TYPES](enums/SCHEMA_TYPES.md) +- [START_OFFSETS](enums/START_OFFSETS.md) - [SUBJECT_NAME_STRATEGY](enums/SUBJECT_NAME_STRATEGY.md) - [TLS_VERSIONS](enums/TLS_VERSIONS.md) diff --git a/api-docs/docs/classes/Connection.md b/api-docs/docs/classes/Connection.md index 977122b..00bc8bd 100644 --- a/api-docs/docs/classes/Connection.md +++ b/api-docs/docs/classes/Connection.md @@ -44,7 +44,7 @@ connection.close(); #### Defined in -[index.d.ts:355](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L355) +[index.d.ts:361](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L361) ## Methods @@ -64,7 +64,7 @@ connection.close(); #### Defined in -[index.d.ts:381](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L381) +[index.d.ts:387](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L387) --- @@ -89,7 +89,7 @@ Create a new topic. #### Defined in -[index.d.ts:362](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L362) +[index.d.ts:368](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L368) --- @@ -114,7 +114,7 @@ Delete a topic. #### Defined in -[index.d.ts:369](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L369) +[index.d.ts:375](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L375) --- @@ -133,4 +133,4 @@ List topics. #### Defined in -[index.d.ts:375](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L375) +[index.d.ts:381](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L381) diff --git a/api-docs/docs/classes/Reader.md b/api-docs/docs/classes/Reader.md index 336571b..e97b666 100644 --- a/api-docs/docs/classes/Reader.md +++ b/api-docs/docs/classes/Reader.md @@ -43,7 +43,7 @@ reader.close(); #### Defined in -[index.d.ts:314](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L314) +[index.d.ts:320](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L320) ## Methods @@ -63,7 +63,7 @@ reader.close(); #### Defined in -[index.d.ts:327](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L327) +[index.d.ts:333](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L333) --- @@ -88,4 +88,4 @@ Read messages from Kafka. #### Defined in -[index.d.ts:321](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L321) +[index.d.ts:327](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L327) diff --git a/api-docs/docs/classes/SchemaRegistry.md b/api-docs/docs/classes/SchemaRegistry.md index 1ec0839..337eb73 100644 --- a/api-docs/docs/classes/SchemaRegistry.md +++ b/api-docs/docs/classes/SchemaRegistry.md @@ -79,7 +79,7 @@ writer.produce({ #### Defined in -[index.d.ts:443](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L443) +[index.d.ts:449](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L449) ## Methods @@ -104,7 +104,7 @@ Create or update a schema on Schema Registry. #### Defined in -[index.d.ts:457](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L457) +[index.d.ts:463](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L463) --- @@ -129,7 +129,7 @@ Deserializes the given data and schema into its original form. #### Defined in -[index.d.ts:478](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L478) +[index.d.ts:484](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L484) --- @@ -154,7 +154,7 @@ Get a schema from Schema Registry by version and subject. #### Defined in -[index.d.ts:450](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L450) +[index.d.ts:456](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L456) --- @@ -179,7 +179,7 @@ Returns the subject name for the given SubjectNameConfig. #### Defined in -[index.d.ts:464](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L464) +[index.d.ts:470](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L470) --- @@ -204,4 +204,4 @@ Serializes the given data and schema into a byte array. #### Defined in -[index.d.ts:471](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L471) +[index.d.ts:477](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L477) diff --git a/api-docs/docs/classes/Writer.md b/api-docs/docs/classes/Writer.md index 2601e08..b625895 100644 --- a/api-docs/docs/classes/Writer.md +++ b/api-docs/docs/classes/Writer.md @@ -51,7 +51,7 @@ writer.close(); #### Defined in -[index.d.ts:272](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L272) +[index.d.ts:278](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L278) ## Methods @@ -71,7 +71,7 @@ writer.close(); #### Defined in -[index.d.ts:285](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L285) +[index.d.ts:291](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L291) --- @@ -96,4 +96,4 @@ Write messages to Kafka. #### Defined in -[index.d.ts:279](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L279) +[index.d.ts:285](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L285) diff --git a/api-docs/docs/enums/BALANCERS.md b/api-docs/docs/enums/BALANCERS.md index 69cd4e2..0618894 100644 --- a/api-docs/docs/enums/BALANCERS.md +++ b/api-docs/docs/enums/BALANCERS.md @@ -18,7 +18,7 @@ #### Defined in -[index.d.ts:63](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L63) +[index.d.ts:69](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L69) --- @@ -28,7 +28,7 @@ #### Defined in -[index.d.ts:62](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L62) +[index.d.ts:68](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L68) --- @@ -38,7 +38,7 @@ #### Defined in -[index.d.ts:61](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L61) +[index.d.ts:67](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L67) --- @@ -48,7 +48,7 @@ #### Defined in -[index.d.ts:64](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L64) +[index.d.ts:70](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L70) --- @@ -58,4 +58,4 @@ #### Defined in -[index.d.ts:60](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L60) +[index.d.ts:66](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L66) diff --git a/api-docs/docs/enums/GROUP_BALANCERS.md b/api-docs/docs/enums/GROUP_BALANCERS.md index e770861..f6f9cf2 100644 --- a/api-docs/docs/enums/GROUP_BALANCERS.md +++ b/api-docs/docs/enums/GROUP_BALANCERS.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:71](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L71) +[index.d.ts:77](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L77) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:69](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L69) +[index.d.ts:75](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L75) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:70](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L70) +[index.d.ts:76](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L76) diff --git a/api-docs/docs/enums/SCHEMA_TYPES.md b/api-docs/docs/enums/SCHEMA_TYPES.md index 15fde5c..3a3a446 100644 --- a/api-docs/docs/enums/SCHEMA_TYPES.md +++ b/api-docs/docs/enums/SCHEMA_TYPES.md @@ -18,7 +18,7 @@ #### Defined in -[index.d.ts:78](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L78) +[index.d.ts:84](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L84) --- @@ -28,7 +28,7 @@ #### Defined in -[index.d.ts:77](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L77) +[index.d.ts:83](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L83) --- @@ -38,7 +38,7 @@ #### Defined in -[index.d.ts:79](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L79) +[index.d.ts:85](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L85) --- @@ -48,7 +48,7 @@ #### Defined in -[index.d.ts:80](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L80) +[index.d.ts:86](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L86) --- @@ -58,4 +58,4 @@ #### Defined in -[index.d.ts:76](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L76) +[index.d.ts:82](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L82) diff --git a/api-docs/docs/enums/START_OFFSETS.md b/api-docs/docs/enums/START_OFFSETS.md new file mode 100644 index 0000000..0b78d57 --- /dev/null +++ b/api-docs/docs/enums/START_OFFSETS.md @@ -0,0 +1,28 @@ +# Enumeration: START_OFFSETS + +## Table of contents + +### Enumeration Members + +- [START_OFFSETS_FIRST_OFFSET](START_OFFSETS.md#start_offsets_first_offset) +- [START_OFFSETS_LAST_OFFSET](START_OFFSETS.md#start_offsets_last_offset) + +## Enumeration Members + +### START_OFFSETS_FIRST_OFFSET + +• **START_OFFSETS_FIRST_OFFSET** + +#### Defined in + +[index.d.ts:54](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L54) + +--- + +### START_OFFSETS_LAST_OFFSET + +• **START_OFFSETS_LAST_OFFSET** + +#### Defined in + +[index.d.ts:53](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L53) diff --git a/api-docs/docs/enums/SUBJECT_NAME_STRATEGY.md b/api-docs/docs/enums/SUBJECT_NAME_STRATEGY.md index cc9dfed..e23108e 100644 --- a/api-docs/docs/enums/SUBJECT_NAME_STRATEGY.md +++ b/api-docs/docs/enums/SUBJECT_NAME_STRATEGY.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:54](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L54) +[index.d.ts:60](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L60) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:53](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L53) +[index.d.ts:59](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L59) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:55](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L55) +[index.d.ts:61](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L61) diff --git a/api-docs/docs/interfaces/BasicAuth.md b/api-docs/docs/interfaces/BasicAuth.md index 88ece75..db090ad 100644 --- a/api-docs/docs/interfaces/BasicAuth.md +++ b/api-docs/docs/interfaces/BasicAuth.md @@ -15,7 +15,7 @@ #### Defined in -[index.d.ts:137](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L137) +[index.d.ts:143](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L143) --- @@ -25,4 +25,4 @@ #### Defined in -[index.d.ts:136](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L136) +[index.d.ts:142](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L142) diff --git a/api-docs/docs/interfaces/ConfigEntry.md b/api-docs/docs/interfaces/ConfigEntry.md index 20782fc..981b6be 100644 --- a/api-docs/docs/interfaces/ConfigEntry.md +++ b/api-docs/docs/interfaces/ConfigEntry.md @@ -15,7 +15,7 @@ #### Defined in -[index.d.ts:205](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L205) +[index.d.ts:211](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L211) --- @@ -25,4 +25,4 @@ #### Defined in -[index.d.ts:206](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L206) +[index.d.ts:212](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L212) diff --git a/api-docs/docs/interfaces/ConnectionConfig.md b/api-docs/docs/interfaces/ConnectionConfig.md index fd6c102..5b4b6f0 100644 --- a/api-docs/docs/interfaces/ConnectionConfig.md +++ b/api-docs/docs/interfaces/ConnectionConfig.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:192](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L192) +[index.d.ts:198](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L198) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:193](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L193) +[index.d.ts:199](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L199) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:194](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L194) +[index.d.ts:200](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L200) diff --git a/api-docs/docs/interfaces/ConsumeConfig.md b/api-docs/docs/interfaces/ConsumeConfig.md index c020393..893c7f2 100644 --- a/api-docs/docs/interfaces/ConsumeConfig.md +++ b/api-docs/docs/interfaces/ConsumeConfig.md @@ -14,4 +14,4 @@ #### Defined in -[index.d.ts:187](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L187) +[index.d.ts:193](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L193) diff --git a/api-docs/docs/interfaces/Container.md b/api-docs/docs/interfaces/Container.md index f665eb2..36226fe 100644 --- a/api-docs/docs/interfaces/Container.md +++ b/api-docs/docs/interfaces/Container.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:233](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L233) +[index.d.ts:239](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L239) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:234](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L234) +[index.d.ts:240](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L240) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:235](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L235) +[index.d.ts:241](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L241) diff --git a/api-docs/docs/interfaces/Message.md b/api-docs/docs/interfaces/Message.md index d284c13..929cf0d 100644 --- a/api-docs/docs/interfaces/Message.md +++ b/api-docs/docs/interfaces/Message.md @@ -25,7 +25,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:130](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L130) +[index.d.ts:136](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L136) --- @@ -35,7 +35,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:127](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L127) +[index.d.ts:133](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L133) --- @@ -45,7 +45,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:128](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L128) +[index.d.ts:134](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L134) --- @@ -55,7 +55,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:126](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L126) +[index.d.ts:132](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L132) --- @@ -65,7 +65,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:125](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L125) +[index.d.ts:131](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L131) --- @@ -75,7 +75,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:131](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L131) +[index.d.ts:137](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L137) --- @@ -85,7 +85,7 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:124](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L124) +[index.d.ts:130](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L130) --- @@ -95,4 +95,4 @@ Message format for producing messages to a topic. #### Defined in -[index.d.ts:129](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L129) +[index.d.ts:135](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L135) diff --git a/api-docs/docs/interfaces/ProduceConfig.md b/api-docs/docs/interfaces/ProduceConfig.md index f699dd2..857c51e 100644 --- a/api-docs/docs/interfaces/ProduceConfig.md +++ b/api-docs/docs/interfaces/ProduceConfig.md @@ -14,4 +14,4 @@ #### Defined in -[index.d.ts:149](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L149) +[index.d.ts:155](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L155) diff --git a/api-docs/docs/interfaces/ReaderConfig.md b/api-docs/docs/interfaces/ReaderConfig.md index bc45e97..11b5fec 100644 --- a/api-docs/docs/interfaces/ReaderConfig.md +++ b/api-docs/docs/interfaces/ReaderConfig.md @@ -42,7 +42,7 @@ #### Defined in -[index.d.ts:154](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L154) +[index.d.ts:160](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L160) --- @@ -52,7 +52,7 @@ #### Defined in -[index.d.ts:167](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L167) +[index.d.ts:173](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L173) --- @@ -62,7 +62,7 @@ #### Defined in -[index.d.ts:177](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L177) +[index.d.ts:183](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L183) --- @@ -72,7 +72,7 @@ #### Defined in -[index.d.ts:165](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L165) +[index.d.ts:171](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L171) --- @@ -82,7 +82,7 @@ #### Defined in -[index.d.ts:155](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L155) +[index.d.ts:161](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L161) --- @@ -92,7 +92,7 @@ #### Defined in -[index.d.ts:156](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L156) +[index.d.ts:162](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L162) --- @@ -102,7 +102,7 @@ #### Defined in -[index.d.ts:166](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L166) +[index.d.ts:172](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L172) --- @@ -112,7 +112,7 @@ #### Defined in -[index.d.ts:179](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L179) +[index.d.ts:185](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L185) --- @@ -122,7 +122,7 @@ #### Defined in -[index.d.ts:172](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L172) +[index.d.ts:178](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L178) --- @@ -132,7 +132,7 @@ #### Defined in -[index.d.ts:178](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L178) +[index.d.ts:184](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L184) --- @@ -142,7 +142,7 @@ #### Defined in -[index.d.ts:161](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L161) +[index.d.ts:167](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L167) --- @@ -152,7 +152,7 @@ #### Defined in -[index.d.ts:163](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L163) +[index.d.ts:169](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L169) --- @@ -162,7 +162,7 @@ #### Defined in -[index.d.ts:160](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L160) +[index.d.ts:166](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L166) --- @@ -172,7 +172,7 @@ #### Defined in -[index.d.ts:180](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L180) +[index.d.ts:186](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L186) --- @@ -182,7 +182,7 @@ #### Defined in -[index.d.ts:158](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L158) +[index.d.ts:164](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L164) --- @@ -192,7 +192,7 @@ #### Defined in -[index.d.ts:168](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L168) +[index.d.ts:174](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L174) --- @@ -202,7 +202,7 @@ #### Defined in -[index.d.ts:159](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L159) +[index.d.ts:165](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L165) --- @@ -212,7 +212,7 @@ #### Defined in -[index.d.ts:176](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L176) +[index.d.ts:182](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L182) --- @@ -222,7 +222,7 @@ #### Defined in -[index.d.ts:175](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L175) +[index.d.ts:181](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L181) --- @@ -232,7 +232,7 @@ #### Defined in -[index.d.ts:162](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L162) +[index.d.ts:168](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L168) --- @@ -242,7 +242,7 @@ #### Defined in -[index.d.ts:164](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L164) +[index.d.ts:170](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L170) --- @@ -252,7 +252,7 @@ #### Defined in -[index.d.ts:171](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L171) +[index.d.ts:177](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L177) --- @@ -262,7 +262,7 @@ #### Defined in -[index.d.ts:173](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L173) +[index.d.ts:179](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L179) --- @@ -272,7 +272,7 @@ #### Defined in -[index.d.ts:181](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L181) +[index.d.ts:187](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L187) --- @@ -282,17 +282,17 @@ #### Defined in -[index.d.ts:170](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L170) +[index.d.ts:176](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L176) --- ### startOffset -• **startOffset**: `number` +• **startOffset**: [`START_OFFSETS`](../enums/START_OFFSETS.md) #### Defined in -[index.d.ts:174](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L174) +[index.d.ts:180](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L180) --- @@ -302,7 +302,7 @@ #### Defined in -[index.d.ts:182](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L182) +[index.d.ts:188](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L188) --- @@ -312,7 +312,7 @@ #### Defined in -[index.d.ts:157](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L157) +[index.d.ts:163](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L163) --- @@ -322,4 +322,4 @@ #### Defined in -[index.d.ts:169](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L169) +[index.d.ts:175](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L175) diff --git a/api-docs/docs/interfaces/ReplicaAssignment.md b/api-docs/docs/interfaces/ReplicaAssignment.md index bf7cf05..ac0b554 100644 --- a/api-docs/docs/interfaces/ReplicaAssignment.md +++ b/api-docs/docs/interfaces/ReplicaAssignment.md @@ -15,7 +15,7 @@ #### Defined in -[index.d.ts:199](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L199) +[index.d.ts:205](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L205) --- @@ -25,4 +25,4 @@ #### Defined in -[index.d.ts:200](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L200) +[index.d.ts:206](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L206) diff --git a/api-docs/docs/interfaces/SASLConfig.md b/api-docs/docs/interfaces/SASLConfig.md index 4c758c2..c1a5233 100644 --- a/api-docs/docs/interfaces/SASLConfig.md +++ b/api-docs/docs/interfaces/SASLConfig.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:87](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L87) +[index.d.ts:93](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L93) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:86](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L86) +[index.d.ts:92](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L92) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:85](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L85) +[index.d.ts:91](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L91) diff --git a/api-docs/docs/interfaces/Schema.md b/api-docs/docs/interfaces/Schema.md index 283fc34..fc1730c 100644 --- a/api-docs/docs/interfaces/Schema.md +++ b/api-docs/docs/interfaces/Schema.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:220](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L220) +[index.d.ts:226](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L226) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:221](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L221) +[index.d.ts:227](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L227) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:222](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L222) +[index.d.ts:228](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L228) diff --git a/api-docs/docs/interfaces/SchemaRegistryConfig.md b/api-docs/docs/interfaces/SchemaRegistryConfig.md index 31814da..2169a14 100644 --- a/api-docs/docs/interfaces/SchemaRegistryConfig.md +++ b/api-docs/docs/interfaces/SchemaRegistryConfig.md @@ -16,7 +16,7 @@ #### Defined in -[index.d.ts:143](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L143) +[index.d.ts:149](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L149) --- @@ -26,7 +26,7 @@ #### Defined in -[index.d.ts:144](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L144) +[index.d.ts:150](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L150) --- @@ -36,4 +36,4 @@ #### Defined in -[index.d.ts:142](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L142) +[index.d.ts:148](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L148) diff --git a/api-docs/docs/interfaces/SubjectNameConfig.md b/api-docs/docs/interfaces/SubjectNameConfig.md index 747fae8..9979d05 100644 --- a/api-docs/docs/interfaces/SubjectNameConfig.md +++ b/api-docs/docs/interfaces/SubjectNameConfig.md @@ -17,7 +17,7 @@ #### Defined in -[index.d.ts:228](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L228) +[index.d.ts:234](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L234) --- @@ -27,7 +27,7 @@ #### Defined in -[index.d.ts:226](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L226) +[index.d.ts:232](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L232) --- @@ -37,7 +37,7 @@ #### Defined in -[index.d.ts:229](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L229) +[index.d.ts:235](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L235) --- @@ -47,4 +47,4 @@ #### Defined in -[index.d.ts:227](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L227) +[index.d.ts:233](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L233) diff --git a/api-docs/docs/interfaces/TLSConfig.md b/api-docs/docs/interfaces/TLSConfig.md index fa40e46..2ebfe95 100644 --- a/api-docs/docs/interfaces/TLSConfig.md +++ b/api-docs/docs/interfaces/TLSConfig.md @@ -19,7 +19,7 @@ #### Defined in -[index.d.ts:95](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L95) +[index.d.ts:101](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L101) --- @@ -29,7 +29,7 @@ #### Defined in -[index.d.ts:96](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L96) +[index.d.ts:102](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L102) --- @@ -39,7 +39,7 @@ #### Defined in -[index.d.ts:92](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L92) +[index.d.ts:98](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L98) --- @@ -49,7 +49,7 @@ #### Defined in -[index.d.ts:93](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L93) +[index.d.ts:99](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L99) --- @@ -59,7 +59,7 @@ #### Defined in -[index.d.ts:94](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L94) +[index.d.ts:100](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L100) --- @@ -69,4 +69,4 @@ #### Defined in -[index.d.ts:97](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L97) +[index.d.ts:103](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L103) diff --git a/api-docs/docs/interfaces/TopicConfig.md b/api-docs/docs/interfaces/TopicConfig.md index bcc81ed..2b3668c 100644 --- a/api-docs/docs/interfaces/TopicConfig.md +++ b/api-docs/docs/interfaces/TopicConfig.md @@ -18,7 +18,7 @@ #### Defined in -[index.d.ts:215](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L215) +[index.d.ts:221](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L221) --- @@ -28,7 +28,7 @@ #### Defined in -[index.d.ts:212](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L212) +[index.d.ts:218](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L218) --- @@ -38,7 +38,7 @@ #### Defined in -[index.d.ts:214](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L214) +[index.d.ts:220](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L220) --- @@ -48,7 +48,7 @@ #### Defined in -[index.d.ts:213](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L213) +[index.d.ts:219](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L219) --- @@ -58,4 +58,4 @@ #### Defined in -[index.d.ts:211](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L211) +[index.d.ts:217](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L217) diff --git a/api-docs/docs/interfaces/WriterConfig.md b/api-docs/docs/interfaces/WriterConfig.md index 9944926..99a1843 100644 --- a/api-docs/docs/interfaces/WriterConfig.md +++ b/api-docs/docs/interfaces/WriterConfig.md @@ -28,7 +28,7 @@ #### Defined in -[index.d.ts:104](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L104) +[index.d.ts:110](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L110) --- @@ -38,7 +38,7 @@ #### Defined in -[index.d.ts:105](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L105) +[index.d.ts:111](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L111) --- @@ -48,7 +48,7 @@ #### Defined in -[index.d.ts:108](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L108) +[index.d.ts:114](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L114) --- @@ -58,7 +58,7 @@ #### Defined in -[index.d.ts:107](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L107) +[index.d.ts:113](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L113) --- @@ -68,7 +68,7 @@ #### Defined in -[index.d.ts:109](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L109) +[index.d.ts:115](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L115) --- @@ -78,7 +78,7 @@ #### Defined in -[index.d.ts:102](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L102) +[index.d.ts:108](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L108) --- @@ -88,7 +88,7 @@ #### Defined in -[index.d.ts:113](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L113) +[index.d.ts:119](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L119) --- @@ -98,7 +98,7 @@ #### Defined in -[index.d.ts:116](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L116) +[index.d.ts:122](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L122) --- @@ -108,7 +108,7 @@ #### Defined in -[index.d.ts:106](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L106) +[index.d.ts:112](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L112) --- @@ -118,7 +118,7 @@ #### Defined in -[index.d.ts:110](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L110) +[index.d.ts:116](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L116) --- @@ -128,7 +128,7 @@ #### Defined in -[index.d.ts:111](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L111) +[index.d.ts:117](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L117) --- @@ -138,7 +138,7 @@ #### Defined in -[index.d.ts:114](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L114) +[index.d.ts:120](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L120) --- @@ -148,7 +148,7 @@ #### Defined in -[index.d.ts:115](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L115) +[index.d.ts:121](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L121) --- @@ -158,7 +158,7 @@ #### Defined in -[index.d.ts:103](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L103) +[index.d.ts:109](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L109) --- @@ -168,4 +168,4 @@ #### Defined in -[index.d.ts:112](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L112) +[index.d.ts:118](https://github.com/mostafa/xk6-kafka/blob/main/api-docs/index.d.ts#L118) diff --git a/api-docs/index.d.ts b/api-docs/index.d.ts index 276daee..f4140cf 100644 --- a/api-docs/index.d.ts +++ b/api-docs/index.d.ts @@ -48,6 +48,12 @@ export enum ISOLATION_LEVEL { ISOLATION_LEVEL_READ_COMMITTED = "isolation_level_read_committed", } +/* Start offsets for consuming messages from a consumer group. */ +export enum START_OFFSETS { + START_OFFSETS_LAST_OFFSET = "start_offset_last_offset", + START_OFFSETS_FIRST_OFFSET = "start_offset_first_offset", // default +} + /* Subject name strategy for storing a schema in Schema Registry. */ export enum SUBJECT_NAME_STRATEGY { TOPIC_NAME_STRATEGY = "TopicNameStrategy", @@ -171,7 +177,7 @@ export interface ReaderConfig { rebalanceTimeout: number; joinGroupBackoff: number; retentionTime: number; - startOffset: number; + startOffset: START_OFFSETS; readBackoffMin: number; readBackoffMax: number; connectLogger: boolean; diff --git a/error_codes.go b/error_codes.go index a02bfe6..2a85212 100644 --- a/error_codes.go +++ b/error_codes.go @@ -32,9 +32,11 @@ const ( failedDecodeBase64 errCode = 2009 // consumer. - failedSetOffset errCode = 3000 - failedReadMessage errCode = 3001 - noMoreMessages errCode = 3002 + failedSetOffset errCode = 3000 + failedReadMessage errCode = 3001 + noMoreMessages errCode = 3002 + partitionAndGroupID errCode = 3003 + topicAndGroupID errCode = 3004 // authentication. failedCreateDialerWithScram errCode = 4000 @@ -81,6 +83,13 @@ var ( // ErrUnknownSerdesType is used when a serdes type is not supported. ErrUnknownSerdesType = NewXk6KafkaError(invalidSerdeType, "Unknown serdes type", nil) + ErrPartitionAndGroupID = NewXk6KafkaError( + partitionAndGroupID, "Partition and groupID cannot be set at the same time", nil) + + ErrTopicAndGroupID = NewXk6KafkaError( + topicAndGroupID, + "When you specifiy groupID, you must set groupTopics instead of topic", nil) + // ErrNotEnoughArguments is used when a function is called with too few arguments. ErrNotEnoughArguments = errors.New("not enough arguments") diff --git a/module.go b/module.go index 2af6e91..949dbda 100644 --- a/module.go +++ b/module.go @@ -59,6 +59,12 @@ func init() { isolationLevelReadCommitted: kafkago.ReadCommitted, } + // Initialize the start offsets map. + StartOffsets = map[string]int64{ + lastOffset: kafkago.LastOffset, // The most recent offset available for a partition. + firstOffset: kafkago.FirstOffset, // The least recent offset available for a partition. + } + // Register the module namespace (aka. JS import path). modules.Register("k6/x/kafka", New()) } @@ -183,6 +189,10 @@ func (m *Module) defineConstants() { mustAddProp("ISOLATION_LEVEL_READ_UNCOMMITTED", isolationLevelReadUncommitted) mustAddProp("ISOLATION_LEVEL_READ_COMMITTED", isolationLevelReadCommitted) + // Start offsets + mustAddProp("FIRST_OFFSET", firstOffset) + mustAddProp("LAST_OFFSET", lastOffset) + // TopicNameStrategy types mustAddProp("TOPIC_NAME_STRATEGY", TopicNameStrategy) mustAddProp("RECORD_NAME_STRATEGY", RecordNameStrategy) diff --git a/reader.go b/reader.go index 6ad7162..e08a8c6 100644 --- a/reader.go +++ b/reader.go @@ -27,8 +27,18 @@ var ( IsolationLevels map[string]kafkago.IsolationLevel - MaxWait = time.Millisecond * 200 - RebalanceTimeout = time.Second * 5 + // Start offsets. + lastOffset = "start_offset_last_offset" + firstOffset = "start_offset_first_offset" + + StartOffsets map[string]int64 + + RebalanceTimeout = time.Second * 5 + HeartbeatInterval = time.Second * 3 + SessionTimeout = time.Second * 30 + PartitionWatchInterval = time.Second * 5 + JoinGroupBackoff = time.Second * 5 + RetentionTime = time.Hour * 24 ) type ReaderConfig struct { @@ -42,7 +52,7 @@ type ReaderConfig struct { GroupID string `json:"groupId"` Topic string `json:"topic"` IsolationLevel string `json:"isolationLevel"` - StartOffset int64 `json:"startOffset"` + StartOffset string `json:"startOffset"` Offset int64 `json:"offset"` Brokers []string `json:"brokers"` GroupTopics []string `json:"groupTopics"` @@ -59,6 +69,7 @@ type ReaderConfig struct { RetentionTime time.Duration `json:"retentionTime"` ReadBackoffMin time.Duration `json:"readBackoffMin"` ReadBackoffMax time.Duration `json:"readBackoffMax"` + OffsetOutOfRangeError bool `json:"offsetOutOfRangeError"` // deprecated, do not use SASL SASLConfig `json:"sasl"` TLS TLSConfig `json:"tls"` } @@ -137,10 +148,6 @@ func (k *Kafka) readerClass(call goja.ConstructorCall) *goja.Object { // reader creates a Kafka reader with the given configuration // nolint: funlen func (k *Kafka) reader(readerConfig *ReaderConfig) *kafkago.Reader { - if readerConfig.GroupID != "" { - readerConfig.Partition = 0 - } - dialer, err := GetDialer(readerConfig.SASL, readerConfig.TLS) if err != nil { if err.Unwrap() != nil { @@ -149,28 +156,53 @@ func (k *Kafka) reader(readerConfig *ReaderConfig) *kafkago.Reader { common.Throw(k.vu.Runtime(), err) } - if readerConfig.MaxWait == 0 { - readerConfig.MaxWait = MaxWait + if readerConfig.Partition != 0 && readerConfig.GroupID != "" { + common.Throw(k.vu.Runtime(), ErrPartitionAndGroupID) + } + + if readerConfig.Topic != "" && readerConfig.GroupID != "" { + common.Throw(k.vu.Runtime(), ErrTopicAndGroupID) } - if readerConfig.RebalanceTimeout == 0 { + if readerConfig.GroupID != "" && + len(readerConfig.GroupTopics) >= 0 && + readerConfig.HeartbeatInterval == 0 { + readerConfig.HeartbeatInterval = HeartbeatInterval + } + + if readerConfig.GroupID != "" && readerConfig.SessionTimeout == 0 { + readerConfig.SessionTimeout = SessionTimeout + } + + if readerConfig.GroupID != "" && readerConfig.RebalanceTimeout == 0 { readerConfig.RebalanceTimeout = RebalanceTimeout } - if readerConfig.QueueCapacity == 0 { - readerConfig.QueueCapacity = 1 + if readerConfig.GroupID != "" && readerConfig.JoinGroupBackoff == 0 { + readerConfig.JoinGroupBackoff = JoinGroupBackoff } - groupBalancers := []kafkago.GroupBalancer{} - for _, balancer := range readerConfig.GroupBalancers { - if b, ok := GroupBalancers[balancer]; ok { - groupBalancers = append(groupBalancers, b) - } + if readerConfig.GroupID != "" && readerConfig.PartitionWatchInterval == 0 { + readerConfig.PartitionWatchInterval = PartitionWatchInterval } - if len(groupBalancers) == 0 { - // Default to [Range, RoundRobin] if no balancer is specified - groupBalancers = append(groupBalancers, GroupBalancers[groupBalancerRange]) - groupBalancers = append(groupBalancers, GroupBalancers[groupBalancerRoundRobin]) + + if readerConfig.GroupID != "" && readerConfig.RetentionTime == 0 { + readerConfig.RetentionTime = RetentionTime + } + + var groupBalancers []kafkago.GroupBalancer + if readerConfig.GroupID != "" { + groupBalancers = make([]kafkago.GroupBalancer, 0, len(readerConfig.GroupBalancers)) + for _, balancer := range readerConfig.GroupBalancers { + if b, ok := GroupBalancers[balancer]; ok { + groupBalancers = append(groupBalancers, b) + } + } + if len(groupBalancers) == 0 { + // Default to [Range, RoundRobin] if no balancer is specified + groupBalancers = append(groupBalancers, GroupBalancers[groupBalancerRange]) + groupBalancers = append(groupBalancers, GroupBalancers[groupBalancerRoundRobin]) + } } isolationLevel := IsolationLevels[isolationLevelReadUncommitted] @@ -178,6 +210,14 @@ func (k *Kafka) reader(readerConfig *ReaderConfig) *kafkago.Reader { isolationLevel = IsolationLevels[readerConfig.IsolationLevel] } + var startOffset int64 + if readerConfig.GroupID != "" && readerConfig.StartOffset != "" { + startOffset = StartOffsets[firstOffset] // Default to FirstOffset + if s, ok := StartOffsets[readerConfig.StartOffset]; ok { + startOffset = s + } + } + consolidatedConfig := kafkago.ReaderConfig{ Brokers: readerConfig.Brokers, GroupID: readerConfig.GroupID, @@ -199,11 +239,12 @@ func (k *Kafka) reader(readerConfig *ReaderConfig) *kafkago.Reader { RebalanceTimeout: readerConfig.RebalanceTimeout, JoinGroupBackoff: readerConfig.JoinGroupBackoff, RetentionTime: readerConfig.RetentionTime, - StartOffset: readerConfig.StartOffset, + StartOffset: startOffset, ReadBackoffMin: readerConfig.ReadBackoffMin, ReadBackoffMax: readerConfig.ReadBackoffMax, IsolationLevel: isolationLevel, MaxAttempts: readerConfig.MaxAttempts, + OffsetOutOfRangeError: readerConfig.OffsetOutOfRangeError, Dialer: dialer, } diff --git a/scripts/test_consumer_group.js b/scripts/test_consumer_group.js new file mode 100644 index 0000000..0798bf4 --- /dev/null +++ b/scripts/test_consumer_group.js @@ -0,0 +1,120 @@ +/* + +This is a k6 test script that imports the xk6-kafka and +tests Kafka with a 200 JSON messages per iteration. + +*/ + +import { check } from "k6"; +// import * as kafka from "k6/x/kafka"; +import { + Writer, + Reader, + Connection, + SchemaRegistry, + CODEC_SNAPPY, + SCHEMA_TYPE_JSON, +} from "k6/x/kafka"; // import kafka extension + +// Prints module-level constants +// console.log(kafka); + +const brokers = ["localhost:9092"]; +const topic = "xk6_kafka_consumer_group_topic"; +const groupID = "my-group"; + +const writer = new Writer({ + brokers: brokers, + topic: topic, + compression: CODEC_SNAPPY, +}); +const reader = new Reader({ + brokers: brokers, + groupID: groupID, + groupTopics: [topic], +}); +const connection = new Connection({ + address: brokers[0], +}); +const schemaRegistry = new SchemaRegistry(); + +if (__VU == 0) { + connection.createTopic({ + topic: topic, + numPartitions: 3, + replicationFactor: 1, + configEntries: [ + { + configName: "compression.type", + configValue: CODEC_SNAPPY, + }, + ], + }); +} + +export const options = { + thresholds: { + // Base thresholds to see if the writer or reader is working + kafka_writer_error_count: ["count == 0"], + kafka_reader_error_count: ["count == 0"], + }, +}; + +export default function () { + let messages = []; + for (let i = 0; i < 100; i++) { + for (let partition = 0; partition < 3; partition++) { + messages.push({ + // The data type of the key is JSON + key: schemaRegistry.serialize({ + data: { + key: "value", + }, + schemaType: SCHEMA_TYPE_JSON, + }), + // The data type of the value is JSON + value: schemaRegistry.serialize({ + data: { + key: "value", + }, + schemaType: SCHEMA_TYPE_JSON, + }), + parition: partition, + }); + } + } + + writer.produce({ messages: messages }); + + // Read one message only + messages = reader.consume({ limit: 10 }); + + check(messages, { + "10 messages is received": (messages) => messages.length == 10, + }); + + check(messages[0], { + "Topic equals to xk6_kafka_consumer_group_topic": (msg) => + msg["topic"] == topic, + "Key contains key/value and is JSON": (msg) => + schemaRegistry.deserialize({ + data: msg.key, + schemaType: SCHEMA_TYPE_JSON, + }).key == "value", + "Value contains key/value and is JSON": (msg) => + typeof schemaRegistry.deserialize({ + data: msg.value, + schemaType: SCHEMA_TYPE_JSON, + }) == "object" && + schemaRegistry.deserialize({ + data: msg.value, + schemaType: SCHEMA_TYPE_JSON, + }).key == "value", + }); +} + +export function teardown(data) { + writer.close(); + reader.close(); + connection.close(); +} diff --git a/scripts/test_json.js b/scripts/test_json.js index dc710df..b038bf3 100644 --- a/scripts/test_json.js +++ b/scripts/test_json.js @@ -22,8 +22,6 @@ import { const brokers = ["localhost:9092"]; const topic = "xk6_kafka_json_topic"; -// const writer = new kafka.Writer(...); -// const reader = new kafka.Reader(...); const writer = new Writer({ brokers: brokers, topic: topic,