From 78362e6a28d12abd45b2002e3c69cfb5c2f4912e Mon Sep 17 00:00:00 2001 From: Charlie Iorg Date: Tue, 26 Sep 2023 16:49:40 -0700 Subject: [PATCH 1/3] added documentaion and example job for using the dead letter queue with the elasticseach_bulk functionality --- asset/asset.json | 2 +- asset/package.json | 2 +- docs/operations/elasticsearch_bulk.md | 49 ++++++++++++++++++++++++--- package.json | 2 +- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/asset/asset.json b/asset/asset.json index 7ebbdde4..04fc9dab 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,4 +1,4 @@ { "name": "elasticsearch", - "version": "3.5.2" + "version": "3.5.3" } diff --git a/asset/package.json b/asset/package.json index 97c785aa..e7375811 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,7 +1,7 @@ { "name": "asset", "displayName": "Asset", - "version": "3.5.2", + "version": "3.5.3", "private": true, "description": "", "license": "MIT", diff --git a/docs/operations/elasticsearch_bulk.md b/docs/operations/elasticsearch_bulk.md index 547b15e9..6a966f96 100644 --- a/docs/operations/elasticsearch_bulk.md +++ b/docs/operations/elasticsearch_bulk.md @@ -28,9 +28,9 @@ Example Job "operations" : [ { "_op": "elasticsearch_reader", + "connection": "es-1", "index": "test_index", - "field": "created", - "connection": "es-1" + "date_field_name": "created" }, { "_op": "elasticsearch_bulk", @@ -77,7 +77,7 @@ Example Job { "_op": "elasticsearch_reader", "index": "test_index", - "field": "created", + "date_field_name": "created", "connection": "es-1" }, { @@ -124,7 +124,7 @@ Example Job { "_op": "elasticsearch_reader", "index": "test_index", - "field": "created", + "date_field_name": "created", "connection": "es-1" }, { @@ -260,3 +260,44 @@ this configuration will be expanded out to the long form underneath the hood ] } ``` + +### Dead Letter Queue Support +The elasticsearch_bulk processor supports the [dead letter queue api](https://github.com/terascope/kafka-assets/blob/master/docs/apis/kafka_dead_letter.md) as of version `3.5.0`. When the dead_letter_queue is triggered records that are rejected by elasticsearch with a `_bulk_sender_rejection` error are forwarded to the kafka topic specified in the dead letter queue api configs. Records that do not have the error are still written to the designated cluster as usual. + +To trigger this behavior add the property and value `_dead_letter_action: kafka_dead_letter` to the `elasticsearch_bulk` _op configs. + +Example Job: +```json +{ + "name" : "testing", + "workers" : 1, + "slicers" : 1, + "lifecycle" : "once", + "assets" : [ + "elasticsearch" + ], + "apis": [ + { + "_name": "kafka_dead_letter", + "connection": "KAFKA_CONNECTION", + "topic": "KAFKA_TOPIC", + "size": 10000 + } + ] + "operations" : [ + { + "_op": "elasticsearch_reader", + "index": "INDEX_NAME", + "date_field_name": "created", + "connection": "ES_CLUSTER_CONNECTION" + }, + { + "_op": "elasticsearch_bulk", + "connection": "ES_CLUSTER_CONNECTION", + "index": "INDEX_NAME", + "type": "events", + "_dead_letter_action": "kafka_dead_letter" + } + ] +} +``` diff --git a/package.json b/package.json index 4dc5e988..c819d6b1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-assets", "displayName": "Elasticsearch Assets", - "version": "3.5.2", + "version": "3.5.3", "private": true, "description": "bundle of processors for teraslice", "homepage": "https://github.com/terascope/elasticsearch-assets#readme", From 78a114a540744650c0b11829b6e1090815a67537 Mon Sep 17 00:00:00 2001 From: Charlie Iorg Date: Tue, 26 Sep 2023 16:52:42 -0700 Subject: [PATCH 2/3] word changes --- docs/operations/elasticsearch_bulk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operations/elasticsearch_bulk.md b/docs/operations/elasticsearch_bulk.md index 6a966f96..a57ee048 100644 --- a/docs/operations/elasticsearch_bulk.md +++ b/docs/operations/elasticsearch_bulk.md @@ -262,7 +262,7 @@ this configuration will be expanded out to the long form underneath the hood ``` ### Dead Letter Queue Support -The elasticsearch_bulk processor supports the [dead letter queue api](https://github.com/terascope/kafka-assets/blob/master/docs/apis/kafka_dead_letter.md) as of version `3.5.0`. When the dead_letter_queue is triggered records that are rejected by elasticsearch with a `_bulk_sender_rejection` error are forwarded to the kafka topic specified in the dead letter queue api configs. Records that do not have the error are still written to the designated cluster as usual. +The elasticsearch_bulk processor supports the [dead letter queue api](https://github.com/terascope/kafka-assets/blob/master/docs/apis/kafka_dead_letter.md) as of version `3.5.0`. When the dead_letter_queue functionality is active records that are rejected by elasticsearch with a `_bulk_sender_rejection` error are forwarded to the kafka topic specified in the dead letter queue api configs. Records that do not have the error are still written to the designated cluster as usual. To trigger this behavior add the property and value `_dead_letter_action: kafka_dead_letter` to the `elasticsearch_bulk` _op configs. From e370580d269ac72e62155a4bc8cf949d1c4879f9 Mon Sep 17 00:00:00 2001 From: Charlie Iorg Date: Tue, 26 Sep 2023 16:57:39 -0700 Subject: [PATCH 3/3] reverted version bump --- asset/asset.json | 2 +- asset/package.json | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/asset/asset.json b/asset/asset.json index 04fc9dab..7ebbdde4 100644 --- a/asset/asset.json +++ b/asset/asset.json @@ -1,4 +1,4 @@ { "name": "elasticsearch", - "version": "3.5.3" + "version": "3.5.2" } diff --git a/asset/package.json b/asset/package.json index e7375811..97c785aa 100644 --- a/asset/package.json +++ b/asset/package.json @@ -1,7 +1,7 @@ { "name": "asset", "displayName": "Asset", - "version": "3.5.3", + "version": "3.5.2", "private": true, "description": "", "license": "MIT", diff --git a/package.json b/package.json index c819d6b1..4dc5e988 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-assets", "displayName": "Elasticsearch Assets", - "version": "3.5.3", + "version": "3.5.2", "private": true, "description": "bundle of processors for teraslice", "homepage": "https://github.com/terascope/elasticsearch-assets#readme",