diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 389fa5021..8cdaacd93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,7 @@ jobs: "🚀 ccloud/fm-aws-cloudwatch-logs-source ccloud/fm-aws-s3-sink ccloud/fm-aws-s3-source ccloud/fm-azure-event-hubs-source ccloud/fm-azure-service-bus-source ccloud/fm-databricks-delta-lake-sink ccloud/fm-gcp-gcs-sink ccloud/fm-gcp-gcs-source ccloud/fm-aws-dynamodb-sink ccloud/fm-azure-data-lake-storage-gen2-sink ccloud/fm-azure-functions-sink ccloud/fm-azure-synapse-analytics-sink ccloud/fm-azure-cosmosdb-sink ccloud/fm-azure-cosmosdb-source ccloud/fully-managed-datadog-metrics-sink", - "🚀 ccloud/fm-salesforce-cdc-source ccloud/fm-salesforce-platform-events-sink ccloud/fm-salesforce-pushtopics-source ccloud/fm-salesforce-sobject-sink ccloud/fm-snowflake-sink ccloud/fm-gcp-pubsub-source ccloud/fm-gcp-spanner-sink ccloud/fm-aws-cloudwatch-metrics-sink ccloud/fm-gcp-bigquery-legacy-sink ccloud/fm-gcp-bigquery-v2-sink ccloud/fm-github-source ccloud/fm-jira-source ccloud/fm-gcp-cloud-functions-sink ccloud/fm-gcp-bigtable-sink ccloud/fm-pagerduty-sink ccloud/fm-azure-log-analytics-sink", + "🚀 ccloud/fm-salesforce-cdc-source ccloud/fm-salesforce-platform-events-sink ccloud/fm-salesforce-pushtopics-source ccloud/fm-salesforce-sobject-sink ccloud/fm-snowflake-sink ccloud/fm-gcp-pubsub-source ccloud/fm-gcp-spanner-sink ccloud/fm-aws-cloudwatch-metrics-sink ccloud/fm-gcp-bigquery-legacy-sink ccloud/fm-gcp-bigquery-v2-sink ccloud/fm-github-source ccloud/fm-jira-source ccloud/fm-gcp-cloud-functions-legacy-sink ccloud/fm-gcp-cloud-functions-gen2-sink ccloud/fm-gcp-bigtable-sink ccloud/fm-pagerduty-sink ccloud/fm-azure-log-analytics-sink", "🚀 ccloud/fm-aws-kinesis-source ccloud/fm-aws-redshift-sink ccloud/fm-aws-sqs-source ccloud/fm-aws-lambda-sink ccloud/fm-azure-blob-storage-sink ccloud/fm-azure-blob-storage-source ccloud/fm-azure-cognitive-search-sink ccloud/fm-mongodb-atlas-source ccloud/fm-mongodb-atlas-sink ccloud/fm-salesforce-bulkapi-source ccloud/fm-salesforce-bulkapi-2-0-source ccloud/custom-connector-connect-aws-s3-sink", diff --git a/ccloud/fm-gcp-bigtable-sink/fully-managed-gcp-bigtable-sink.sh b/ccloud/fm-gcp-bigtable-sink/fully-managed-gcp-bigtable-sink.sh index 6c3b4e731..941f9c135 100755 --- a/ccloud/fm-gcp-bigtable-sink/fully-managed-gcp-bigtable-sink.sh +++ b/ccloud/fm-gcp-bigtable-sink/fully-managed-gcp-bigtable-sink.sh @@ -32,11 +32,11 @@ cd - bootstrap_ccloud_environment -log "Creating big_query_stats topic" +log "Creating big_query_cloud_stats topic" set +e -playground topic delete --topic big_query_stats +playground topic delete --topic big_query_cloud_stats sleep 3 -playground topic create --topic big_query_stats +playground topic create --topic big_query_cloud_stats set -e log "Doing gsutil authentication" @@ -56,9 +56,9 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gcloud bigtab function cleanup_cloud_resources { - log "Delete GCP BigTable table kafka_big_query_stats" + log "Delete GCP BigTable table kafka_big_query_cloud_stats" check_if_continue - docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE deletetable kafka_big_query_stats + docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE deletetable kafka_big_query_cloud_stats log "Delete GCP BigTable instance $GCP_BIGTABLE_INSTANCE" check_if_continue @@ -71,8 +71,8 @@ EOF trap cleanup_cloud_resources EXIT -log "Sending messages to topic big_query_stats" -playground topic produce -t big_query_stats --nb-messages 1 --forced-value '{"users": {"name":"Bob","friends": "1000"}}' --key "simple-key-1" << 'EOF' +log "Sending messages to topic big_query_cloud_stats" +playground topic produce -t big_query_cloud_stats --nb-messages 1 --forced-value '{"users": {"name":"Bob","friends": "1000"}}' --key "simple-key-1" << 'EOF' { "type": "record", "name": "myrecord", @@ -97,10 +97,10 @@ playground topic produce -t big_query_stats --nb-messages 1 --forced-value '{"us ] } EOF -playground topic produce -t big_query_stats --nb-messages 1 --forced-value '{"users": {"name":"Jess","friends": "10000"}}' --key "simple-key-2" << 'EOF' +playground topic produce -t big_query_cloud_stats --nb-messages 1 --forced-value '{"users": {"name":"Jess","friends": "10000"}}' --key "simple-key-2" << 'EOF' {"type":"record","name":"myrecord","fields":[{"name":"users","type":{"name":"columnfamily","type":"record","fields":[{"name":"name","type":"string"},{"name":"friends","type":"string"}]}}]} EOF -playground topic produce -t big_query_stats --nb-messages 1 --forced-value '{"users": {"name":"John","friends": "10000"}}' --key "simple-key-3" << 'EOF' +playground topic produce -t big_query_cloud_stats --nb-messages 1 --forced-value '{"users": {"name":"John","friends": "10000"}}' --key "simple-key-3" << 'EOF' {"type":"record","name":"myrecord","fields":[{"name":"users","type":{"name":"columnfamily","type":"record","fields":[{"name":"name","type":"string"},{"name":"friends","type":"string"}]}}]} EOF @@ -118,7 +118,7 @@ playground connector create-or-update --connector $connector_name << EOF "kafka.auth.mode": "KAFKA_API_KEY", "kafka.api.key": "$CLOUD_KEY", "kafka.api.secret": "$CLOUD_SECRET", - "topics": "big_query_stats", + "topics": "big_query_cloud_stats", "gcp.bigtable.credentials.json" : "$GCP_KEYFILE_CONTENT", "gcp.bigtable.instance.id": "$GCP_BIGTABLE_INSTANCE", "gcp.bigtable.project.id": "$GCP_PROJECT", @@ -144,7 +144,7 @@ then docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json log "Verify data is in GCP BigTable" - docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_stats > /tmp/result.log 2>&1 + docker run -i --volumes-from gcloud-config google/cloud-sdk:latest cbt -project $GCP_PROJECT -instance $GCP_BIGTABLE_INSTANCE read kafka_big_query_cloud_stats > /tmp/result.log 2>&1 cat /tmp/result.log grep "Bob" /tmp/result.log fi diff --git a/ccloud/fm-gcp-cloud-functions-gen2-sink/README.md b/ccloud/fm-gcp-cloud-functions-gen2-sink/README.md new file mode 100644 index 000000000..8b4292c54 --- /dev/null +++ b/ccloud/fm-gcp-cloud-functions-gen2-sink/README.md @@ -0,0 +1,39 @@ +# Fully Managed Google Cloud Functions Gen 2 Sink connector + + + +## Objective + +Quickly test [Fully Managed Google Cloud Functions Gen 2 Sink](https://docs.confluent.io/cloud/current/connectors/cc-google-cloud-functions-gen2-sink.html) connector. + + +* Active Google Cloud Platform (GCP) account with authorization to create resources + +## Google Cloud Functions Setup + +* Navigate to the [Google Cloud Console](https://console.cloud.google.com/) + +* Go to the [Cloud Functions](https://console.cloud.google.com/functions) tab. + +![Cloud functions setup](Screenshot1.png) + +* Create a new function. Use the default code that is provided. + +![Cloud functions setup](Screenshot2.png) + +Note down the project id, the region (example `us-central1`), and the function name (example `function-1`) as they will be used later. + +N.B: make sure service account has `Cloud Run Invoker` role. + +## Prerequisites + +See [here](https://kafka-docker-playground.io/#/how-to-use?id=%f0%9f%8c%a4%ef%b8%8f-confluent-cloud-examples) + + +## How to run + +Simply run: + +```bash +$ just use command and search for fully-managed-google-cloud-functions .sh in this folder +``` diff --git a/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot1.png b/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot1.png new file mode 100644 index 000000000..ffa6a1996 Binary files /dev/null and b/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot1.png differ diff --git a/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot2.png b/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot2.png new file mode 100644 index 000000000..bf231c101 Binary files /dev/null and b/ccloud/fm-gcp-cloud-functions-gen2-sink/Screenshot2.png differ diff --git a/ccloud/fm-gcp-cloud-functions-gen2-sink/fully-managed-google-cloud-functions-gen2-sink.sh b/ccloud/fm-gcp-cloud-functions-gen2-sink/fully-managed-google-cloud-functions-gen2-sink.sh new file mode 100755 index 000000000..7750334d1 --- /dev/null +++ b/ccloud/fm-gcp-cloud-functions-gen2-sink/fully-managed-google-cloud-functions-gen2-sink.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +source ${DIR}/../../scripts/utils.sh +if [ -z "$GCP_PROJECT" ] +then + logerror "GCP_PROJECT is not set. Export it as environment variable or pass it as argument" + exit 1 +fi + +GCP_FUNCTION_REGION=${1:-europe-west2} +GCP_FUNCTION_FUNCTION=${2:-function-gen2} + +cd ../../ccloud/fm-gcp-cloud-functions-gen2-sink +GCP_KEYFILE="${DIR}/keyfile.json" +if [ ! -f ${GCP_KEYFILE} ] && [ -z "$GCP_KEYFILE_CONTENT" ] +then + logerror "ERROR: either the file ${GCP_KEYFILE} is not present or environment variable GCP_KEYFILE_CONTENT is not set!" + exit 1 +else + if [ -f ${GCP_KEYFILE} ] + then + GCP_KEYFILE_CONTENT=$(cat keyfile.json | jq -aRs . | sed 's/^"//' | sed 's/"$//') + else + log "Creating ${GCP_KEYFILE} based on environment variable GCP_KEYFILE_CONTENT" + echo -e "$GCP_KEYFILE_CONTENT" | sed 's/\\"/"/g' > ${GCP_KEYFILE} + fi +fi +cd - + +log "Doing gsutil authentication" +set +e +docker rm -f gcloud-config +set -e +docker run -i -v ${GCP_KEYFILE}:/tmp/keyfile.json --name gcloud-config google/cloud-sdk:latest gcloud auth activate-service-account --project ${GCP_PROJECT} --key-file /tmp/keyfile.json + +bootstrap_ccloud_environment + + + +set +e +playground topic delete --topic functions-gen2-messages +sleep 3 +playground topic create --topic functions-gen2-messages +set -e + + +log "Produce test data to the functions-gen2-messages topic in Kafka" +playground topic produce -t functions-gen2-messages --nb-messages 10 --forced-value '{"f1":"value%g"}' << 'EOF' +{ + "type": "record", + "name": "myrecord", + "fields": [ + { + "name": "f1", + "type": "string" + } + ] +} +EOF + +connector_name="GoogleCloudFunctionsGen2Sink_$USER" +set +e +playground connector delete --connector $connector_name > /dev/null 2>&1 +set -e + +log "Creating fully managed connector" +playground connector create-or-update --connector $connector_name << EOF +{ + "connector.class": "GoogleCloudFunctionsGen2Sink", + "name": "$connector_name", + "kafka.auth.mode": "KAFKA_API_KEY", + "kafka.api.key": "$CLOUD_KEY", + "kafka.api.secret": "$CLOUD_SECRET", + "topics": "functions-gen2-messages", + "gcf.name": "$GCP_FUNCTION_FUNCTION", + "gcf.project.id": "$GCP_PROJECT", + "gcf.region.name": "$GCP_FUNCTION_REGION", + "gcp.credentials.json" : "$GCP_KEYFILE_CONTENT", + "input.data.format" : "AVRO", + "tasks.max" : "1" +} +EOF +wait_for_ccloud_connector_up $connector_name 180 + +sleep 10 + + +connectorId=$(get_ccloud_connector_lcc $connector_name) + +log "Verifying topic success-$connectorId" +playground topic consume --topic success-$connectorId --min-expected-messages 10 --timeout 60 + +log "Do you want to delete the fully managed connector $connector_name ?" +check_if_continue + +playground connector delete --connector $connector_name diff --git a/ccloud/fm-gcp-cloud-functions-sink/stop.sh b/ccloud/fm-gcp-cloud-functions-gen2-sink/stop.sh similarity index 100% rename from ccloud/fm-gcp-cloud-functions-sink/stop.sh rename to ccloud/fm-gcp-cloud-functions-gen2-sink/stop.sh diff --git a/ccloud/fm-gcp-cloud-functions-sink/.gitignore b/ccloud/fm-gcp-cloud-functions-legacy-sink/.gitignore similarity index 100% rename from ccloud/fm-gcp-cloud-functions-sink/.gitignore rename to ccloud/fm-gcp-cloud-functions-legacy-sink/.gitignore diff --git a/ccloud/fm-gcp-cloud-functions-sink/README.md b/ccloud/fm-gcp-cloud-functions-legacy-sink/README.md similarity index 80% rename from ccloud/fm-gcp-cloud-functions-sink/README.md rename to ccloud/fm-gcp-cloud-functions-legacy-sink/README.md index 92d8d310f..bfae5f3cc 100644 --- a/ccloud/fm-gcp-cloud-functions-sink/README.md +++ b/ccloud/fm-gcp-cloud-functions-legacy-sink/README.md @@ -1,10 +1,10 @@ -# Fully Managed Google Cloud Functions Sink connector +# Fully Managed Google Cloud Functions (Legacy) Sink connector ## Objective -Quickly test [Fully Managed Google Cloud Functions Sink](https://docs.confluent.io/cloud/current/connectors/cc-google-functions-sink.html#google-cloud-functions-sink-connector-for-ccloud) connector. +Quickly test [Fully Managed Google Cloud Functions (Legacy) Sink](https://docs.confluent.io/cloud/current/connectors/cc-google-functions-sink.html) connector. * Active Google Cloud Platform (GCP) account with authorization to create resources diff --git a/ccloud/fm-gcp-cloud-functions-sink/Screenshot1.png b/ccloud/fm-gcp-cloud-functions-legacy-sink/Screenshot1.png similarity index 100% rename from ccloud/fm-gcp-cloud-functions-sink/Screenshot1.png rename to ccloud/fm-gcp-cloud-functions-legacy-sink/Screenshot1.png diff --git a/ccloud/fm-gcp-cloud-functions-sink/Screenshot2.png b/ccloud/fm-gcp-cloud-functions-legacy-sink/Screenshot2.png similarity index 100% rename from ccloud/fm-gcp-cloud-functions-sink/Screenshot2.png rename to ccloud/fm-gcp-cloud-functions-legacy-sink/Screenshot2.png diff --git a/ccloud/fm-gcp-cloud-functions-sink/fully-managed-google-cloud-functions-sink.sh b/ccloud/fm-gcp-cloud-functions-legacy-sink/fully-managed-google-cloud-functions-legacy-sink.sh similarity index 97% rename from ccloud/fm-gcp-cloud-functions-sink/fully-managed-google-cloud-functions-sink.sh rename to ccloud/fm-gcp-cloud-functions-legacy-sink/fully-managed-google-cloud-functions-legacy-sink.sh index c948a7bfc..5df1e6620 100755 --- a/ccloud/fm-gcp-cloud-functions-sink/fully-managed-google-cloud-functions-sink.sh +++ b/ccloud/fm-gcp-cloud-functions-legacy-sink/fully-managed-google-cloud-functions-legacy-sink.sh @@ -12,6 +12,7 @@ fi GCP_FUNCTION_REGION=${1:-europe-west2} GCP_FUNCTION_FUNCTION=${2:-function-1} +cd ../../ccloud/fm-gcp-cloud-functions-legacy-sink GCP_KEYFILE="${DIR}/keyfile.json" if [ ! -f ${GCP_KEYFILE} ] && [ -z "$GCP_KEYFILE_CONTENT" ] then @@ -26,6 +27,7 @@ else echo -e "$GCP_KEYFILE_CONTENT" | sed 's/\\"/"/g' > ${GCP_KEYFILE} fi fi +cd - bootstrap_ccloud_environment diff --git a/ccloud/fm-gcp-cloud-functions-legacy-sink/stop.sh b/ccloud/fm-gcp-cloud-functions-legacy-sink/stop.sh new file mode 100755 index 000000000..227b07f61 --- /dev/null +++ b/ccloud/fm-gcp-cloud-functions-legacy-sink/stop.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +source ${DIR}/../../scripts/utils.sh + +maybe_delete_ccloud_environment \ No newline at end of file diff --git a/ccloud/replicator/connect-onprem-to-cloud-avro.sh b/ccloud/replicator/connect-onprem-to-cloud-avro.sh index 081bc7094..6a4344887 100755 --- a/ccloud/replicator/connect-onprem-to-cloud-avro.sh +++ b/ccloud/replicator/connect-onprem-to-cloud-avro.sh @@ -12,6 +12,8 @@ playground start-environment --environment ccloud --docker-compose-override-file log "Creating topic in Confluent Cloud (auto.create.topics.enable=false)" set +e +playground topic delete --topic products-avro +sleep 3 playground topic create --topic products-avro set -e diff --git a/docs/content-template.md b/docs/content-template.md index e7a7475f4..83f3bbc40 100644 --- a/docs/content-template.md +++ b/docs/content-template.md @@ -218,7 +218,8 @@ - [Google BigQuery (Legacy) Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-bigquery-legacy-sink) :ccloud/fm-gcp-bigquery-legacy-sink: - [Google BigQuery V2 Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-bigquery-v2-sink) :ccloud/fm-gcp-bigquery-v2-sink: - [Google Cloud BigTable Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-bigtable-sink) :ccloud/fm-gcp-bigtable-sink: - - [Google Cloud Functions Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-cloud-functions-sink) :ccloud/fm-gcp-cloud-functions-sink: + - [Google Cloud Functions Legacy Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-cloud-functions-legacy-sink) :ccloud/fm-gcp-cloud-functions-legacy-sink: + - [Google Cloud Functions Gen 2 Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-cloud-functions-gen2-sink) :ccloud/fm-gcp-cloud-functions-gen2-sink: * [Google Cloud Spanner Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-spanner-sink) :ccloud/fm-gcp-spanner-sink: - [Google Cloud Storage Sink](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-gcs-sink) :ccloud/fm-gcp-gcs-sink: - [Google Cloud Storage Source](https://github.com/vdesabou/kafka-docker-playground/tree/master/ccloud/fm-gcp-gcs-source) :ccloud/fm-gcp-gcs-source: