diff --git a/ccloud/fm-aws-s3-sink/fully-managed-s3-sink.sh b/ccloud/fm-aws-s3-sink/fully-managed-s3-sink.sh index 9618c43d8..f2ad731d3 100755 --- a/ccloud/fm-aws-s3-sink/fully-managed-s3-sink.sh +++ b/ccloud/fm-aws-s3-sink/fully-managed-s3-sink.sh @@ -116,7 +116,7 @@ aws s3 cp --only-show-errors --recursive s3://$AWS_BUCKET_NAME/$TAG/s3_topic /tm cp /tmp/s3_topic/*/*/*/*/s3_topic+0+0000000000.avro /tmp/s3_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp/:/tmp vdesabou/avro-tools tojson /tmp/s3_topic+0+0000000000.avro | grep value999 +playground tools read-avro-file --file /tmp/s3_topic+0+0000000000.avro | grep value999 log "Do you want to delete the fully managed connector $connector_name ?" check_if_continue diff --git a/ccloud/fm-azure-blob-storage-sink/fully-managed-azure-blob-storage-sink.sh b/ccloud/fm-azure-blob-storage-sink/fully-managed-azure-blob-storage-sink.sh index c2bfabc42..edece5274 100755 --- a/ccloud/fm-azure-blob-storage-sink/fully-managed-azure-blob-storage-sink.sh +++ b/ccloud/fm-azure-blob-storage-sink/fully-managed-azure-blob-storage-sink.sh @@ -136,7 +136,7 @@ playground connector show-lag --connector $connector_name # log "Getting one of the avro files locally and displaying content with avro-tools" # az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/year=2024/month=04/day=04/hour=08/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro -# docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro +# playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro log "Do you want to delete the fully managed connector $connector_name ?" diff --git a/ccloud/fm-azure-data-lake-storage-gen2-sink/fully-managed-azure-data-lake-storage-gen2-sink.sh b/ccloud/fm-azure-data-lake-storage-gen2-sink/fully-managed-azure-data-lake-storage-gen2-sink.sh index 8f8852fb3..db9875a34 100755 --- a/ccloud/fm-azure-data-lake-storage-gen2-sink/fully-managed-azure-data-lake-storage-gen2-sink.sh +++ b/ccloud/fm-azure-data-lake-storage-gen2-sink/fully-managed-azure-data-lake-storage-gen2-sink.sh @@ -159,7 +159,7 @@ playground connector show-lag --connector $connector_name # log "Getting one of the avro files locally and displaying content with avro-tools" # az storage blob download --container-name topics --name datalake_topic/partition=0/datalake_topic+0+0000000000.avro --file /tmp/datalake_topic+0+0000000000.avro --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}" -# docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro +# playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro log "Do you want to delete the fully managed connector $connector_name ?" check_if_continue diff --git a/ccloud/fm-gcp-gcs-sink/fully-managed-gcs-sink.sh b/ccloud/fm-gcp-gcs-sink/fully-managed-gcs-sink.sh index ebdb72d12..672211660 100755 --- a/ccloud/fm-gcp-gcs-sink/fully-managed-gcs-sink.sh +++ b/ccloud/fm-gcp-gcs-sink/fully-managed-gcs-sink.sh @@ -103,7 +103,7 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs: log "Getting one of the avro files locally and displaying content with avro-tools" docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/*/*/*/*/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro log "Do you want to delete the fully managed connector $connector_name ?" check_if_continue diff --git a/ccloud/fm-sftp-sink/fully-managed-sftp-sink.sh b/ccloud/fm-sftp-sink/fully-managed-sftp-sink.sh index e27f0e916..7c4cccad9 100755 --- a/ccloud/fm-sftp-sink/fully-managed-sftp-sink.sh +++ b/ccloud/fm-sftp-sink/fully-managed-sftp-sink.sh @@ -95,7 +95,7 @@ log "Listing content of ./upload/topics/test_sftp_sink" docker exec sftp-server bash -c "ls /home/foo/upload/topics/test_sftp_sink/*/*/*/*" docker exec sftp-server bash -c "cp /home/foo/upload/topics/test_sftp_sink/*/*/*/*/test_sftp_sink+0+0000000000.avro /tmp/" docker cp sftp-server:/tmp/test_sftp_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro log "Do you want to delete the fully managed connector $connector_name ?" diff --git a/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink-proxy.sh b/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink-proxy.sh index ce9b59948..3493b741f 100755 --- a/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink-proxy.sh +++ b/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink-proxy.sh @@ -127,4 +127,4 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${ log "Getting one of the avro files locally and displaying content with avro-tools" az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro \ No newline at end of file diff --git a/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink.sh b/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink.sh index 09b75e0f0..b6a9d87ec 100755 --- a/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink.sh +++ b/connect/connect-azure-blob-storage-sink/azure-blob-storage-sink.sh @@ -126,4 +126,4 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${ log "Getting one of the avro files locally and displaying content with avro-tools" az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro \ No newline at end of file diff --git a/connect/connect-azure-blob-storage-source/azure-blob-storage-source-backup-and-restore.sh b/connect/connect-azure-blob-storage-source/azure-blob-storage-source-backup-and-restore.sh index 3008e1e15..1e69a891b 100755 --- a/connect/connect-azure-blob-storage-source/azure-blob-storage-source-backup-and-restore.sh +++ b/connect/connect-azure-blob-storage-source/azure-blob-storage-source-backup-and-restore.sh @@ -135,7 +135,7 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${ log "Getting one of the avro files locally and displaying content with avro-tools" az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro log "Creating Azure Blob Storage Source connector" diff --git a/connect/connect-azure-blob-storage-source/azure-blob-storage-source-proxy-backup-and-restore.sh b/connect/connect-azure-blob-storage-source/azure-blob-storage-source-proxy-backup-and-restore.sh index 7fce8e5c9..336aaec28 100755 --- a/connect/connect-azure-blob-storage-source/azure-blob-storage-source-proxy-backup-and-restore.sh +++ b/connect/connect-azure-blob-storage-source/azure-blob-storage-source-proxy-backup-and-restore.sh @@ -141,7 +141,7 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${ log "Getting one of the avro files locally and displaying content with avro-tools" az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro log "Creating Azure Blob Storage Source connector" diff --git a/connect/connect-azure-data-lake-storage-gen1-sink/azure-data-lake-storage-gen1-sink.sh b/connect/connect-azure-data-lake-storage-gen1-sink/azure-data-lake-storage-gen1-sink.sh index 726909aff..962f21674 100755 --- a/connect/connect-azure-data-lake-storage-gen1-sink/azure-data-lake-storage-gen1-sink.sh +++ b/connect/connect-azure-data-lake-storage-gen1-sink/azure-data-lake-storage-gen1-sink.sh @@ -134,4 +134,4 @@ az dls fs list --account "${AZURE_DATALAKE_ACCOUNT_NAME}" --path /topics log "Getting one of the avro files locally and displaying content with avro-tools" az dls fs download --account "${AZURE_DATALAKE_ACCOUNT_NAME}" --overwrite --source-path /topics/datalake_topic/partition=0/datalake_topic+0+0000000000.avro --destination-path /tmp/datalake_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro \ No newline at end of file diff --git a/connect/connect-azure-data-lake-storage-gen2-sink/azure-data-lake-storage-gen2-sink.sh b/connect/connect-azure-data-lake-storage-gen2-sink/azure-data-lake-storage-gen2-sink.sh index 36d3b0259..07977bfbc 100755 --- a/connect/connect-azure-data-lake-storage-gen2-sink/azure-data-lake-storage-gen2-sink.sh +++ b/connect/connect-azure-data-lake-storage-gen2-sink/azure-data-lake-storage-gen2-sink.sh @@ -149,7 +149,7 @@ az storage fs file list --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}" --file-s log "Getting one of the avro files locally and displaying content with avro-tools" az storage blob download --container-name topics --name datalake_topic/partition=0/datalake_topic+0+0000000000.avro --file /tmp/datalake_topic+0+0000000000.avro --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}" -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro # keep AD app # log "Deleting active directory app" diff --git a/connect/connect-ftps-sink/README.md b/connect/connect-ftps-sink/README.md index f153738df..ce7246019 100644 --- a/connect/connect-ftps-sink/README.md +++ b/connect/connect-ftps-sink/README.md @@ -89,7 +89,7 @@ test_ftps_sink+0+0000000009.avro ``` ```bash -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_ftps_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_ftps_sink+0+0000000000.avro {"f1":"value1"} ``` diff --git a/connect/connect-ftps-sink/ftps-sink.sh b/connect/connect-ftps-sink/ftps-sink.sh index a8f048ac3..cc8107fb5 100755 --- a/connect/connect-ftps-sink/ftps-sink.sh +++ b/connect/connect-ftps-sink/ftps-sink.sh @@ -70,4 +70,4 @@ docker exec ftps-server bash -c "ls /home/vsftpd/bob/test_ftps_sink/partition\=0 docker cp ftps-server:/home/vsftpd/bob/test_ftps_sink/partition\=0/test_ftps_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_ftps_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_ftps_sink+0+0000000000.avro diff --git a/connect/connect-gcp-dataproc-sink/gcp-dataproc-sink.sh b/connect/connect-gcp-dataproc-sink/gcp-dataproc-sink.sh index e1e5c5fa4..d0a6e70d4 100755 --- a/connect/connect-gcp-dataproc-sink/gcp-dataproc-sink.sh +++ b/connect/connect-gcp-dataproc-sink/gcp-dataproc-sink.sh @@ -127,7 +127,7 @@ sleep 10 # docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro docker rm -f gcloud-config diff --git a/connect/connect-gcp-gcs-sink/README.md b/connect/connect-gcp-gcs-sink/README.md index e581d4dca..edbabdf33 100644 --- a/connect/connect-gcp-gcs-sink/README.md +++ b/connect/connect-gcp-gcs-sink/README.md @@ -133,7 +133,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro 19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable {"f1":"value1"} {"f1":"value2"} @@ -215,7 +215,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic-ssl/partition=0/gcs_topic-ssl+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic-ssl+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/gcs_topic-ssl+0+0000000000.avro 19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable {"f1":"This is a message sent with SSL authentication 1"} {"f1":"This is a message sent with SSL authentication 2"} @@ -270,7 +270,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic-sasl-ssl/partition=0/gcs_topic-sasl-ssl+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic-sasl-ssl+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/gcs_topic-sasl-ssl+0+0000000000.avro 19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable {"f1":"This is a message sent with SASL_SSL authentication 1"} {"f1":"This is a message sent with SASL_SSL authentication 2"} @@ -325,7 +325,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro {"f1":"This is a message sent with Kerberos GSSAPI authentication 1"} {"f1":"This is a message sent with Kerberos GSSAPI authentication 2"} {"f1":"This is a message sent with Kerberos GSSAPI authentication 3"} @@ -384,7 +384,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro {"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 1"} {"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 2"} {"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 3"} @@ -441,7 +441,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/ -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro {"f1":"This is a message sent with RBAC SASL/PLAIN authentication 1"} {"f1":"This is a message sent with RBAC SASL/PLAIN authentication 2"} {"f1":"This is a message sent with RBAC SASL/PLAIN authentication 3"} diff --git a/connect/connect-gcp-gcs-sink/gcs-sink.sh b/connect/connect-gcp-gcs-sink/gcs-sink.sh index d581f3787..56100b074 100755 --- a/connect/connect-gcp-gcs-sink/gcs-sink.sh +++ b/connect/connect-gcp-gcs-sink/gcs-sink.sh @@ -92,6 +92,6 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs: log "Getting one of the avro files locally and displaying content with avro-tools" docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro docker rm -f gcloud-config \ No newline at end of file diff --git a/connect/connect-gcp-gcs-source/gcs-source-backup-and-restore.sh b/connect/connect-gcp-gcs-source/gcs-source-backup-and-restore.sh index a8e92d841..07c74c334 100755 --- a/connect/connect-gcp-gcs-source/gcs-source-backup-and-restore.sh +++ b/connect/connect-gcp-gcs-source/gcs-source-backup-and-restore.sh @@ -106,7 +106,7 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs: log "Getting one of the avro files locally and displaying content with avro-tools" docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro docker rm -f gcloud-config diff --git a/connect/connect-hdfs2-sink/hdfs2-sink-ha-kerberos.sh b/connect/connect-hdfs2-sink/hdfs2-sink-ha-kerberos.sh index 889d558e0..564d0259b 100755 --- a/connect/connect-hdfs2-sink/hdfs2-sink-ha-kerberos.sh +++ b/connect/connect-hdfs2-sink/hdfs2-sink-ha-kerberos.sh @@ -89,4 +89,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode1 bash -c "kinit -kt /opt/hadoop/etc/hadoop/nn.keytab nn/namenode1.kerberos-demo.local && /opt/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp namenode1:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro diff --git a/connect/connect-hdfs2-sink/hdfs2-sink-kerberos.sh b/connect/connect-hdfs2-sink/hdfs2-sink-kerberos.sh index d4f826bd2..ba9e52305 100755 --- a/connect/connect-hdfs2-sink/hdfs2-sink-kerberos.sh +++ b/connect/connect-hdfs2-sink/hdfs2-sink-kerberos.sh @@ -97,7 +97,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro # renew ticket manually: # docker exec connect kinit -kt /tmp/connect.keytab connect/connect.kerberos-demo.local \ No newline at end of file diff --git a/connect/connect-hdfs2-sink/hdfs2-sink.sh b/connect/connect-hdfs2-sink/hdfs2-sink.sh index ade8fe2d0..6c1de37ad 100755 --- a/connect/connect-hdfs2-sink/hdfs2-sink.sh +++ b/connect/connect-hdfs2-sink/hdfs2-sink.sh @@ -64,7 +64,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode bash -c "/opt/hadoop-2.7.4/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro log "Check data with beeline" docker exec -i hive-server beeline > /tmp/result.log 2>&1 <<-EOF diff --git a/connect/connect-hdfs2-source/hdfs2-source-kerberos.sh b/connect/connect-hdfs2-source/hdfs2-source-kerberos.sh index bad397edc..5e8b2049d 100755 --- a/connect/connect-hdfs2-source/hdfs2-source-kerberos.sh +++ b/connect/connect-hdfs2-source/hdfs2-source-kerberos.sh @@ -101,7 +101,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp" docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro # renew ticket manually: # docker exec connect kinit -kt /tmp/connect.keytab connect/connect.kerberos.local diff --git a/connect/connect-hdfs2-source/hdfs2-source.sh b/connect/connect-hdfs2-source/hdfs2-source.sh index 25a3058a2..766b12732 100755 --- a/connect/connect-hdfs2-source/hdfs2-source.sh +++ b/connect/connect-hdfs2-source/hdfs2-source.sh @@ -66,7 +66,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode bash -c "/opt/hadoop-2.7.4/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp" docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro log "Check data with beeline" docker exec -i hive-server beeline > /tmp/result.log 2>&1 <<-EOF diff --git a/connect/connect-hdfs3-sink/hdfs3-sink-ha-kerberos.sh b/connect/connect-hdfs3-sink/hdfs3-sink-ha-kerberos.sh index 4315d74e2..04f06569d 100755 --- a/connect/connect-hdfs3-sink/hdfs3-sink-ha-kerberos.sh +++ b/connect/connect-hdfs3-sink/hdfs3-sink-ha-kerberos.sh @@ -85,4 +85,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode1 bash -c "kinit -kt /opt/hadoop/etc/hadoop/nn.keytab nn/namenode1.kerberos-demo.local && /opt/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp namenode1:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro diff --git a/connect/connect-hdfs3-sink/hdfs3-sink-kerberos.sh b/connect/connect-hdfs3-sink/hdfs3-sink-kerberos.sh index ad97313b0..bf4259935 100755 --- a/connect/connect-hdfs3-sink/hdfs3-sink-kerberos.sh +++ b/connect/connect-hdfs3-sink/hdfs3-sink-kerberos.sh @@ -95,4 +95,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro diff --git a/connect/connect-hdfs3-sink/hdfs3-sink.sh b/connect/connect-hdfs3-sink/hdfs3-sink.sh index ab4f44af9..37b9e13e1 100755 --- a/connect/connect-hdfs3-sink/hdfs3-sink.sh +++ b/connect/connect-hdfs3-sink/hdfs3-sink.sh @@ -61,7 +61,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode bash -c "/opt/hadoop-3.1.3/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp" docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro sleep 60 diff --git a/connect/connect-hdfs3-source/hdfs3-source-kerberos.sh b/connect/connect-hdfs3-source/hdfs3-source-kerberos.sh index 1024f0be3..ac8265553 100755 --- a/connect/connect-hdfs3-source/hdfs3-source-kerberos.sh +++ b/connect/connect-hdfs3-source/hdfs3-source-kerberos.sh @@ -102,7 +102,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp" docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro log "Creating HDFS Source connector" playground connector create-or-update --connector hdfs3-source-kerberos << EOF diff --git a/connect/connect-hdfs3-source/hdfs3-source.sh b/connect/connect-hdfs3-source/hdfs3-source.sh index 4248674cf..c154f49c2 100755 --- a/connect/connect-hdfs3-source/hdfs3-source.sh +++ b/connect/connect-hdfs3-source/hdfs3-source.sh @@ -69,7 +69,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool docker exec namenode bash -c "/opt/hadoop-3.1.3/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp" docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro +playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro if version_gt $TAG_BASE "5.9.0" || [[ "$TAG" = *ubi8 ]] then diff --git a/connect/connect-minio-s3-sink/README.md b/connect/connect-minio-s3-sink/README.md index 03c1a7b09..a6434ff60 100644 --- a/connect/connect-minio-s3-sink/README.md +++ b/connect/connect-minio-s3-sink/README.md @@ -83,7 +83,7 @@ Getting one of the avro files locally and displaying content with avro-tools: ```bash $ docker container restart copy-files $ docker container logs --tail=3 copy-files -$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/minio_topic+0+0000000000.avro +$ playground tools read-avro-file --file /tmp/minio_topic+0+0000000000.avro ``` ```json diff --git a/connect/connect-minio-s3-sink/minio-sink.sh b/connect/connect-minio-s3-sink/minio-sink.sh index fdba5923b..f89606abf 100755 --- a/connect/connect-minio-s3-sink/minio-sink.sh +++ b/connect/connect-minio-s3-sink/minio-sink.sh @@ -62,4 +62,4 @@ docker container logs --tail=4 list-buckets log "Getting one of the avro files locally and displaying content with avro-tools" docker container restart copy-files docker container logs --tail=3 copy-files -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/minio_topic+0+0000000000.avro +playground tools read-avro-file --file /tmp/minio_topic+0+0000000000.avro diff --git a/connect/connect-sftp-sink/README.md b/connect/connect-sftp-sink/README.md index d0d4ee813..868cb9d67 100644 --- a/connect/connect-sftp-sink/README.md +++ b/connect/connect-sftp-sink/README.md @@ -90,7 +90,7 @@ test_sftp_sink+0+0000000000.avro test_sftp_sink+0+0000000003.avro test_sftp_si ``` ```bash -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro {"f1":"value1"} {"f1":"value2"} {"f1":"value3"} diff --git a/connect/connect-sftp-sink/sftp-sink-kerberos.sh b/connect/connect-sftp-sink/sftp-sink-kerberos.sh index db464a36c..28ab277d4 100755 --- a/connect/connect-sftp-sink/sftp-sink-kerberos.sh +++ b/connect/connect-sftp-sink/sftp-sink-kerberos.sh @@ -95,4 +95,4 @@ docker exec ssh-server bash -c "ls /home/sshuser/upload/topics/test_sftp_sink/pa docker cp ssh-server:/home/sshuser/upload/topics/test_sftp_sink/partition\=0/test_sftp_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file diff --git a/connect/connect-sftp-sink/sftp-sink.sh b/connect/connect-sftp-sink/sftp-sink.sh index 2f7d59d65..989d64a2c 100755 --- a/connect/connect-sftp-sink/sftp-sink.sh +++ b/connect/connect-sftp-sink/sftp-sink.sh @@ -52,4 +52,4 @@ docker exec sftp-server bash -c "ls /home/foo/upload/topics/test_sftp_sink/parti docker cp sftp-server:/home/foo/upload/topics/test_sftp_sink/partition\=0/test_sftp_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file diff --git a/other/connect-override-policy-sftp-sink/README.md b/other/connect-override-policy-sftp-sink/README.md index 22f4a2faf..025f9bca3 100644 --- a/other/connect-override-policy-sftp-sink/README.md +++ b/other/connect-override-policy-sftp-sink/README.md @@ -106,7 +106,7 @@ test_sftp_sink+0+0000000000.avro test_sftp_sink+0+0000000003.avro test_sftp_si ``` ```bash -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro {"f1":"value1"} {"f1":"value2"} {"f1":"value3"} diff --git a/other/connect-override-policy-sftp-sink/sftp-sink.sh b/other/connect-override-policy-sftp-sink/sftp-sink.sh index 1fa1eaac7..4c28b5bbf 100755 --- a/other/connect-override-policy-sftp-sink/sftp-sink.sh +++ b/other/connect-override-policy-sftp-sink/sftp-sink.sh @@ -73,4 +73,4 @@ docker exec sftp-server bash -c "ls /home/foo/upload/topics/test_sftp_sink/parti docker cp sftp-server:/home/foo/upload/topics/test_sftp_sink/partition\=0/test_sftp_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro \ No newline at end of file diff --git a/reproduction-models b/reproduction-models index da6f6bdb9..9f065d869 160000 --- a/reproduction-models +++ b/reproduction-models @@ -1 +1 @@ -Subproject commit da6f6bdb972a5efe89d1a62f992a958bcdf0a344 +Subproject commit 9f065d86991b0245b513e6cc90895872a44d8c9e diff --git a/troubleshooting/control-center-error-500-missing-connector/start.sh b/troubleshooting/control-center-error-500-missing-connector/start.sh index 957c44897..6f366d643 100755 --- a/troubleshooting/control-center-error-500-missing-connector/start.sh +++ b/troubleshooting/control-center-error-500-missing-connector/start.sh @@ -98,7 +98,7 @@ docker exec sftp-server bash -c "ls /home/foo/upload/topics/test_sftp_sink/parti docker cp sftp-server:/home/foo/upload/topics/test_sftp_sink/partition\=0/test_sftp_sink+0+0000000000.avro /tmp/ -docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro +playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro log "Remove /usr/share/confluent-hub-components/jcustenborder-kafka-connect-spooldir"