Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
vdesabou committed Jun 6, 2024
1 parent 3c016bf commit d8f54af
Show file tree
Hide file tree
Showing 36 changed files with 41 additions and 41 deletions.
2 changes: 1 addition & 1 deletion ccloud/fm-aws-s3-sink/fully-managed-s3-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ aws s3 cp --only-show-errors --recursive s3://$AWS_BUCKET_NAME/$TAG/s3_topic /tm

cp /tmp/s3_topic/*/*/*/*/s3_topic+0+0000000000.avro /tmp/s3_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp/:/tmp vdesabou/avro-tools tojson /tmp/s3_topic+0+0000000000.avro | grep value999
playground tools read-avro-file --file /tmp/s3_topic+0+0000000000.avro | grep value999

log "Do you want to delete the fully managed connector $connector_name ?"
check_if_continue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ playground connector show-lag --connector $connector_name
# log "Getting one of the avro files locally and displaying content with avro-tools"
# az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/year=2024/month=04/day=04/hour=08/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro

# docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro
# playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro


log "Do you want to delete the fully managed connector $connector_name ?"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ playground connector show-lag --connector $connector_name
# log "Getting one of the avro files locally and displaying content with avro-tools"
# az storage blob download --container-name topics --name datalake_topic/partition=0/datalake_topic+0+0000000000.avro --file /tmp/datalake_topic+0+0000000000.avro --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}"

# docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro
# playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro

log "Do you want to delete the fully managed connector $connector_name ?"
check_if_continue
Expand Down
2 changes: 1 addition & 1 deletion ccloud/fm-gcp-gcs-sink/fully-managed-gcs-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs:
log "Getting one of the avro files locally and displaying content with avro-tools"
docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/*/*/*/*/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro

log "Do you want to delete the fully managed connector $connector_name ?"
check_if_continue
Expand Down
2 changes: 1 addition & 1 deletion ccloud/fm-sftp-sink/fully-managed-sftp-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ log "Listing content of ./upload/topics/test_sftp_sink"
docker exec sftp-server bash -c "ls /home/foo/upload/topics/test_sftp_sink/*/*/*/*"
docker exec sftp-server bash -c "cp /home/foo/upload/topics/test_sftp_sink/*/*/*/*/test_sftp_sink+0+0000000000.avro /tmp/"
docker cp sftp-server:/tmp/test_sftp_sink+0+0000000000.avro /tmp/
docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro
playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro


log "Do you want to delete the fully managed connector $connector_name ?"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,4 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${
log "Getting one of the avro files locally and displaying content with avro-tools"
az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,4 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${
log "Getting one of the avro files locally and displaying content with avro-tools"
az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${
log "Getting one of the avro files locally and displaying content with avro-tools"
az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro


log "Creating Azure Blob Storage Source connector"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ az storage fs file list --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${
log "Getting one of the avro files locally and displaying content with avro-tools"
az storage blob download --account-name "${AZURE_ACCOUNT_NAME}" --account-key "${AZURE_ACCOUNT_KEY}" --container-name "${AZURE_CONTAINER_NAME}" --name topics/blob_topic/partition=0/blob_topic+0+0000000000.avro --file /tmp/blob_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/blob_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/blob_topic+0+0000000000.avro


log "Creating Azure Blob Storage Source connector"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -134,4 +134,4 @@ az dls fs list --account "${AZURE_DATALAKE_ACCOUNT_NAME}" --path /topics
log "Getting one of the avro files locally and displaying content with avro-tools"
az dls fs download --account "${AZURE_DATALAKE_ACCOUNT_NAME}" --overwrite --source-path /topics/datalake_topic/partition=0/datalake_topic+0+0000000000.avro --destination-path /tmp/datalake_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ az storage fs file list --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}" --file-s
log "Getting one of the avro files locally and displaying content with avro-tools"
az storage blob download --container-name topics --name datalake_topic/partition=0/datalake_topic+0+0000000000.avro --file /tmp/datalake_topic+0+0000000000.avro --account-name "${AZURE_DATALAKE_ACCOUNT_NAME}"

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/datalake_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/datalake_topic+0+0000000000.avro

# keep AD app
# log "Deleting active directory app"
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-ftps-sink/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ test_ftps_sink+0+0000000009.avro
```
```bash
docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_ftps_sink+0+0000000000.avro
playground tools read-avro-file --file /tmp/test_ftps_sink+0+0000000000.avro
{"f1":"value1"}
```
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-ftps-sink/ftps-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,4 @@ docker exec ftps-server bash -c "ls /home/vsftpd/bob/test_ftps_sink/partition\=0

docker cp ftps-server:/home/vsftpd/bob/test_ftps_sink/partition\=0/test_ftps_sink+0+0000000000.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_ftps_sink+0+0000000000.avro
playground tools read-avro-file --file /tmp/test_ftps_sink+0+0000000000.avro
2 changes: 1 addition & 1 deletion connect/connect-gcp-dataproc-sink/gcp-dataproc-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ sleep 10
# docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/


docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro

docker rm -f gcloud-config

Expand Down
12 changes: 6 additions & 6 deletions connect/connect-gcp-gcs-sink/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro
19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
{"f1":"value1"}
{"f1":"value2"}
Expand Down Expand Up @@ -215,7 +215,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic-ssl/partition=0/gcs_topic-ssl+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic-ssl+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/gcs_topic-ssl+0+0000000000.avro
19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
{"f1":"This is a message sent with SSL authentication 1"}
{"f1":"This is a message sent with SSL authentication 2"}
Expand Down Expand Up @@ -270,7 +270,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic-sasl-ssl/partition=0/gcs_topic-sasl-ssl+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic-sasl-ssl+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/gcs_topic-sasl-ssl+0+0000000000.avro
19/09/30 16:48:13 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
{"f1":"This is a message sent with SASL_SSL authentication 1"}
{"f1":"This is a message sent with SASL_SSL authentication 2"}
Expand Down Expand Up @@ -325,7 +325,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro
{"f1":"This is a message sent with Kerberos GSSAPI authentication 1"}
{"f1":"This is a message sent with Kerberos GSSAPI authentication 2"}
{"f1":"This is a message sent with Kerberos GSSAPI authentication 3"}
Expand Down Expand Up @@ -384,7 +384,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro
{"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 1"}
{"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 2"}
{"f1":"This is a message sent with LDAP Authorizer SASL/PLAIN authentication 3"}
Expand Down Expand Up @@ -441,7 +441,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ gsutil cp gs://$GCS_BUCKET_NAME/topics/rbac_gcs_topic/partition=0/rbac_gcs_topic+0+0000000000.avro /tmp/
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/rbac_gcs_topic+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/rbac_gcs_topic+0+0000000000.avro
{"f1":"This is a message sent with RBAC SASL/PLAIN authentication 1"}
{"f1":"This is a message sent with RBAC SASL/PLAIN authentication 2"}
{"f1":"This is a message sent with RBAC SASL/PLAIN authentication 3"}
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-gcp-gcs-sink/gcs-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,6 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs:
log "Getting one of the avro files locally and displaying content with avro-tools"
docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro

docker rm -f gcloud-config
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ docker run -i --volumes-from gcloud-config google/cloud-sdk:latest gsutil ls gs:
log "Getting one of the avro files locally and displaying content with avro-tools"
docker run -i --volumes-from gcloud-config -v /tmp:/tmp/ google/cloud-sdk:latest gsutil cp gs://$GCS_BUCKET_NAME/topics/gcs_topic/partition=0/gcs_topic+0+0000000000.avro /tmp/gcs_topic+0+0000000000.avro

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/gcs_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/gcs_topic+0+0000000000.avro

docker rm -f gcloud-config

Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs2-sink/hdfs2-sink-ha-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -89,4 +89,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode1 bash -c "kinit -kt /opt/hadoop/etc/hadoop/nn.keytab nn/namenode1.kerberos-demo.local && /opt/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp namenode1:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro
2 changes: 1 addition & 1 deletion connect/connect-hdfs2-sink/hdfs2-sink-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro

# renew ticket manually:
# docker exec connect kinit -kt /tmp/connect.keytab connect/connect.kerberos-demo.local
2 changes: 1 addition & 1 deletion connect/connect-hdfs2-sink/hdfs2-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode bash -c "/opt/hadoop-2.7.4/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro

log "Check data with beeline"
docker exec -i hive-server beeline > /tmp/result.log 2>&1 <<-EOF
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs2-source/hdfs2-source-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp"
docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro

# renew ticket manually:
# docker exec connect kinit -kt /tmp/connect.keytab connect/connect.kerberos.local
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs2-source/hdfs2-source.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode bash -c "/opt/hadoop-2.7.4/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp"
docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro

log "Check data with beeline"
docker exec -i hive-server beeline > /tmp/result.log 2>&1 <<-EOF
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs3-sink/hdfs3-sink-ha-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,4 +85,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode1 bash -c "kinit -kt /opt/hadoop/etc/hadoop/nn.keytab nn/namenode1.kerberos-demo.local && /opt/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp namenode1:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro
2 changes: 1 addition & 1 deletion connect/connect-hdfs3-sink/hdfs3-sink-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,4 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro
2 changes: 1 addition & 1 deletion connect/connect-hdfs3-sink/hdfs3-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode bash -c "/opt/hadoop-3.1.3/bin/hadoop fs -copyToLocal /topics/test_hdfs/partition=0/test_hdfs+0+0000000000+0000000002.avro /tmp"
docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000002.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000002.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000002.avro


sleep 60
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs3-source/hdfs3-source-kerberos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec hadoop bash -c "/usr/local/hadoop/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp"
docker cp hadoop:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro

log "Creating HDFS Source connector"
playground connector create-or-update --connector hdfs3-source-kerberos << EOF
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-hdfs3-source/hdfs3-source.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ log "Getting one of the avro files locally and displaying content with avro-tool
docker exec namenode bash -c "/opt/hadoop-3.1.3/bin/hadoop fs -copyToLocal /topics/test_hdfs/f1=value1/test_hdfs+0+0000000000+0000000000.avro /tmp"
docker cp namenode:/tmp/test_hdfs+0+0000000000+0000000000.avro /tmp/

docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_hdfs+0+0000000000+0000000000.avro
playground tools read-avro-file --file /tmp/test_hdfs+0+0000000000+0000000000.avro

if version_gt $TAG_BASE "5.9.0" || [[ "$TAG" = *ubi8 ]]
then
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-minio-s3-sink/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ Getting one of the avro files locally and displaying content with avro-tools:
```bash
$ docker container restart copy-files
$ docker container logs --tail=3 copy-files
$ docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/minio_topic+0+0000000000.avro
$ playground tools read-avro-file --file /tmp/minio_topic+0+0000000000.avro
```

```json
Expand Down
2 changes: 1 addition & 1 deletion connect/connect-minio-s3-sink/minio-sink.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,4 @@ docker container logs --tail=4 list-buckets
log "Getting one of the avro files locally and displaying content with avro-tools"
docker container restart copy-files
docker container logs --tail=3 copy-files
docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/minio_topic+0+0000000000.avro
playground tools read-avro-file --file /tmp/minio_topic+0+0000000000.avro
2 changes: 1 addition & 1 deletion connect/connect-sftp-sink/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ test_sftp_sink+0+0000000000.avro test_sftp_sink+0+0000000003.avro test_sftp_si
```
```bash
docker run --quiet --rm -v /tmp:/tmp vdesabou/avro-tools tojson /tmp/test_sftp_sink+0+0000000000.avro
playground tools read-avro-file --file /tmp/test_sftp_sink+0+0000000000.avro
{"f1":"value1"}
{"f1":"value2"}
{"f1":"value3"}
Expand Down
Loading

0 comments on commit d8f54af

Please sign in to comment.