Merge pull request #5928 from confluentinc/26092024

updates
vdesabou · Sep 26, 2024 · 0ee38ba · 0ee38ba
2 parents a66f5bc + 7688de1
commit 0ee38ba
Show file tree

Hide file tree

Showing 19 changed files with 855 additions and 502 deletions.
diff --git a/connect/connect-jdbc-sybase-sink/sybase-sink.sh b/connect/connect-jdbc-sybase-sink/sybase-sink.sh
@@ -16,7 +16,8 @@ playground connector create-or-update --connector sybase-sink  << EOF
   "connection.user": "sa",
   "connection.password": "password",
   "topics": "orders",
-  "auto.create": "true"
+  "auto.create": "true",
+  "table.name.format": "master.dbo.orders"
 }
 EOF
 

diff --git a/scripts/cli/completions.bash b/scripts/cli/completions.bash
diff --git a/scripts/cli/confluent-hub-plugin-list.txt b/scripts/cli/confluent-hub-plugin-list.txt
@@ -146,6 +146,7 @@ jcustenborder/kafka-connect-redis
 jcustenborder/kafka-connect-spooldir
 jcustenborder/kafka-connect-transform-common
 jcustenborder/kafka-connect-twitter
+jrndio/jr-source-connector
 juxt/kafka-connect-crux
 kaliy/kafka-connect-rss
 kategmbh/kafka-adapter-for-sap-process-orchestration

diff --git a/scripts/cli/playground b/scripts/cli/playground
diff --git a/scripts/cli/playground.json b/scripts/cli/playground.json
@@ -48,6 +48,18 @@
           "description": "when running a fully managed connector example, it opens the connector in browser\n  \n",
           "usage": "playground config open-ccloud-connector-in-browser COMMAND",
           "options": []
+        },
+        {
+          "name": "container-kill-all-before-run",
+          "description": "when running an example, always call playground container kill-all first. If not set, it will call playground stop instead.\n  \n",
+          "usage": "playground config container-kill-all-before-run [ENABLED]",
+          "options": []
+        },
+        {
+          "name": "check-repo-version",
+          "description": "when running an example, always check if repo version is older than 3 days, if disabled, it will skip this check.\n  \n",
+          "usage": "playground config check-repo-version [ENABLED]",
+          "options": []
         }
       ]
     },
@@ -73,7 +85,7 @@
     },
     {
       "name": "run",
-      "description": "🕹️ Run any example !\n\n🔥 It start an interactive mode where you'll be fully guided !\n\n\n⛅ When running Confluent Cloud (ccloud) example:\n\n  All you have to do is to be already logged in with confluent CLI.\n\n  By default, a new Confluent Cloud environment with a Cluster will be created.\n\n  You can configure the new cluster by setting:\n\n  --cluster-type (or CLUSTER_TYPE environment variable): The type of cluster (possible values: basic, standard and dedicated, default basic)\n  --cluster-cloud (or CLUSTER_CLOUD environment variable): The Cloud provider (possible values: aws, gcp and azure, default aws)\n  --cluster-region (or CLUSTER_REGION environment variable): The Cloud region (use confluent kafka region list to get the list, default eu-west-2 for aws, westeurope for azure and europe-west2 for gcp)\n  --cluster-environment (or ENVIRONMENT environment variable) (optional): The environment id where want your new cluster (example: txxxxx) \n\n  In case you want to use your own existing cluster, you need to setup, in addition to previous ones:\n\n  --cluster-name (or CLUSTER_NAME environment variable): The cluster name\n  --cluster-creds (or CLUSTER_CREDS environment variable): The Kafka api key and secret to use, it should be separated with colon (example: <API_KEY>:<API_KEY_SECRET>)\n  --cluster-schema-registry-creds (or SCHEMA_REGISTRY_CREDS environment variable) (optional, if not set, new one will be created): The Schema Registry api key and secret to use, it should be separated with colon (example: <SR_API_KEY>:<SR_API_KEY_SECRET>)\n\n❗ this command will kill all containers using playground container kill-all\n",
+      "description": "🕹️ Run any example !\n\n🔥 It start an interactive mode where you'll be fully guided !\n\n\n⛅ When running Confluent Cloud (ccloud) example:\n\n  All you have to do is to be already logged in with confluent CLI.\n\n  By default, a new Confluent Cloud environment with a Cluster will be created.\n\n  You can configure the new cluster by setting:\n\n  --cluster-type (or CLUSTER_TYPE environment variable): The type of cluster (possible values: basic, standard and dedicated, default basic)\n  --cluster-cloud (or CLUSTER_CLOUD environment variable): The Cloud provider (possible values: aws, gcp and azure, default aws)\n  --cluster-region (or CLUSTER_REGION environment variable): The Cloud region (use confluent kafka region list to get the list, default eu-west-2 for aws, westeurope for azure and europe-west2 for gcp)\n  --cluster-environment (or ENVIRONMENT environment variable) (optional): The environment id where want your new cluster (example: txxxxx) \n\n  In case you want to use your own existing cluster, you need to setup, in addition to previous ones:\n\n  --cluster-name (or CLUSTER_NAME environment variable): The cluster name\n  --cluster-creds (or CLUSTER_CREDS environment variable): The Kafka api key and secret to use, it should be separated with colon (example: <API_KEY>:<API_KEY_SECRET>)\n  --cluster-schema-registry-creds (or SCHEMA_REGISTRY_CREDS environment variable) (optional, if not set, new one will be created): The Schema Registry api key and secret to use, it should be separated with colon (example: <SR_API_KEY>:<SR_API_KEY_SECRET>)\n\n❗ this command will kill all containers using \"playground container kill-all\", you can disable this by running \"playground config container-kill-all-before-run false\"\n",
       "usage": "playground run [OPTIONS]",
       "options": [
         {

diff --git a/scripts/cli/playground.yaml b/scripts/cli/playground.yaml
@@ -49,6 +49,18 @@ subcommands:
             
         usage: playground config open-ccloud-connector-in-browser COMMAND
         options: []
+      - name: container-kill-all-before-run
+        description: | 
+          when running an example, always call playground container kill-all first. If not set, it will call playground stop instead.
+            
+        usage: playground config container-kill-all-before-run [ENABLED]
+        options: []
+      - name: check-repo-version
+        description: | 
+          when running an example, always check if repo version is older than 3 days, if disabled, it will skip this check.
+            
+        usage: playground config check-repo-version [ENABLED]
+        options: []
   - name: open-ccloud-connector-in-browser
     description: | 
         when running a fully managed connector example, it opens the connector in browser
@@ -92,7 +104,7 @@ subcommands:
           --cluster-creds (or CLUSTER_CREDS environment variable): The Kafka api key and secret to use, it should be separated with colon (example: <API_KEY>:<API_KEY_SECRET>)
           --cluster-schema-registry-creds (or SCHEMA_REGISTRY_CREDS environment variable) (optional, if not set, new one will be created): The Schema Registry api key and secret to use, it should be separated with colon (example: <SR_API_KEY>:<SR_API_KEY_SECRET>)
         
-        ❗ this command will kill all containers using playground container kill-all
+        ❗ this command will kill all containers using "playground container kill-all", you can disable this by running "playground config container-kill-all-before-run false"
     usage: playground run [OPTIONS]
     options:
     - names:

diff --git a/scripts/cli/src/bashly.yml b/scripts/cli/src/bashly.yml
@@ -382,6 +382,29 @@ commands:
         examples:
         - playground config open-ccloud-connector-in-browser browser Safari
 
+  - name: container-kill-all-before-run
+    help: |
+      when running an example, always call playground container kill-all first. If not set, it will call playground stop instead.
+    args:
+    - name: enabled
+      required: false
+      default: "true"
+    examples:
+    - playground config container-kill-all-before-run false
+    - playground config container-kill-all-before-run true
+
+  - name: check-repo-version
+    help: |
+      when running an example, always check if repo version is older than 3 days, if disabled, it will skip this check.
+    args:
+    - name: enabled
+      required: false
+      default: "true"
+    examples:
+    - playground config check-repo-version false
+    - playground config check-repo-version true
+
+
 - name: run
   group: Run
   dependencies:
@@ -411,7 +434,7 @@ commands:
       --cluster-creds (or CLUSTER_CREDS environment variable): The Kafka api key and secret to use, it should be separated with colon (example: <API_KEY>:<API_KEY_SECRET>)
       --cluster-schema-registry-creds (or SCHEMA_REGISTRY_CREDS environment variable) (optional, if not set, new one will be created): The Schema Registry api key and secret to use, it should be separated with colon (example: <SR_API_KEY>:<SR_API_KEY_SECRET>)
 
-    ❗ this command will kill all containers using playground container kill-all
+    ❗ this command will kill all containers using "playground container kill-all", you can disable this by running "playground config container-kill-all-before-run false"
 
   flags:
   - &environment-run

diff --git a/scripts/cli/src/commands/config/check-repo-version.sh b/scripts/cli/src/commands/config/check-repo-version.sh
@@ -0,0 +1,2 @@
+log "📋 configuring check-repo-version with ${args[enabled]}"
+playground config set check-repo-version "${args[enabled]}"
diff --git a/scripts/cli/src/commands/config/container-kill-all-before-run.sh b/scripts/cli/src/commands/config/container-kill-all-before-run.sh
@@ -0,0 +1,2 @@
+log "📋 configuring container-kill-all-before-run with ${args[enabled]}"
+playground config set container-kill-all-before-run "${args[enabled]}"
diff --git a/scripts/cli/src/commands/connector/create-or-update.sh b/scripts/cli/src/commands/connector/create-or-update.sh
@@ -200,7 +200,7 @@ then
             echo "EOF" >> $tmp_dir_clipboard/tmp
 
             cat $tmp_dir_clipboard/tmp | pbcopy
-            log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config set clipboard false')"
+            log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config clipboard false')"
         fi
     fi
 fi

diff --git a/scripts/cli/src/commands/connector/select-config.sh b/scripts/cli/src/commands/connector/select-config.sh
@@ -67,7 +67,7 @@ do
             echo "$res" > $tmp_dir/tmp
 
             cat $tmp_dir/tmp | pbcopy
-            log "📋 config has been copied to the clipboard (disable with 'playground config set clipboard false')"
+            log "📋 config has been copied to the clipboard (disable with 'playground config clipboard false')"
         fi
     fi
 done
diff --git a/scripts/cli/src/commands/connector/show-config.sh b/scripts/cli/src/commands/connector/show-config.sh
@@ -62,7 +62,7 @@ do
                 echo "EOF" >> $tmp_dir/tmp
 
                 cat $tmp_dir/tmp | pbcopy
-                log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config set clipboard false')"
+                log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config clipboard false')"
             fi
         fi
     else
@@ -108,7 +108,7 @@ do
                 echo "EOF" >> $tmp_dir/tmp
 
                 cat $tmp_dir/tmp | pbcopy
-                log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config set clipboard false')"
+                log "📋 $connector_type connector config has been copied to the clipboard (disable with 'playground config clipboard false')"
             fi
         fi
     fi

diff --git a/scripts/cli/src/commands/connector/snippets.sh b/scripts/cli/src/commands/connector/snippets.sh
@@ -163,7 +163,7 @@ then
         if [ "$clipboard" == "true" ] || [ "$clipboard" == "" ]
         then
             cat $clipboard_file | pbcopy
-            log "📋 config has been copied to the clipboard (disable with 'playground config set clipboard false')"
+            log "📋 config has been copied to the clipboard (disable with 'playground config clipboard false')"
         fi
     fi
 fi
diff --git a/scripts/cli/src/commands/container/kill-all.sh b/scripts/cli/src/commands/container/kill-all.sh
@@ -1,2 +1,2 @@
-log "💀 Kill all docker containers"
+log "💀 kill all docker containers"
 docker rm -f $(docker ps -qa) > /dev/null 2>&1
diff --git a/scripts/cli/src/commands/get-docker-compose.sh b/scripts/cli/src/commands/get-docker-compose.sh
@@ -11,6 +11,7 @@ then
 fi
 echo "$docker_command" > /tmp/tmp
 sed -e "s|up -d|config|g" \
+    -e "s|--quiet-pull||g" \
     /tmp/tmp > /tmp/playground-command-config
 
 bash /tmp/playground-command-config 
diff --git a/scripts/cli/src/commands/repro/bootstrap.sh b/scripts/cli/src/commands/repro/bootstrap.sh
@@ -1290,7 +1290,7 @@ then
     if [ "$clipboard" == "true" ] || [ "$clipboard" == "" ]
     then
         echo "playground run -f $repro_dir/$repro_test_filename"| pbcopy
-        log "📋 command to run generated example has been copied to the clipboard (disable with 'playground config set clipboard false')"
+        log "📋 command to run generated example has been copied to the clipboard (disable with 'playground config clipboard false')"
     fi
 fi
 

diff --git a/scripts/cli/src/commands/run.sh b/scripts/cli/src/commands/run.sh
@@ -71,6 +71,22 @@ then
   verify_installed "confluent"
 fi
 
+set +e
+container_kill_all_before_run=$(playground config get container-kill-all-before-run)
+if [ "$container_kill_all_before_run" == "" ]
+then
+    playground config set container-kill-all-before-run true
+fi
+
+if [ "$container_kill_all_before_run" == "true" ] || [ "$container_kill_all_before_run" == "" ]
+then
+  log "💀 kill all docker containers (disable with 'playground config container-kill-all-before-run false')"
+  playground container kill-all
+else
+  playground stop
+fi
+set -e
+
 playground state set run.test_file "$test_file"
 test_file_directory="$(dirname "${test_file}")"
 filename=$(basename -- "$test_file")
@@ -1442,9 +1458,7 @@ else
     log "🚀 Running example without any flags"
   fi
 fi
-set +e
-playground container kill-all
-set -e
+
 playground state set run.connector_type "$(get_connector_type | tr -d '\n')"
 playground state set run.test_file "$test_file"
 echo "" >> "$root_folder/playground-run-history"
@@ -1461,7 +1475,7 @@ then
     if [ "$clipboard" == "true" ] || [ "$clipboard" == "" ]
     then
         echo "playground run -f $test_file $flag_list" | pbcopy
-        log "📋 command to run again example has been copied to the clipboard (disable with 'playground config set clipboard false')"
+        log "📋 command to run again example has been copied to the clipboard (disable with 'playground config clipboard false')"
     fi
 fi
 

diff --git a/scripts/cli/src/commands/start-environment.sh b/scripts/cli/src/commands/start-environment.sh
@@ -13,7 +13,19 @@ fi
 test_file_directory="$(dirname "${test_file}")"
 
 set +e
-playground container kill-all
+container_kill_all_before_run=$(playground config get container-kill-all-before-run)
+if [ "$container_kill_all_before_run" == "" ]
+then
+    playground config set container-kill-all-before-run true
+fi
+
+if [ "$container_kill_all_before_run" == "true" ] || [ "$container_kill_all_before_run" == "" ]
+then
+  log "💀 kill all docker containers (disable with 'playground config container-kill-all-before-run false')"
+  playground container kill-all
+else
+  playground stop
+fi
 set -e
 
 if [[ -n "$wait_for_control_center" ]]

diff --git a/scripts/cli/src/lib/utils_function.sh b/scripts/cli/src/lib/utils_function.sh
@@ -445,29 +445,38 @@ function check_bash_version() {
 }
 
 function check_playground_version() {
-  set +e
-  X=3
-  git fetch
-  latest_commit_date=$(git log -1 --format=%cd --date=short)
-  remote_commit_date=$(git log -1 --format=%cd --date=short origin/master)
-
-  if [[ "$OSTYPE" == "darwin"* ]]
+  check_repo_version=$(playground config get check-repo-version)
+  if [ "$check_repo_version" == "" ]
   then
-    latest_commit_date_seconds=$(date -j -f "%Y-%m-%d" "$latest_commit_date" +%s)
-    remote_commit_date_seconds=$(date -j -f "%Y-%m-%d" "$remote_commit_date" +%s)
-  else
-    latest_commit_date_seconds=$(date -d "$latest_commit_date" +%s)
-    remote_commit_date_seconds=$(date -d "$remote_commit_date" +%s)
+      playground config set check-repo-version true
   fi
 
-  difference=$(( (remote_commit_date_seconds - latest_commit_date_seconds) / (60*60*24) ))
-
-  if [ $difference -gt $X ]
+  if [ "$check_repo_version" == "true" ] || [ "$check_repo_version" == "" ]
   then
-      logwarn "🥶 The current repo version is older than $X days ($difference days), please refresh your version using git pull !"
-      check_if_continue
+    set +e
+    X=3
+    git fetch
+    latest_commit_date=$(git log -1 --format=%cd --date=short)
+    remote_commit_date=$(git log -1 --format=%cd --date=short origin/master)
+
+    if [[ "$OSTYPE" == "darwin"* ]]
+    then
+      latest_commit_date_seconds=$(date -j -f "%Y-%m-%d" "$latest_commit_date" +%s)
+      remote_commit_date_seconds=$(date -j -f "%Y-%m-%d" "$remote_commit_date" +%s)
+    else
+      latest_commit_date_seconds=$(date -d "$latest_commit_date" +%s)
+      remote_commit_date_seconds=$(date -d "$remote_commit_date" +%s)
+    fi
+
+    difference=$(( (remote_commit_date_seconds - latest_commit_date_seconds) / (60*60*24) ))
+
+    if [ $difference -gt $X ]
+    then
+        logwarn "🥶 The current repo version is older than $X days ($difference days), please refresh your version using git pull ! (disable with 'playground config check-repo-version false')"
+        check_if_continue
+    fi
+    set -e
   fi
-  set -e
 }
 
 function set_profiles() {