Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaohanzhan-db committed Dec 2, 2023
1 parent 2bb208b commit 8e2a29c
Showing 1 changed file with 0 additions and 14 deletions.
14 changes: 0 additions & 14 deletions scripts/data_prep/convert_delta_to_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,6 @@

log = logging.getLogger(__name__)

"""
Sample tables are created here
- E2-dogfood: https://e2-dogfood.staging.cloud.databricks.com/?o=6051921418418893#notebook/3642707736157009/command/551761898400018
- Data Force One: https://dbc-559ffd80-2bfc.cloud.databricks.com/?o=7395834863327820#notebook/2500382962301597/command/2500382962301599
The script can be called as:
- python scripts/data_prep/convert_delta_to_json.py --delta_table_name 'main.streaming.dummy_table' --json_output_path /tmp/delta2json2 --debug False --http_path 'sql/protocolv1/o/7395834863327820/1116-234530-6seh113n'
- python scripts/data_prep/convert_delta_to_json.py --delta_table_name 'main.streaming.dummy_table' --json_output_path /tmp/delta2json2 --debug False --http_path {http_path} --DATABRICKS_HOST {your host} --DATABRICKS_TOKEN {your token}
"""

def stream_delta_to_json(connection, tablename, json_output_folder, key = 'name', batch_size=1<<20):

cursor = connection.cursor()
Expand Down

0 comments on commit 8e2a29c

Please sign in to comment.